Downloading crates ...
  Downloaded similar v2.2.1
  Downloaded tokio-test v0.4.2
  Downloaded quick-error v1.2.3
  Downloaded newline-converter v0.3.0
  Downloaded wait-timeout v0.2.0
  Downloaded unarray v0.1.4
  Downloaded tokio-stream v0.1.8
  Downloaded structmeta v0.2.0
  Downloaded httptest v0.15.4
  Downloaded test-strategy v0.3.1
  Downloaded subprocess v0.2.9
  Downloaded convert_case v0.5.0
  Downloaded bit-vec v0.6.3
  Downloaded bit-set v0.5.3
  Downloaded structmeta-derive v0.2.0
  Downloaded regex-automata v0.1.10
  Downloaded expectorate v1.0.7
  Downloaded proptest v1.2.0
  Downloaded bstr v0.2.17
  Downloaded rand_xorshift v0.3.0
  Downloaded rusty-fork v0.3.0
   Compiling unicode-ident v1.0.11
   Compiling cfg-if v1.0.0
       Fresh autocfg v1.1.0
       Fresh version_check v0.9.4
     Running `rustc --crate-name unicode_ident --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/unicode-ident-1.0.11/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=81e7752fff89e70f -C extra-filename=-81e7752fff89e70f --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
     Running `rustc --crate-name cfg_if --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/cfg-if-1.0.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=a93276574f0edf39 -C extra-filename=-a93276574f0edf39 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling libc v0.2.147
   Compiling once_cell v1.14.0
     Running `rustc --crate-name libc /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/libc-0.2.147/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="extra_traits"' --cfg 'feature="std"' -C metadata=a748caf4ceff51bd -C extra-filename=-a748caf4ceff51bd --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow --cfg freebsd11 --cfg libc_priv_mod_use --cfg libc_union --cfg libc_const_size_of --cfg libc_align --cfg libc_int128 --cfg libc_core_cvoid --cfg libc_packedN --cfg libc_cfg_target_vendor --cfg libc_non_exhaustive --cfg libc_long_array --cfg libc_ptr_addr_of --cfg libc_underscore_const_names --cfg libc_const_extern_fn`
     Running `rustc --crate-name once_cell --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/once_cell-1.14.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="alloc"' --cfg 'feature="default"' --cfg 'feature="race"' --cfg 'feature="std"' --cfg 'feature="unstable"' -C metadata=fdb5f9e769d1e589 -C extra-filename=-fdb5f9e769d1e589 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling itoa v1.0.1
     Running `rustc --crate-name itoa --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/itoa-1.0.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=f0781104e344570e -C extra-filename=-f0781104e344570e --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling memchr v2.5.0
   Compiling libm v0.2.6
     Running `rustc --crate-name memchr --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/memchr-2.5.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=9611c546f9b73cca -C extra-filename=-9611c546f9b73cca --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow --cfg memchr_runtime_simd --cfg memchr_runtime_sse2 --cfg memchr_runtime_sse42 --cfg memchr_runtime_avx`
   Compiling scopeguard v1.1.0
     Running `rustc --crate-name libm --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/libm-0.2.6/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' -C metadata=e5a688f05412e317 -C extra-filename=-e5a688f05412e317 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
     Running `rustc --crate-name scopeguard /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/scopeguard-1.1.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=00b7ece4eb7b8e7e -C extra-filename=-00b7ece4eb7b8e7e --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling log v0.4.14
     Running `rustc --crate-name log /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/log-0.4.14/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="std"' -C metadata=930e200cffaa7930 -C extra-filename=-930e200cffaa7930 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern cfg_if=/work/oxidecomputer/crucible/target/debug/deps/libcfg_if-a93276574f0edf39.rmeta --cap-lints allow --cfg atomic_cas --cfg has_atomics`
   Compiling proc-macro2 v1.0.63
     Running `rustc --crate-name proc_macro2 --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/proc-macro2-1.0.63/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="proc-macro"' -C metadata=a0e1129b171da08c -C extra-filename=-a0e1129b171da08c --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern unicode_ident=/work/oxidecomputer/crucible/target/debug/deps/libunicode_ident-81e7752fff89e70f.rmeta --cap-lints allow --cfg wrap_proc_macro`
   Compiling pin-project-lite v0.2.13
     Running `rustc --crate-name pin_project_lite --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/pin-project-lite-0.2.13/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=c035e320730303c2 -C extra-filename=-c035e320730303c2 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling smallvec v1.10.0
     Running `rustc --crate-name smallvec --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/smallvec-1.10.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=397f26bd8c84e528 -C extra-filename=-397f26bd8c84e528 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling lock_api v0.4.6
     Running `rustc --crate-name lock_api --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/lock_api-0.4.6/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=4b01d37c549347e9 -C extra-filename=-4b01d37c549347e9 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern scopeguard=/work/oxidecomputer/crucible/target/debug/deps/libscopeguard-00b7ece4eb7b8e7e.rmeta --cap-lints allow`
   Compiling ryu v1.0.9
     Running `rustc --crate-name ryu --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/ryu-1.0.9/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=062e5ac4087417b3 -C extra-filename=-062e5ac4087417b3 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling futures-core v0.3.28
     Running `rustc --crate-name futures_core --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-core-0.3.28/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="alloc"' --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=46c6e3a1b3966417 -C extra-filename=-46c6e3a1b3966417 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling pin-utils v0.1.0
     Running `rustc --crate-name pin_utils --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/pin-utils-0.1.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=bcfb754cd1ab8c67 -C extra-filename=-bcfb754cd1ab8c67 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
       Fresh pkg-config v0.3.24
   Compiling bitflags v1.3.2
     Running `rustc --crate-name bitflags --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/bitflags-1.3.2/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' -C metadata=b5bc15fb96af61fc -C extra-filename=-b5bc15fb96af61fc --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling fnv v1.0.7
     Running `rustc --crate-name fnv /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/fnv-1.0.7/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=5641130f60a8056b -C extra-filename=-5641130f60a8056b --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling futures-sink v0.3.28
     Running `rustc --crate-name futures_sink --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-sink-0.3.28/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="alloc"' --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=a6d6ed8a846c5f8a -C extra-filename=-a6d6ed8a846c5f8a --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling quote v1.0.29
     Running `rustc --crate-name quote --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/quote-1.0.29/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="proc-macro"' -C metadata=2ffbaa80dd156621 -C extra-filename=-2ffbaa80dd156621 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rmeta --cap-lints allow`
   Compiling num-traits v0.2.16
     Running `rustc --crate-name num_traits --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/num-traits-0.2.16/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="libm"' --cfg 'feature="std"' -C metadata=8e50de91aba3f8f9 -C extra-filename=-8e50de91aba3f8f9 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern libm=/work/oxidecomputer/crucible/target/debug/deps/liblibm-e5a688f05412e317.rmeta --cap-lints allow --cfg has_to_int_unchecked --cfg has_reverse_bits --cfg has_leading_trailing_ones --cfg has_div_euclid --cfg has_copysign --cfg has_is_subnormal --cfg has_int_to_from_bytes --cfg has_float_to_from_bytes`
   Compiling futures-task v0.3.28
     Running `rustc --crate-name futures_task --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-task-0.3.28/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="alloc"' --cfg 'feature="std"' -C metadata=12b58f257ddc96a4 -C extra-filename=-12b58f257ddc96a4 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling futures-channel v0.3.28
     Running `rustc --crate-name futures_channel --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-channel-0.3.28/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="alloc"' --cfg 'feature="default"' --cfg 'feature="futures-sink"' --cfg 'feature="sink"' --cfg 'feature="std"' -C metadata=34a7a018f58dc5a7 -C extra-filename=-34a7a018f58dc5a7 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern futures_core=/work/oxidecomputer/crucible/target/debug/deps/libfutures_core-46c6e3a1b3966417.rmeta --extern futures_sink=/work/oxidecomputer/crucible/target/debug/deps/libfutures_sink-a6d6ed8a846c5f8a.rmeta --cap-lints allow`
   Compiling syn v2.0.23
     Running `rustc --crate-name syn --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/syn-2.0.23/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="clone-impls"' --cfg 'feature="default"' --cfg 'feature="derive"' --cfg 'feature="extra-traits"' --cfg 'feature="full"' --cfg 'feature="parsing"' --cfg 'feature="printing"' --cfg 'feature="proc-macro"' --cfg 'feature="quote"' --cfg 'feature="visit"' --cfg 'feature="visit-mut"' -C metadata=baedf68a9175a6da -C extra-filename=-baedf68a9175a6da --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rmeta --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rmeta --extern unicode_ident=/work/oxidecomputer/crucible/target/debug/deps/libunicode_ident-81e7752fff89e70f.rmeta --cap-lints allow`
   Compiling getrandom v0.2.5
     Running `rustc --crate-name getrandom --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/getrandom-0.2.5/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="std"' -C metadata=567199de146d617e -C extra-filename=-567199de146d617e --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern cfg_if=/work/oxidecomputer/crucible/target/debug/deps/libcfg_if-a93276574f0edf39.rmeta --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --cap-lints allow`
   Compiling syn v1.0.107
     Running `rustc --crate-name syn --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/syn-1.0.107/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="clone-impls"' --cfg 'feature="default"' --cfg 'feature="derive"' --cfg 'feature="extra-traits"' --cfg 'feature="full"' --cfg 'feature="parsing"' --cfg 'feature="printing"' --cfg 'feature="proc-macro"' --cfg 'feature="quote"' --cfg 'feature="visit"' --cfg 'feature="visit-mut"' -C metadata=837f9a049f77ca38 -C extra-filename=-837f9a049f77ca38 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rmeta --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rmeta --extern unicode_ident=/work/oxidecomputer/crucible/target/debug/deps/libunicode_ident-81e7752fff89e70f.rmeta --cap-lints allow --cfg syn_disable_nightly_tests`
   Compiling jobserver v0.1.25
     Running `rustc --crate-name jobserver --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/jobserver-0.1.25/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C metadata=0ee11fba78dd3235 -C extra-filename=-0ee11fba78dd3235 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --cap-lints allow`
   Compiling num_cpus v1.13.1
     Running `rustc --crate-name num_cpus /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/num_cpus-1.13.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=67a451bebfcc5086 -C extra-filename=-67a451bebfcc5086 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --cap-lints allow`
   Compiling ahash v0.7.6
     Running `rustc --crate-name ahash --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/ahash-0.7.6/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=c781b32f1fcd8d92 -C extra-filename=-c781b32f1fcd8d92 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern getrandom=/work/oxidecomputer/crucible/target/debug/deps/libgetrandom-567199de146d617e.rmeta --extern once_cell=/work/oxidecomputer/crucible/target/debug/deps/libonce_cell-fdb5f9e769d1e589.rmeta --cap-lints allow --cfg 'feature="runtime-rng"' --cfg 'feature="folded_multiply"'`
   Compiling parking_lot_core v0.9.1
     Running `rustc --crate-name parking_lot_core --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/parking_lot_core-0.9.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=f45d0642d52c20c1 -C extra-filename=-f45d0642d52c20c1 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern cfg_if=/work/oxidecomputer/crucible/target/debug/deps/libcfg_if-a93276574f0edf39.rmeta --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --extern smallvec=/work/oxidecomputer/crucible/target/debug/deps/libsmallvec-397f26bd8c84e528.rmeta --cap-lints allow`
   Compiling cc v1.0.73
     Running `rustc --crate-name cc --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/cc-1.0.73/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no --cfg 'feature="jobserver"' --cfg 'feature="parallel"' -C metadata=2976d4b8f46fa671 -C extra-filename=-2976d4b8f46fa671 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern jobserver=/work/oxidecomputer/crucible/target/debug/deps/libjobserver-0ee11fba78dd3235.rmeta --cap-lints allow`
   Compiling signal-hook-registry v1.4.0
     Running `rustc --crate-name signal_hook_registry /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/signal-hook-registry-1.4.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=4d955479f235827e -C extra-filename=-4d955479f235827e --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --cap-lints allow`
   Compiling parking_lot v0.12.0
     Running `rustc --crate-name parking_lot --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/parking_lot-0.12.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="send_guard"' -C metadata=970d5c0acece447c -C extra-filename=-970d5c0acece447c --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern lock_api=/work/oxidecomputer/crucible/target/debug/deps/liblock_api-4b01d37c549347e9.rmeta --extern parking_lot_core=/work/oxidecomputer/crucible/target/debug/deps/libparking_lot_core-f45d0642d52c20c1.rmeta --cap-lints allow`
   Compiling hashbrown v0.12.3
     Running `rustc --crate-name hashbrown --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/hashbrown-0.12.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="ahash"' --cfg 'feature="default"' --cfg 'feature="inline-more"' --cfg 'feature="raw"' -C metadata=3ad6614047c487f9 -C extra-filename=-3ad6614047c487f9 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern ahash=/work/oxidecomputer/crucible/target/debug/deps/libahash-c781b32f1fcd8d92.rmeta --cap-lints allow`
   Compiling rand_core v0.6.4
     Running `rustc --crate-name rand_core --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/rand_core-0.6.4/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="alloc"' --cfg 'feature="getrandom"' --cfg 'feature="std"' -C metadata=e2870cc0e01c33c9 -C extra-filename=-e2870cc0e01c33c9 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern getrandom=/work/oxidecomputer/crucible/target/debug/deps/libgetrandom-567199de146d617e.rmeta --cap-lints allow`
   Compiling mio v0.8.8
     Running `rustc --crate-name mio --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/mio-0.8.8/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="log"' --cfg 'feature="net"' --cfg 'feature="os-ext"' --cfg 'feature="os-poll"' -C metadata=27a8136cf12de2bb -C extra-filename=-27a8136cf12de2bb --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --extern log=/work/oxidecomputer/crucible/target/debug/deps/liblog-930e200cffaa7930.rmeta --cap-lints allow`
   Compiling percent-encoding v2.3.0
     Running `rustc --crate-name percent_encoding --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/percent-encoding-2.3.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="alloc"' --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=0000aebce3d30803 -C extra-filename=-0000aebce3d30803 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling futures-io v0.3.28
     Running `rustc --crate-name futures_io --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-io-0.3.28/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="std"' -C metadata=bcbbef0c8c581d67 -C extra-filename=-bcbbef0c8c581d67 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling slab v0.4.5
     Running `rustc --crate-name slab --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/slab-0.4.5/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=5b7c79e345d6363e -C extra-filename=-5b7c79e345d6363e --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling socket2 v0.5.3
     Running `rustc --crate-name socket2 --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/socket2-0.5.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="all"' -C metadata=b464b617227db85e -C extra-filename=-b464b617227db85e --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --cap-lints allow`
   Compiling time v0.1.44
   Compiling iana-time-zone v0.1.47
     Running `rustc --crate-name time /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/time-0.1.44/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=fcbf6ea26d482f3a -C extra-filename=-fcbf6ea26d482f3a --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --cap-lints allow`
     Running `rustc --crate-name iana_time_zone --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/iana-time-zone-0.1.47/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="fallback"' -C metadata=3051bc3a975f54c9 -C extra-filename=-3051bc3a975f54c9 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling tinyvec_macros v0.1.0
     Running `rustc --crate-name tinyvec_macros --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tinyvec_macros-0.1.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=71472bb1a287b427 -C extra-filename=-71472bb1a287b427 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling tinyvec v1.5.1
     Running `rustc --crate-name tinyvec --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tinyvec-1.5.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="alloc"' --cfg 'feature="default"' --cfg 'feature="tinyvec_macros"' -C metadata=592495f429dcf8bd -C extra-filename=-592495f429dcf8bd --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern tinyvec_macros=/work/oxidecomputer/crucible/target/debug/deps/libtinyvec_macros-71472bb1a287b427.rmeta --cap-lints allow`
   Compiling unicode-bidi v0.3.7
     Running `rustc --crate-name unicode_bidi --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/unicode-bidi-0.3.7/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=2c825f5beb05a037 -C extra-filename=-2c825f5beb05a037 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling tracing-core v0.1.30
     Running `rustc --crate-name tracing_core --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tracing-core-0.1.30/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="once_cell"' --cfg 'feature="std"' --cfg 'feature="valuable"' -C metadata=adac59f754126e83 -C extra-filename=-adac59f754126e83 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern once_cell=/work/oxidecomputer/crucible/target/debug/deps/libonce_cell-fdb5f9e769d1e589.rmeta --cap-lints allow`
   Compiling ring v0.16.20
     Running `rustc --crate-name build_script_build --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/ring-0.16.20/build.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no --cfg 'feature="alloc"' --cfg 'feature="default"' --cfg 'feature="dev_urandom_fallback"' --cfg 'feature="once_cell"' -C metadata=9674c8d76dda18b5 -C extra-filename=-9674c8d76dda18b5 --out-dir /work/oxidecomputer/crucible/target/debug/build/ring-9674c8d76dda18b5 -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern cc=/work/oxidecomputer/crucible/target/debug/deps/libcc-2976d4b8f46fa671.rlib --cap-lints allow`
   Compiling form_urlencoded v1.2.0
     Running `rustc --crate-name form_urlencoded --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/form_urlencoded-1.2.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="alloc"' --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=ef731295a29c9150 -C extra-filename=-ef731295a29c9150 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern percent_encoding=/work/oxidecomputer/crucible/target/debug/deps/libpercent_encoding-0000aebce3d30803.rmeta --cap-lints allow`
   Compiling dyn-clone v1.0.5
     Running `rustc --crate-name dyn_clone --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dyn-clone-1.0.5/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=6ca330b50294836a -C extra-filename=-6ca330b50294836a --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling untrusted v0.7.1
     Running `rustc --crate-name untrusted --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/untrusted-0.7.1/src/untrusted.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=4b93784238d33e58 -C extra-filename=-4b93784238d33e58 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling matches v0.1.9
     Running `rustc --crate-name matches /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/matches-0.1.9/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=7dd63e8ea065bea3 -C extra-filename=-7dd63e8ea065bea3 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling spin v0.5.2
     Running `rustc --crate-name spin /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/spin-0.5.2/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=bfb6115ad3135235 -C extra-filename=-bfb6115ad3135235 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling base64 v0.21.3
     Running `rustc --crate-name base64 --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/base64-0.21.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=c139bdd129e780ac -C extra-filename=-c139bdd129e780ac --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling unicode-normalization v0.1.19
     Running `rustc --crate-name unicode_normalization --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/unicode-normalization-0.1.19/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=3df8261a03d4248e -C extra-filename=-3df8261a03d4248e --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern tinyvec=/work/oxidecomputer/crucible/target/debug/deps/libtinyvec-592495f429dcf8bd.rmeta --cap-lints allow`
   Compiling lazy_static v1.4.0
     Running `rustc --crate-name lazy_static /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/lazy_static-1.4.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=9c1a21fa7aad0259 -C extra-filename=-9c1a21fa7aad0259 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling heck v0.4.1
     Running `rustc --crate-name heck --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/heck-0.4.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' -C metadata=29af75c938b110f7 -C extra-filename=-29af75c938b110f7 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
     Running `/work/oxidecomputer/crucible/target/debug/build/ring-9674c8d76dda18b5/build-script-build`
   Compiling ppv-lite86 v0.2.16
     Running `rustc --crate-name ppv_lite86 --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/ppv-lite86-0.2.16/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="simd"' --cfg 'feature="std"' -C metadata=a7fd0e8d54744928 -C extra-filename=-a7fd0e8d54744928 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling anyhow v1.0.71
     Running `rustc --crate-name anyhow --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/anyhow-1.0.71/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=c23d7ea2a714731e -C extra-filename=-c23d7ea2a714731e --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling rustls-pemfile v1.0.3
     Running `rustc --crate-name rustls_pemfile --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/rustls-pemfile-1.0.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=e52b2a6debfcae48 -C extra-filename=-e52b2a6debfcae48 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern base64=/work/oxidecomputer/crucible/target/debug/deps/libbase64-c139bdd129e780ac.rmeta --cap-lints allow`
   Compiling openssl-sys v0.9.90
     Running `rustc --crate-name build_script_main --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/openssl-sys-0.9.90/build/main.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no -C metadata=b13a834c02d58f09 -C extra-filename=-b13a834c02d58f09 --out-dir /work/oxidecomputer/crucible/target/debug/build/openssl-sys-b13a834c02d58f09 -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern cc=/work/oxidecomputer/crucible/target/debug/deps/libcc-2976d4b8f46fa671.rlib --extern pkg_config=/work/oxidecomputer/crucible/target/debug/deps/libpkg_config-e05c47386f4bdcc0.rlib --cap-lints allow`
   Compiling rand_chacha v0.3.1
     Running `rustc --crate-name rand_chacha --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/rand_chacha-0.3.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=add466c063ef8725 -C extra-filename=-add466c063ef8725 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern ppv_lite86=/work/oxidecomputer/crucible/target/debug/deps/libppv_lite86-a7fd0e8d54744928.rmeta --extern rand_core=/work/oxidecomputer/crucible/target/debug/deps/librand_core-e2870cc0e01c33c9.rmeta --cap-lints allow`
   Compiling socket2 v0.4.9
     Running `rustc --crate-name socket2 --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/socket2-0.4.9/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="all"' -C metadata=3c3e3607c1c6d64e -C extra-filename=-3c3e3607c1c6d64e --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --cap-lints allow`
   Compiling idna v0.2.3
     Running `rustc --crate-name idna --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/idna-0.2.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=67f12269d91917c9 -C extra-filename=-67f12269d91917c9 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern matches=/work/oxidecomputer/crucible/target/debug/deps/libmatches-7dd63e8ea065bea3.rmeta --extern unicode_bidi=/work/oxidecomputer/crucible/target/debug/deps/libunicode_bidi-2c825f5beb05a037.rmeta --extern unicode_normalization=/work/oxidecomputer/crucible/target/debug/deps/libunicode_normalization-3df8261a03d4248e.rmeta --cap-lints allow`
   Compiling rand v0.8.5
     Running `rustc --crate-name rand --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/rand-0.8.5/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="alloc"' --cfg 'feature="default"' --cfg 'feature="getrandom"' --cfg 'feature="libc"' --cfg 'feature="min_const_gen"' --cfg 'feature="rand_chacha"' --cfg 'feature="small_rng"' --cfg 'feature="std"' --cfg 'feature="std_rng"' -C metadata=1f91a9ea4aed49ee -C extra-filename=-1f91a9ea4aed49ee --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --extern rand_chacha=/work/oxidecomputer/crucible/target/debug/deps/librand_chacha-add466c063ef8725.rmeta --extern rand_core=/work/oxidecomputer/crucible/target/debug/deps/librand_core-e2870cc0e01c33c9.rmeta --cap-lints allow`
     Running `/work/oxidecomputer/crucible/target/debug/build/openssl-sys-b13a834c02d58f09/build-script-main`
   Compiling try-lock v0.2.3
     Running `rustc --crate-name try_lock /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/try-lock-0.2.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=02baba71598f88d3 -C extra-filename=-02baba71598f88d3 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling httparse v1.8.0
     Running `rustc --crate-name httparse --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/httparse-1.8.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=59406412a39ce707 -C extra-filename=-59406412a39ce707 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow --cfg httparse_simd`
   Compiling want v0.3.0
     Running `rustc --crate-name want --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/want-0.3.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=5737a0d118420ef7 -C extra-filename=-5737a0d118420ef7 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern log=/work/oxidecomputer/crucible/target/debug/deps/liblog-930e200cffaa7930.rmeta --extern try_lock=/work/oxidecomputer/crucible/target/debug/deps/libtry_lock-02baba71598f88d3.rmeta --cap-lints allow`
   Compiling httpdate v1.0.2
     Running `rustc --crate-name httpdate --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/httpdate-1.0.2/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=e3ef82a990113a54 -C extra-filename=-e3ef82a990113a54 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling tower-service v0.3.1
     Running `rustc --crate-name tower_service --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tower-service-0.3.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=51da71f2ad5117ee -C extra-filename=-51da71f2ad5117ee --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling url v2.2.2
     Running `rustc --crate-name url --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/url-2.2.2/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=ff56943ab9066fdc -C extra-filename=-ff56943ab9066fdc --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern form_urlencoded=/work/oxidecomputer/crucible/target/debug/deps/libform_urlencoded-ef731295a29c9150.rmeta --extern idna=/work/oxidecomputer/crucible/target/debug/deps/libidna-67f12269d91917c9.rmeta --extern matches=/work/oxidecomputer/crucible/target/debug/deps/libmatches-7dd63e8ea065bea3.rmeta --extern percent_encoding=/work/oxidecomputer/crucible/target/debug/deps/libpercent_encoding-0000aebce3d30803.rmeta --cap-lints allow`
   Compiling foreign-types-shared v0.1.1
     Running `rustc --crate-name foreign_types_shared /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/foreign-types-shared-0.1.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=623b4e0ee39a8057 -C extra-filename=-623b4e0ee39a8057 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling foreign-types v0.3.2
     Running `rustc --crate-name foreign_types /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/foreign-types-0.3.2/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=07949a1c938aca9c -C extra-filename=-07949a1c938aca9c --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern foreign_types_shared=/work/oxidecomputer/crucible/target/debug/deps/libforeign_types_shared-623b4e0ee39a8057.rmeta --cap-lints allow`
   Compiling openssl v0.10.55
     Running `/work/oxidecomputer/crucible/target/debug/build/openssl-754c33330368d7dd/build-script-build`
     Running `rustc --crate-name openssl_sys --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/openssl-sys-0.9.90/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=fe2ced2b5ac7bf5a -C extra-filename=-fe2ced2b5ac7bf5a --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --cap-lints allow -L native=/usr/ssl-3/lib/amd64 -l ssl -l crypto --cfg openssl --cfg 'osslconf="OPENSSL_NO_SSL3_METHOD"' --cfg ossl300 --cfg ossl101 --cfg ossl102 --cfg ossl102f --cfg ossl102h --cfg ossl110 --cfg ossl110f --cfg ossl110g --cfg ossl110h --cfg ossl111 --cfg ossl111b --cfg ossl111c --cfg ossl111d`
   Compiling native-tls v0.2.11
     Running `/work/oxidecomputer/crucible/target/debug/build/native-tls-52d6aeab7008e848/build-script-build`
   Compiling ahash v0.8.3
     Running `rustc --crate-name ahash --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/ahash-0.8.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=0c4a58ad1daaf354 -C extra-filename=-0c4a58ad1daaf354 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern cfg_if=/work/oxidecomputer/crucible/target/debug/deps/libcfg_if-a93276574f0edf39.rmeta --extern once_cell=/work/oxidecomputer/crucible/target/debug/deps/libonce_cell-fdb5f9e769d1e589.rmeta --cap-lints allow --cfg 'feature="folded_multiply"'`
   Compiling typenum v1.15.0
     Running `rustc --crate-name typenum --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/typenum-1.15.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=21543e9c4f362850 -C extra-filename=-21543e9c4f362850 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling serde_derive_internals v0.26.0
     Running `rustc --crate-name serde_derive_internals /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/serde_derive_internals-0.26.0/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C metadata=6aea929a774cf30f -C extra-filename=-6aea929a774cf30f --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rmeta --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rmeta --extern syn=/work/oxidecomputer/crucible/target/debug/deps/libsyn-837f9a049f77ca38.rmeta --cap-lints allow`
   Compiling crossbeam-utils v0.8.8
     Running `rustc --crate-name crossbeam_utils --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/crossbeam-utils-0.8.8/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="lazy_static"' --cfg 'feature="std"' -C metadata=a1fb255bfa31483a -C extra-filename=-a1fb255bfa31483a --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern cfg_if=/work/oxidecomputer/crucible/target/debug/deps/libcfg_if-a93276574f0edf39.rmeta --extern lazy_static=/work/oxidecomputer/crucible/target/debug/deps/liblazy_static-9c1a21fa7aad0259.rmeta --cap-lints allow`
   Compiling unicode-width v0.1.9
     Running `rustc --crate-name unicode_width /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/unicode-width-0.1.9/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' -C metadata=87c5262a4c4bb0e9 -C extra-filename=-87c5262a4c4bb0e9 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling openssl-probe v0.1.5
     Running `rustc --crate-name openssl_probe /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/openssl-probe-0.1.5/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=ef939aeef8872804 -C extra-filename=-ef939aeef8872804 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling hashbrown v0.13.2
     Running `rustc --crate-name hashbrown --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/hashbrown-0.13.2/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="ahash"' --cfg 'feature="default"' --cfg 'feature="inline-more"' -C metadata=d4fb045aef0e24c1 -C extra-filename=-d4fb045aef0e24c1 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern ahash=/work/oxidecomputer/crucible/target/debug/deps/libahash-0c4a58ad1daaf354.rmeta --cap-lints allow`
   Compiling generic-array v0.14.5
     Running `rustc --crate-name generic_array /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/generic-array-0.14.5/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="more_lengths"' -C metadata=bc31b01978a602e7 -C extra-filename=-bc31b01978a602e7 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern typenum=/work/oxidecomputer/crucible/target/debug/deps/libtypenum-21543e9c4f362850.rmeta --cap-lints allow --cfg relaxed_coherence`
   Compiling aho-corasick v1.0.1
     Running `rustc --crate-name aho_corasick --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/aho-corasick-1.0.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="perf-literal"' --cfg 'feature="std"' -C metadata=afc99e972f7e39a1 -C extra-filename=-afc99e972f7e39a1 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern memchr=/work/oxidecomputer/crucible/target/debug/deps/libmemchr-9611c546f9b73cca.rmeta --cap-lints allow`
   Compiling mime v0.3.16
     Running `rustc --crate-name mime /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/mime-0.3.16/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=ac14a9115eddd3c2 -C extra-filename=-ac14a9115eddd3c2 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling byteorder v1.4.3
     Running `rustc --crate-name byteorder --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/byteorder-1.4.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=1306999913c8e1b3 -C extra-filename=-1306999913c8e1b3 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling regex-syntax v0.7.2
     Running `rustc --crate-name regex_syntax --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/regex-syntax-0.7.2/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="std"' --cfg 'feature="unicode"' --cfg 'feature="unicode-age"' --cfg 'feature="unicode-bool"' --cfg 'feature="unicode-case"' --cfg 'feature="unicode-gencat"' --cfg 'feature="unicode-perl"' --cfg 'feature="unicode-script"' --cfg 'feature="unicode-segment"' -C metadata=c9e6cb4ff3b1ed69 -C extra-filename=-c9e6cb4ff3b1ed69 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling regress v0.6.0
     Running `rustc --crate-name regress --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/regress-0.6.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="backend-pikevm"' --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=10da65958da1c830 -C extra-filename=-10da65958da1c830 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern hashbrown=/work/oxidecomputer/crucible/target/debug/deps/libhashbrown-d4fb045aef0e24c1.rmeta --extern memchr=/work/oxidecomputer/crucible/target/debug/deps/libmemchr-9611c546f9b73cca.rmeta --cap-lints allow`
   Compiling encoding_rs v0.8.30
     Running `rustc --crate-name encoding_rs --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/encoding_rs-0.8.30/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="alloc"' --cfg 'feature="default"' -C metadata=3255048793b3f7a6 -C extra-filename=-3255048793b3f7a6 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern cfg_if=/work/oxidecomputer/crucible/target/debug/deps/libcfg_if-a93276574f0edf39.rmeta --cap-lints allow`
   Compiling crossbeam-channel v0.5.3
     Running `rustc --crate-name crossbeam_channel --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/crossbeam-channel-0.5.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="crossbeam-utils"' --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=19ff6fa486e51c63 -C extra-filename=-19ff6fa486e51c63 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern cfg_if=/work/oxidecomputer/crucible/target/debug/deps/libcfg_if-a93276574f0edf39.rmeta --extern crossbeam_utils=/work/oxidecomputer/crucible/target/debug/deps/libcrossbeam_utils-a1fb255bfa31483a.rmeta --cap-lints allow`
   Compiling ipnet v2.4.0
     Running `rustc --crate-name ipnet /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/ipnet-2.4.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' -C metadata=8b250db103a32779 -C extra-filename=-8b250db103a32779 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling webpki-roots v0.25.2
     Running `rustc --crate-name webpki_roots --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/webpki-roots-0.25.2/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=31272bd9a7615638 -C extra-filename=-31272bd9a7615638 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling crypto-common v0.1.6
     Running `rustc --crate-name crypto_common --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/crypto-common-0.1.6/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="getrandom"' --cfg 'feature="rand_core"' --cfg 'feature="std"' -C metadata=0953bfc5dcef84b9 -C extra-filename=-0953bfc5dcef84b9 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern generic_array=/work/oxidecomputer/crucible/target/debug/deps/libgeneric_array-bc31b01978a602e7.rmeta --extern rand_core=/work/oxidecomputer/crucible/target/debug/deps/librand_core-e2870cc0e01c33c9.rmeta --extern typenum=/work/oxidecomputer/crucible/target/debug/deps/libtypenum-21543e9c4f362850.rmeta --cap-lints allow`
   Compiling io-lifetimes v1.0.3
     Running `rustc --crate-name io_lifetimes --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/io-lifetimes-1.0.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="close"' --cfg 'feature="libc"' --cfg 'feature="windows-sys"' -C metadata=df7ee936a2a5cbac -C extra-filename=-df7ee936a2a5cbac --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --cap-lints allow --cfg io_safety_is_in_std --cfg panic_in_const_fn`
   Compiling getopts v0.2.21
     Running `rustc --crate-name getopts /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/getopts-0.2.21/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=93a8419d37acce69 -C extra-filename=-93a8419d37acce69 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern unicode_width=/work/oxidecomputer/crucible/target/debug/deps/libunicode_width-87c5262a4c4bb0e9.rmeta --cap-lints allow`
       Fresh unsafe-libyaml v0.2.5
   Compiling ucd-trie v0.1.3
     Running `rustc --crate-name ucd_trie --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/ucd-trie-0.1.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=a4e8188b5963a3f1 -C extra-filename=-a4e8188b5963a3f1 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling pest v2.1.3
     Running `rustc --crate-name pest /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/pest-2.1.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=e1aabf8a6c7bc1ff -C extra-filename=-e1aabf8a6c7bc1ff --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern ucd_trie=/work/oxidecomputer/crucible/target/debug/deps/libucd_trie-a4e8188b5963a3f1.rmeta --cap-lints allow`
       Fresh maplit v1.0.2
       Fresh unicode-xid v0.2.2
   Compiling synstructure v0.12.6
     Running `rustc --crate-name synstructure --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/synstructure-0.12.6/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no --cfg 'feature="default"' --cfg 'feature="proc-macro"' -C metadata=48a8bcebf08faced -C extra-filename=-48a8bcebf08faced --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rmeta --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rmeta --extern syn=/work/oxidecomputer/crucible/target/debug/deps/libsyn-837f9a049f77ca38.rmeta --extern unicode_xid=/work/oxidecomputer/crucible/target/debug/deps/libunicode_xid-911d92403f8fb404.rmeta --cap-lints allow`
   Compiling regex v1.8.3
     Running `rustc --crate-name regex --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/regex-1.8.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="aho-corasick"' --cfg 'feature="default"' --cfg 'feature="memchr"' --cfg 'feature="perf"' --cfg 'feature="perf-cache"' --cfg 'feature="perf-dfa"' --cfg 'feature="perf-inline"' --cfg 'feature="perf-literal"' --cfg 'feature="std"' --cfg 'feature="unicode"' --cfg 'feature="unicode-age"' --cfg 'feature="unicode-bool"' --cfg 'feature="unicode-case"' --cfg 'feature="unicode-gencat"' --cfg 'feature="unicode-perl"' --cfg 'feature="unicode-script"' --cfg 'feature="unicode-segment"' -C metadata=f9e3a4eb3da387ce -C extra-filename=-f9e3a4eb3da387ce --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern aho_corasick=/work/oxidecomputer/crucible/target/debug/deps/libaho_corasick-afc99e972f7e39a1.rmeta --extern memchr=/work/oxidecomputer/crucible/target/debug/deps/libmemchr-9611c546f9b73cca.rmeta --extern regex_syntax=/work/oxidecomputer/crucible/target/debug/deps/libregex_syntax-c9e6cb4ff3b1ed69.rmeta --cap-lints allow`
   Compiling pest_meta v2.1.3
     Running `rustc --crate-name pest_meta /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/pest_meta-2.1.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C metadata=74e8955a7d734149 -C extra-filename=-74e8955a7d734149 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern maplit=/work/oxidecomputer/crucible/target/debug/deps/libmaplit-695e3e42e8316fa9.rmeta --extern pest=/work/oxidecomputer/crucible/target/debug/deps/libpest-e1aabf8a6c7bc1ff.rmeta --cap-lints allow`
   Compiling time-core v0.1.1
     Running `rustc --crate-name time_core --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/time-core-0.1.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=fd216cb66fe61cb9 -C extra-filename=-fd216cb66fe61cb9 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling hashbrown v0.14.0
     Running `rustc --crate-name hashbrown --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/hashbrown-0.14.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="raw"' -C metadata=615db2aaa1e4d335 -C extra-filename=-615db2aaa1e4d335 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling subtle v2.4.1
     Running `rustc --crate-name subtle /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/subtle-2.4.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=ee0c8d46ce57336b -C extra-filename=-ee0c8d46ce57336b --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling equivalent v1.0.0
     Running `rustc --crate-name equivalent /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/equivalent-1.0.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=7a40718821784752 -C extra-filename=-7a40718821784752 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling time-macros v0.2.10
     Running `rustc --crate-name time_macros --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/time-macros-0.2.10/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type proc-macro --emit=dep-info,link -C prefer-dynamic -C embed-bitcode=no --cfg 'feature="formatting"' --cfg 'feature="parsing"' -C metadata=e07155b8a4270ad7 -C extra-filename=-e07155b8a4270ad7 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern time_core=/work/oxidecomputer/crucible/target/debug/deps/libtime_core-fd216cb66fe61cb9.rlib --extern proc_macro --cap-lints allow`
   Compiling pest_generator v2.1.3
     Running `rustc --crate-name pest_generator /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/pest_generator-2.1.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C metadata=01755500ddb0705d -C extra-filename=-01755500ddb0705d --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern pest=/work/oxidecomputer/crucible/target/debug/deps/libpest-e1aabf8a6c7bc1ff.rmeta --extern pest_meta=/work/oxidecomputer/crucible/target/debug/deps/libpest_meta-74e8955a7d734149.rmeta --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rmeta --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rmeta --extern syn=/work/oxidecomputer/crucible/target/debug/deps/libsyn-837f9a049f77ca38.rmeta --cap-lints allow`
   Compiling indexmap v2.0.0
     Running `rustc --crate-name indexmap --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/indexmap-2.0.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=32a62b7926f710bd -C extra-filename=-32a62b7926f710bd --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern equivalent=/work/oxidecomputer/crucible/target/debug/deps/libequivalent-7a40718821784752.rmeta --extern hashbrown=/work/oxidecomputer/crucible/target/debug/deps/libhashbrown-615db2aaa1e4d335.rmeta --cap-lints allow`
   Compiling serde_derive v1.0.167
     Running `rustc --crate-name serde_derive /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/serde_derive-1.0.167/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type proc-macro --emit=dep-info,link -C prefer-dynamic -C embed-bitcode=no --cfg 'feature="default"' -C metadata=3441a1f9756a6d5b -C extra-filename=-3441a1f9756a6d5b --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rlib --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rlib --extern syn=/work/oxidecomputer/crucible/target/debug/deps/libsyn-baedf68a9175a6da.rlib --extern proc_macro --cap-lints allow`
   Compiling thiserror-impl v1.0.40
     Running `rustc --crate-name thiserror_impl --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/thiserror-impl-1.0.40/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type proc-macro --emit=dep-info,link -C prefer-dynamic -C embed-bitcode=no -C metadata=bf767c319ff2d238 -C extra-filename=-bf767c319ff2d238 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rlib --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rlib --extern syn=/work/oxidecomputer/crucible/target/debug/deps/libsyn-baedf68a9175a6da.rlib --extern proc_macro --cap-lints allow`
   Compiling futures-macro v0.3.28
     Running `rustc --crate-name futures_macro --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-macro-0.3.28/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type proc-macro --emit=dep-info,link -C prefer-dynamic -C embed-bitcode=no -C metadata=81cbf12cf17c7f91 -C extra-filename=-81cbf12cf17c7f91 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rlib --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rlib --extern syn=/work/oxidecomputer/crucible/target/debug/deps/libsyn-baedf68a9175a6da.rlib --extern proc_macro --cap-lints allow`
   Compiling tokio-macros v2.1.0
     Running `rustc --crate-name tokio_macros --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-macros-2.1.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type proc-macro --emit=dep-info,link -C prefer-dynamic -C embed-bitcode=no -C metadata=129d27199cb07668 -C extra-filename=-129d27199cb07668 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rlib --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rlib --extern syn=/work/oxidecomputer/crucible/target/debug/deps/libsyn-baedf68a9175a6da.rlib --extern proc_macro --cap-lints allow`
   Compiling schemars_derive v0.8.12
     Running `rustc --crate-name schemars_derive --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/schemars_derive-0.8.12/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type proc-macro --emit=dep-info,link -C prefer-dynamic -C embed-bitcode=no -C metadata=ca659e99c5534de4 -C extra-filename=-ca659e99c5534de4 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rlib --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rlib --extern serde_derive_internals=/work/oxidecomputer/crucible/target/debug/deps/libserde_derive_internals-6aea929a774cf30f.rlib --extern syn=/work/oxidecomputer/crucible/target/debug/deps/libsyn-837f9a049f77ca38.rlib --extern proc_macro --cap-lints allow`
   Compiling tracing-attributes v0.1.23
     Running `rustc --crate-name tracing_attributes --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tracing-attributes-0.1.23/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type proc-macro --emit=dep-info,link -C prefer-dynamic -C embed-bitcode=no -C metadata=af60b1f7cb0d953c -C extra-filename=-af60b1f7cb0d953c --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rlib --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rlib --extern syn=/work/oxidecomputer/crucible/target/debug/deps/libsyn-837f9a049f77ca38.rlib --extern proc_macro --cap-lints allow`
   Compiling openssl-macros v0.1.0
     Running `rustc --crate-name openssl_macros --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/openssl-macros-0.1.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type proc-macro --emit=dep-info,link -C prefer-dynamic -C embed-bitcode=no -C metadata=57d0261583d18db5 -C extra-filename=-57d0261583d18db5 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rlib --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rlib --extern syn=/work/oxidecomputer/crucible/target/debug/deps/libsyn-837f9a049f77ca38.rlib --extern proc_macro --cap-lints allow`
   Compiling zerocopy-derive v0.2.0
     Running `rustc --crate-name zerocopy_derive --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/zerocopy-derive-0.2.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type proc-macro --emit=dep-info,link -C prefer-dynamic -C embed-bitcode=no -C metadata=f3f98c5abdba899d -C extra-filename=-f3f98c5abdba899d --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rlib --extern syn=/work/oxidecomputer/crucible/target/debug/deps/libsyn-837f9a049f77ca38.rlib --extern synstructure=/work/oxidecomputer/crucible/target/debug/deps/libsynstructure-48a8bcebf08faced.rlib --extern proc_macro --cap-lints allow`
     Running `rustc --crate-name openssl --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/openssl-0.10.55/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' -C metadata=4a7553c915defdd5 -C extra-filename=-4a7553c915defdd5 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern bitflags=/work/oxidecomputer/crucible/target/debug/deps/libbitflags-b5bc15fb96af61fc.rmeta --extern cfg_if=/work/oxidecomputer/crucible/target/debug/deps/libcfg_if-a93276574f0edf39.rmeta --extern foreign_types=/work/oxidecomputer/crucible/target/debug/deps/libforeign_types-07949a1c938aca9c.rmeta --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --extern once_cell=/work/oxidecomputer/crucible/target/debug/deps/libonce_cell-fdb5f9e769d1e589.rmeta --extern openssl_macros=/work/oxidecomputer/crucible/target/debug/deps/libopenssl_macros-57d0261583d18db5.so --extern ffi=/work/oxidecomputer/crucible/target/debug/deps/libopenssl_sys-fe2ced2b5ac7bf5a.rmeta --cap-lints allow -L native=/usr/ssl-3/lib/amd64 --cfg 'osslconf="OPENSSL_NO_SSL3_METHOD"' --cfg ossl101 --cfg ossl102 --cfg ossl110 --cfg ossl110g --cfg ossl110h --cfg ossl111 --cfg ossl300`
   Compiling futures-util v0.3.28
     Running `rustc --crate-name futures_util --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.28/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="alloc"' --cfg 'feature="async-await"' --cfg 'feature="async-await-macro"' --cfg 'feature="channel"' --cfg 'feature="default"' --cfg 'feature="futures-channel"' --cfg 'feature="futures-io"' --cfg 'feature="futures-macro"' --cfg 'feature="futures-sink"' --cfg 'feature="io"' --cfg 'feature="memchr"' --cfg 'feature="sink"' --cfg 'feature="slab"' --cfg 'feature="std"' -C metadata=b4da5d5433271d56 -C extra-filename=-b4da5d5433271d56 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern futures_channel=/work/oxidecomputer/crucible/target/debug/deps/libfutures_channel-34a7a018f58dc5a7.rmeta --extern futures_core=/work/oxidecomputer/crucible/target/debug/deps/libfutures_core-46c6e3a1b3966417.rmeta --extern futures_io=/work/oxidecomputer/crucible/target/debug/deps/libfutures_io-bcbbef0c8c581d67.rmeta --extern futures_macro=/work/oxidecomputer/crucible/target/debug/deps/libfutures_macro-81cbf12cf17c7f91.so --extern futures_sink=/work/oxidecomputer/crucible/target/debug/deps/libfutures_sink-a6d6ed8a846c5f8a.rmeta --extern futures_task=/work/oxidecomputer/crucible/target/debug/deps/libfutures_task-12b58f257ddc96a4.rmeta --extern memchr=/work/oxidecomputer/crucible/target/debug/deps/libmemchr-9611c546f9b73cca.rmeta --extern pin_project_lite=/work/oxidecomputer/crucible/target/debug/deps/libpin_project_lite-c035e320730303c2.rmeta --extern pin_utils=/work/oxidecomputer/crucible/target/debug/deps/libpin_utils-bcfb754cd1ab8c67.rmeta --extern slab=/work/oxidecomputer/crucible/target/debug/deps/libslab-5b7c79e345d6363e.rmeta --cap-lints allow`
   Compiling thiserror v1.0.40
     Running `rustc --crate-name thiserror --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/thiserror-1.0.40/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=07eca56f531a0e5d -C extra-filename=-07eca56f531a0e5d --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern thiserror_impl=/work/oxidecomputer/crucible/target/debug/deps/libthiserror_impl-bf767c319ff2d238.so --cap-lints allow`
   Compiling num_threads v0.1.5
     Running `rustc --crate-name num_threads /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/num_threads-0.1.5/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=4e84d104f1db9110 -C extra-filename=-4e84d104f1db9110 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling winnow v0.4.6
     Running `rustc --crate-name winnow --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/winnow-0.4.6/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="alloc"' --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=e34c187c773d92ef -C extra-filename=-e34c187c773d92ef --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling time v0.3.23
     Running `rustc --crate-name time --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/time-0.3.23/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="alloc"' --cfg 'feature="default"' --cfg 'feature="formatting"' --cfg 'feature="local-offset"' --cfg 'feature="macros"' --cfg 'feature="parsing"' --cfg 'feature="std"' -C metadata=9b604407a0d52f86 -C extra-filename=-9b604407a0d52f86 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern itoa=/work/oxidecomputer/crucible/target/debug/deps/libitoa-f0781104e344570e.rmeta --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --extern num_threads=/work/oxidecomputer/crucible/target/debug/deps/libnum_threads-4e84d104f1db9110.rmeta --extern time_core=/work/oxidecomputer/crucible/target/debug/deps/libtime_core-fd216cb66fe61cb9.rmeta --extern time_macros=/work/oxidecomputer/crucible/target/debug/deps/libtime_macros-e07155b8a4270ad7.so --cap-lints allow`
   Compiling pest_derive v2.1.0
     Running `rustc --crate-name pest_derive /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/pest_derive-2.1.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type proc-macro --emit=dep-info,link -C prefer-dynamic -C embed-bitcode=no -C metadata=8b18b291ae7a4e87 -C extra-filename=-8b18b291ae7a4e87 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern pest=/work/oxidecomputer/crucible/target/debug/deps/libpest-e1aabf8a6c7bc1ff.rlib --extern pest_generator=/work/oxidecomputer/crucible/target/debug/deps/libpest_generator-01755500ddb0705d.rlib --extern proc_macro --cap-lints allow`
   Compiling zerocopy v0.3.0
     Running `rustc --crate-name zerocopy --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/zerocopy-0.3.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=0b893e01c09e6e03 -C extra-filename=-0b893e01c09e6e03 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern byteorder=/work/oxidecomputer/crucible/target/debug/deps/libbyteorder-1306999913c8e1b3.rmeta --extern zerocopy_derive=/work/oxidecomputer/crucible/target/debug/deps/libzerocopy_derive-f3f98c5abdba899d.so --cap-lints allow`
   Compiling tracing v0.1.37
     Running `rustc --crate-name tracing --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tracing-0.1.37/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="attributes"' --cfg 'feature="default"' --cfg 'feature="std"' --cfg 'feature="tracing-attributes"' -C metadata=1de351a7f2b0ab55 -C extra-filename=-1de351a7f2b0ab55 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern cfg_if=/work/oxidecomputer/crucible/target/debug/deps/libcfg_if-a93276574f0edf39.rmeta --extern pin_project_lite=/work/oxidecomputer/crucible/target/debug/deps/libpin_project_lite-c035e320730303c2.rmeta --extern tracing_attributes=/work/oxidecomputer/crucible/target/debug/deps/libtracing_attributes-af60b1f7cb0d953c.so --extern tracing_core=/work/oxidecomputer/crucible/target/debug/deps/libtracing_core-adac59f754126e83.rmeta --cap-lints allow`
   Compiling slog v2.7.0
     Running `rustc --crate-name slog /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/slog-2.7.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="dynamic-keys"' --cfg 'feature="max_level_trace"' --cfg 'feature="release_max_level_debug"' --cfg 'feature="std"' -C metadata=84fd25666c3c26ee -C extra-filename=-84fd25666c3c26ee --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow --cfg integer128`
       Fresh rustversion v1.0.14
   Compiling errno v0.3.1
     Running `rustc --crate-name errno --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/errno-0.3.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=e243723ea5172a32 -C extra-filename=-e243723ea5172a32 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --cap-lints allow`
   Compiling dtrace-parser v0.1.14
     Running `rustc --crate-name dtrace_parser --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dtrace-parser-0.1.14/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=647a421ab06e4ff3 -C extra-filename=-647a421ab06e4ff3 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern pest=/work/oxidecomputer/crucible/target/debug/deps/libpest-e1aabf8a6c7bc1ff.rmeta --extern pest_derive=/work/oxidecomputer/crucible/target/debug/deps/libpest_derive-8b18b291ae7a4e87.so --extern thiserror=/work/oxidecomputer/crucible/target/debug/deps/libthiserror-07eca56f531a0e5d.rmeta --cap-lints allow`
   Compiling rustix v0.37.7
     Running `rustc --crate-name rustix --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/rustix-0.37.7/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="fs"' --cfg 'feature="io-lifetimes"' --cfg 'feature="libc"' --cfg 'feature="std"' --cfg 'feature="use-libc-auxv"' -C metadata=4f0213bb214bbfd6 -C extra-filename=-4f0213bb214bbfd6 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern bitflags=/work/oxidecomputer/crucible/target/debug/deps/libbitflags-b5bc15fb96af61fc.rmeta --extern libc_errno=/work/oxidecomputer/crucible/target/debug/deps/liberrno-e243723ea5172a32.rmeta --extern io_lifetimes=/work/oxidecomputer/crucible/target/debug/deps/libio_lifetimes-df7ee936a2a5cbac.rmeta --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --cap-lints allow --cfg libc --cfg solarish`
   Compiling dof v0.1.5
     Running `rustc --crate-name dof --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dof-0.1.5/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=48f06b38719b0295 -C extra-filename=-48f06b38719b0295 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern thiserror=/work/oxidecomputer/crucible/target/debug/deps/libthiserror-07eca56f531a0e5d.rmeta --extern zerocopy=/work/oxidecomputer/crucible/target/debug/deps/libzerocopy-0b893e01c09e6e03.rmeta --cap-lints allow`
   Compiling thread-id v4.0.0
     Running `rustc --crate-name thread_id /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/thread-id-4.0.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=eaa0d9ff93152533 -C extra-filename=-eaa0d9ff93152533 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --cap-lints allow`
   Compiling fastrand v1.7.0
     Running `rustc --crate-name fastrand --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/fastrand-1.7.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=d1e60a5c45d284ad -C extra-filename=-d1e60a5c45d284ad --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling async-trait v0.1.73
     Running `rustc --crate-name async_trait --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/async-trait-0.1.73/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type proc-macro --emit=dep-info,link -C prefer-dynamic -C embed-bitcode=no -C metadata=a300e84178ee0ad1 -C extra-filename=-a300e84178ee0ad1 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rlib --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rlib --extern syn=/work/oxidecomputer/crucible/target/debug/deps/libsyn-baedf68a9175a6da.rlib --extern proc_macro --cap-lints allow`
   Compiling block-buffer v0.10.2
     Running `rustc --crate-name block_buffer --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/block-buffer-0.10.2/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=0de17b0eee43f62e -C extra-filename=-0de17b0eee43f62e --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern generic_array=/work/oxidecomputer/crucible/target/debug/deps/libgeneric_array-bc31b01978a602e7.rmeta --cap-lints allow`
   Compiling thread_local v1.1.4
     Running `rustc --crate-name thread_local --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/thread_local-1.1.4/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=771d112d9219dc9f -C extra-filename=-771d112d9219dc9f --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern once_cell=/work/oxidecomputer/crucible/target/debug/deps/libonce_cell-fdb5f9e769d1e589.rmeta --cap-lints allow`
   Compiling serde v1.0.167
     Running `rustc --crate-name serde /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/serde-1.0.167/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="alloc"' --cfg 'feature="default"' --cfg 'feature="derive"' --cfg 'feature="rc"' --cfg 'feature="serde_derive"' --cfg 'feature="std"' -C metadata=2779165e31567af2 -C extra-filename=-2779165e31567af2 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern serde_derive=/work/oxidecomputer/crucible/target/debug/deps/libserde_derive-3441a1f9756a6d5b.so --cap-lints allow`
   Compiling tempfile v3.5.0
     Running `rustc --crate-name tempfile --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tempfile-3.5.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=05cb6a5f7e86cb66 -C extra-filename=-05cb6a5f7e86cb66 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern cfg_if=/work/oxidecomputer/crucible/target/debug/deps/libcfg_if-a93276574f0edf39.rmeta --extern fastrand=/work/oxidecomputer/crucible/target/debug/deps/libfastrand-d1e60a5c45d284ad.rmeta --extern rustix=/work/oxidecomputer/crucible/target/debug/deps/librustix-4f0213bb214bbfd6.rmeta --cap-lints allow`
     Running `rustc --crate-name native_tls /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/native-tls-0.2.11/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=320c05ab5bbd33c9 -C extra-filename=-320c05ab5bbd33c9 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern log=/work/oxidecomputer/crucible/target/debug/deps/liblog-930e200cffaa7930.rmeta --extern openssl=/work/oxidecomputer/crucible/target/debug/deps/libopenssl-4a7553c915defdd5.rmeta --extern openssl_probe=/work/oxidecomputer/crucible/target/debug/deps/libopenssl_probe-ef939aeef8872804.rmeta --extern openssl_sys=/work/oxidecomputer/crucible/target/debug/deps/libopenssl_sys-fe2ced2b5ac7bf5a.rmeta --cap-lints allow -L native=/usr/ssl-3/lib/amd64 --cfg have_min_max_version`
   Compiling futures-executor v0.3.28
     Running `rustc --crate-name futures_executor --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-executor-0.3.28/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=61f53162107ffb32 -C extra-filename=-61f53162107ffb32 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern futures_core=/work/oxidecomputer/crucible/target/debug/deps/libfutures_core-46c6e3a1b3966417.rmeta --extern futures_task=/work/oxidecomputer/crucible/target/debug/deps/libfutures_task-12b58f257ddc96a4.rmeta --extern futures_util=/work/oxidecomputer/crucible/target/debug/deps/libfutures_util-b4da5d5433271d56.rmeta --cap-lints allow`
   Compiling strsim v0.10.0
     Running `rustc --crate-name strsim /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/strsim-0.10.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=3314ed6444b5bb69 -C extra-filename=-3314ed6444b5bb69 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling digest v0.10.6
     Running `rustc --crate-name digest --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/digest-0.10.6/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="alloc"' --cfg 'feature="block-buffer"' --cfg 'feature="core-api"' --cfg 'feature="default"' --cfg 'feature="mac"' --cfg 'feature="std"' --cfg 'feature="subtle"' -C metadata=d0d34473efe105d1 -C extra-filename=-d0d34473efe105d1 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern block_buffer=/work/oxidecomputer/crucible/target/debug/deps/libblock_buffer-0de17b0eee43f62e.rmeta --extern crypto_common=/work/oxidecomputer/crucible/target/debug/deps/libcrypto_common-0953bfc5dcef84b9.rmeta --extern subtle=/work/oxidecomputer/crucible/target/debug/deps/libsubtle-ee0c8d46ce57336b.rmeta --cap-lints allow`
   Compiling dirs-sys-next v0.1.2
     Running `rustc --crate-name dirs_sys_next --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dirs-sys-next-0.1.2/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=4419c59e785c767e -C extra-filename=-4419c59e785c767e --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --cap-lints allow`
   Compiling libz-sys v1.1.8
     Running `rustc --crate-name build_script_build --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/libz-sys-1.1.8/build.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no --cfg 'feature="libc"' -C metadata=664a34d733156495 -C extra-filename=-664a34d733156495 --out-dir /work/oxidecomputer/crucible/target/debug/build/libz-sys-664a34d733156495 -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern cc=/work/oxidecomputer/crucible/target/debug/deps/libcc-2976d4b8f46fa671.rlib --extern pkg_config=/work/oxidecomputer/crucible/target/debug/deps/libpkg_config-e05c47386f4bdcc0.rlib --cap-lints allow`
   Compiling dirs-next v2.0.0
     Running `rustc --crate-name dirs_next --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dirs-next-2.0.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=369b0b19f2fe8c2c -C extra-filename=-369b0b19f2fe8c2c --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern cfg_if=/work/oxidecomputer/crucible/target/debug/deps/libcfg_if-a93276574f0edf39.rmeta --extern dirs_sys_next=/work/oxidecomputer/crucible/target/debug/deps/libdirs_sys_next-4419c59e785c767e.rmeta --cap-lints allow`
   Compiling match_cfg v0.1.0
     Running `rustc --crate-name match_cfg /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/match_cfg-0.1.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="use_core"' -C metadata=a8964d97fe20de7b -C extra-filename=-a8964d97fe20de7b --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling siphasher v0.3.10
     Running `rustc --crate-name siphasher --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/siphasher-0.3.10/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=2eb27360f66646f3 -C extra-filename=-2eb27360f66646f3 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling fallible-iterator v0.2.0
     Running `rustc --crate-name fallible_iterator --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/fallible-iterator-0.2.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=2f475e2aff163c98 -C extra-filename=-2f475e2aff163c98 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
     Running `/work/oxidecomputer/crucible/target/debug/build/libz-sys-664a34d733156495/build-script-build`
   Compiling hostname v0.3.1
     Running `rustc --crate-name hostname /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/hostname-0.3.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' -C metadata=4a0f8b1a56e5681a -C extra-filename=-4a0f8b1a56e5681a --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --extern match_cfg=/work/oxidecomputer/crucible/target/debug/deps/libmatch_cfg-a8964d97fe20de7b.rmeta --cap-lints allow`
   Compiling term v0.7.0
     Running `rustc --crate-name term --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/term-0.7.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' -C metadata=9ecdb763f4f627bf -C extra-filename=-9ecdb763f4f627bf --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern dirs_next=/work/oxidecomputer/crucible/target/debug/deps/libdirs_next-369b0b19f2fe8c2c.rmeta --cap-lints allow`
   Compiling futures v0.3.28
     Running `rustc --crate-name futures --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-0.3.28/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="alloc"' --cfg 'feature="async-await"' --cfg 'feature="default"' --cfg 'feature="executor"' --cfg 'feature="futures-executor"' --cfg 'feature="std"' -C metadata=df761c89bfa71e54 -C extra-filename=-df761c89bfa71e54 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern futures_channel=/work/oxidecomputer/crucible/target/debug/deps/libfutures_channel-34a7a018f58dc5a7.rmeta --extern futures_core=/work/oxidecomputer/crucible/target/debug/deps/libfutures_core-46c6e3a1b3966417.rmeta --extern futures_executor=/work/oxidecomputer/crucible/target/debug/deps/libfutures_executor-61f53162107ffb32.rmeta --extern futures_io=/work/oxidecomputer/crucible/target/debug/deps/libfutures_io-bcbbef0c8c581d67.rmeta --extern futures_sink=/work/oxidecomputer/crucible/target/debug/deps/libfutures_sink-a6d6ed8a846c5f8a.rmeta --extern futures_task=/work/oxidecomputer/crucible/target/debug/deps/libfutures_task-12b58f257ddc96a4.rmeta --extern futures_util=/work/oxidecomputer/crucible/target/debug/deps/libfutures_util-b4da5d5433271d56.rmeta --cap-lints allow`
   Compiling atty v0.2.14
     Running `rustc --crate-name atty /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/atty-0.2.14/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=bfb6a2cdc762f7c4 -C extra-filename=-bfb6a2cdc762f7c4 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --cap-lints allow`
       Fresh autocfg v0.1.8
   Compiling rand_core v0.4.2
     Running `rustc --crate-name rand_core /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/rand_core-0.4.2/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="alloc"' --cfg 'feature="std"' -C metadata=cf5252fd1c1f594a -C extra-filename=-cf5252fd1c1f594a --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling take_mut v0.2.2
     Running `rustc --crate-name take_mut /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/take_mut-0.2.2/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=0ebd75564ded4875 -C extra-filename=-0ebd75564ded4875 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling cpufeatures v0.2.1
     Running `rustc --crate-name cpufeatures --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/cpufeatures-0.2.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=6e25ef20a8fa218f -C extra-filename=-6e25ef20a8fa218f --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling slog-async v2.8.0
     Running `rustc --crate-name slog_async /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/slog-async-2.8.0/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' -C metadata=03e743f036119074 -C extra-filename=-03e743f036119074 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern crossbeam_channel=/work/oxidecomputer/crucible/target/debug/deps/libcrossbeam_channel-19ff6fa486e51c63.rmeta --extern slog=/work/oxidecomputer/crucible/target/debug/deps/libslog-84fd25666c3c26ee.rmeta --extern take_mut=/work/oxidecomputer/crucible/target/debug/deps/libtake_mut-0ebd75564ded4875.rmeta --extern thread_local=/work/oxidecomputer/crucible/target/debug/deps/libthread_local-771d112d9219dc9f.rmeta --cap-lints allow --cfg integer128`
   Compiling slog-term v2.9.0
     Running `rustc --crate-name slog_term --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/slog-term-2.9.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=31c687431f6dd53c -C extra-filename=-31c687431f6dd53c --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern atty=/work/oxidecomputer/crucible/target/debug/deps/libatty-bfb6a2cdc762f7c4.rmeta --extern slog=/work/oxidecomputer/crucible/target/debug/deps/libslog-84fd25666c3c26ee.rmeta --extern term=/work/oxidecomputer/crucible/target/debug/deps/libterm-9ecdb763f4f627bf.rmeta --extern thread_local=/work/oxidecomputer/crucible/target/debug/deps/libthread_local-771d112d9219dc9f.rmeta --extern time=/work/oxidecomputer/crucible/target/debug/deps/libtime-9b604407a0d52f86.rmeta --cap-lints allow`
   Compiling libgit2-sys v0.15.2+1.6.4
     Running `rustc --crate-name build_script_build --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/libgit2-sys-0.15.2+1.6.4/build.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no -C metadata=69da610a4d2f950f -C extra-filename=-69da610a4d2f950f --out-dir /work/oxidecomputer/crucible/target/debug/build/libgit2-sys-69da610a4d2f950f -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern cc=/work/oxidecomputer/crucible/target/debug/deps/libcc-2976d4b8f46fa671.rlib --extern pkg_config=/work/oxidecomputer/crucible/target/debug/deps/libpkg_config-e05c47386f4bdcc0.rlib --cap-lints allow`
     Running `rustc --crate-name libz_sys --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/libz-sys-1.1.8/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no --cfg 'feature="libc"' -C metadata=a3111f279c2174e3 -C extra-filename=-a3111f279c2174e3 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --cap-lints allow -L native=/usr/lib/amd64 -l z`
   Compiling errno v0.2.8
     Running `rustc --crate-name errno /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/errno-0.2.8/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=af3769d2acbbbf20 -C extra-filename=-af3769d2acbbbf20 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --cap-lints allow`
   Compiling utf8parse v0.2.1
     Running `rustc --crate-name utf8parse --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/utf8parse-0.2.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' -C metadata=7be01c1e08c14495 -C extra-filename=-7be01c1e08c14495 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
       Fresh vcpkg v0.2.15
   Compiling rand_core v0.3.1
     Running `rustc --crate-name rand_core /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/rand_core-0.3.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=588d7d54223a4bae -C extra-filename=-588d7d54223a4bae --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern rand_core=/work/oxidecomputer/crucible/target/debug/deps/librand_core-cf5252fd1c1f594a.rmeta --cap-lints allow`
       Fresh ident_case v1.0.1
   Compiling static_assertions v1.1.0
     Running `rustc --crate-name static_assertions /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/static_assertions-1.1.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=457e7de98121159f -C extra-filename=-457e7de98121159f --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling darling_core v0.20.1
     Running `rustc --crate-name darling_core --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/darling_core-0.20.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no --cfg 'feature="strsim"' --cfg 'feature="suggestions"' -C metadata=9504cdbd254aa6a2 -C extra-filename=-9504cdbd254aa6a2 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern fnv=/work/oxidecomputer/crucible/target/debug/deps/libfnv-5641130f60a8056b.rmeta --extern ident_case=/work/oxidecomputer/crucible/target/debug/deps/libident_case-90a2253e7db4d178.rmeta --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rmeta --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rmeta --extern strsim=/work/oxidecomputer/crucible/target/debug/deps/libstrsim-3314ed6444b5bb69.rmeta --extern syn=/work/oxidecomputer/crucible/target/debug/deps/libsyn-baedf68a9175a6da.rmeta --cap-lints allow`
   Compiling memoffset v0.7.1
     Running `rustc --crate-name memoffset /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/memoffset-0.7.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' -C metadata=7efa3f7abe9851cf -C extra-filename=-7efa3f7abe9851cf --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow --cfg tuple_ty --cfg allow_clippy --cfg maybe_uninit --cfg doctests --cfg raw_ref_macros`
   Compiling num-integer v0.1.44
     Running `rustc --crate-name num_integer /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/num-integer-0.1.44/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="std"' -C metadata=ada2de842cf2fef0 -C extra-filename=-ada2de842cf2fef0 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern num_traits=/work/oxidecomputer/crucible/target/debug/deps/libnum_traits-8e50de91aba3f8f9.rmeta --cap-lints allow --cfg has_i128`
       Fresh semver v0.1.20
   Compiling bitflags v2.3.3
     Running `rustc --crate-name bitflags --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/bitflags-2.3.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=109244799287a8c3 -C extra-filename=-109244799287a8c3 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
     Running `/work/oxidecomputer/crucible/target/debug/build/libgit2-sys-69da610a4d2f950f/build-script-build`
       Fresh rustc_version v0.1.7
   Compiling libsqlite3-sys v0.26.0
     Running `rustc --crate-name libsqlite3_sys --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/libsqlite3-sys-0.26.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="min_sqlite_version_3_14_0"' --cfg 'feature="pkg-config"' --cfg 'feature="vcpkg"' -C metadata=d6c034ac2a660d5f -C extra-filename=-d6c034ac2a660d5f --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow -L native=/usr/lib/amd64 -l sqlite3`
   Compiling nix v0.26.2
     Running `rustc --crate-name nix --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/nix-0.26.2/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="acct"' --cfg 'feature="aio"' --cfg 'feature="default"' --cfg 'feature="dir"' --cfg 'feature="env"' --cfg 'feature="event"' --cfg 'feature="feature"' --cfg 'feature="fs"' --cfg 'feature="hostname"' --cfg 'feature="inotify"' --cfg 'feature="ioctl"' --cfg 'feature="kmod"' --cfg 'feature="memoffset"' --cfg 'feature="mman"' --cfg 'feature="mount"' --cfg 'feature="mqueue"' --cfg 'feature="net"' --cfg 'feature="personality"' --cfg 'feature="pin-utils"' --cfg 'feature="poll"' --cfg 'feature="process"' --cfg 'feature="pthread"' --cfg 'feature="ptrace"' --cfg 'feature="quota"' --cfg 'feature="reboot"' --cfg 'feature="resource"' --cfg 'feature="sched"' --cfg 'feature="signal"' --cfg 'feature="socket"' --cfg 'feature="term"' --cfg 'feature="time"' --cfg 'feature="ucontext"' --cfg 'feature="uio"' --cfg 'feature="user"' --cfg 'feature="zerocopy"' -C metadata=3635aff9412bf811 -C extra-filename=-3635aff9412bf811 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern bitflags=/work/oxidecomputer/crucible/target/debug/deps/libbitflags-b5bc15fb96af61fc.rmeta --extern cfg_if=/work/oxidecomputer/crucible/target/debug/deps/libcfg_if-a93276574f0edf39.rmeta --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --extern memoffset=/work/oxidecomputer/crucible/target/debug/deps/libmemoffset-7efa3f7abe9851cf.rmeta --extern pin_utils=/work/oxidecomputer/crucible/target/debug/deps/libpin_utils-bcfb754cd1ab8c67.rmeta --extern static_assertions=/work/oxidecomputer/crucible/target/debug/deps/libstatic_assertions-457e7de98121159f.rmeta --cap-lints allow`
   Compiling rustix v0.36.5
     Running `rustc --crate-name rustix --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/rustix-0.36.5/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="io-lifetimes"' --cfg 'feature="libc"' --cfg 'feature="std"' --cfg 'feature="termios"' --cfg 'feature="use-libc-auxv"' -C metadata=06529533a7816508 -C extra-filename=-06529533a7816508 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern bitflags=/work/oxidecomputer/crucible/target/debug/deps/libbitflags-b5bc15fb96af61fc.rmeta --extern libc_errno=/work/oxidecomputer/crucible/target/debug/deps/liberrno-af3769d2acbbbf20.rmeta --extern io_lifetimes=/work/oxidecomputer/crucible/target/debug/deps/libio_lifetimes-df7ee936a2a5cbac.rmeta --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --cap-lints allow --cfg libc`
   Compiling anstyle-parse v0.2.0
     Running `rustc --crate-name anstyle_parse --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/anstyle-parse-0.2.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="utf8"' -C metadata=1cddf38a13db6213 -C extra-filename=-1cddf38a13db6213 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern utf8parse=/work/oxidecomputer/crucible/target/debug/deps/libutf8parse-7be01c1e08c14495.rmeta --cap-lints allow`
   Compiling async-stream-impl v0.3.5
     Running `rustc --crate-name async_stream_impl --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/async-stream-impl-0.3.5/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type proc-macro --emit=dep-info,link -C prefer-dynamic -C embed-bitcode=no -C metadata=499322f076b313bc -C extra-filename=-499322f076b313bc --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rlib --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rlib --extern syn=/work/oxidecomputer/crucible/target/debug/deps/libsyn-baedf68a9175a6da.rlib --extern proc_macro --cap-lints allow`
   Compiling hashlink v0.8.0
     Running `rustc --crate-name hashlink --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/hashlink-0.8.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=20d837d226474f7f -C extra-filename=-20d837d226474f7f --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern hashbrown=/work/oxidecomputer/crucible/target/debug/deps/libhashbrown-3ad6614047c487f9.rmeta --cap-lints allow`
   Compiling fallible-streaming-iterator v0.1.9
     Running `rustc --crate-name fallible_streaming_iterator /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/fallible-streaming-iterator-0.1.9/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=95608b7ccd5e88f6 -C extra-filename=-95608b7ccd5e88f6 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling anstyle-query v1.0.0
     Running `rustc --crate-name anstyle_query --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/anstyle-query-1.0.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=f6b83c0a54455ea0 -C extra-filename=-f6b83c0a54455ea0 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling anstyle v1.0.0
     Running `rustc --crate-name anstyle --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/anstyle-1.0.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=b22d2b8ea1e30552 -C extra-filename=-b22d2b8ea1e30552 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling colorchoice v1.0.0
     Running `rustc --crate-name colorchoice --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/colorchoice-1.0.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=88cb686aa85d1c3b -C extra-filename=-88cb686aa85d1c3b --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling rusqlite v0.29.0
     Running `rustc --crate-name rusqlite --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/rusqlite-0.29.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=e4d2316a88b06837 -C extra-filename=-e4d2316a88b06837 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern bitflags=/work/oxidecomputer/crucible/target/debug/deps/libbitflags-109244799287a8c3.rmeta --extern fallible_iterator=/work/oxidecomputer/crucible/target/debug/deps/libfallible_iterator-2f475e2aff163c98.rmeta --extern fallible_streaming_iterator=/work/oxidecomputer/crucible/target/debug/deps/libfallible_streaming_iterator-95608b7ccd5e88f6.rmeta --extern hashlink=/work/oxidecomputer/crucible/target/debug/deps/libhashlink-20d837d226474f7f.rmeta --extern libsqlite3_sys=/work/oxidecomputer/crucible/target/debug/deps/liblibsqlite3_sys-d6c034ac2a660d5f.rmeta --extern smallvec=/work/oxidecomputer/crucible/target/debug/deps/libsmallvec-397f26bd8c84e528.rmeta --cap-lints allow -L native=/usr/lib/amd64`
   Compiling terminal_size v0.2.5
     Running `rustc --crate-name terminal_size --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/terminal_size-0.2.5/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=0b293d2f7e4e73b7 -C extra-filename=-0b293d2f7e4e73b7 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern rustix=/work/oxidecomputer/crucible/target/debug/deps/librustix-06529533a7816508.rmeta --cap-lints allow`
   Compiling anstream v0.5.0
     Running `rustc --crate-name anstream --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/anstream-0.5.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="auto"' --cfg 'feature="default"' --cfg 'feature="wincon"' -C metadata=ca70a88d33d17944 -C extra-filename=-ca70a88d33d17944 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern anstyle=/work/oxidecomputer/crucible/target/debug/deps/libanstyle-b22d2b8ea1e30552.rmeta --extern anstyle_parse=/work/oxidecomputer/crucible/target/debug/deps/libanstyle_parse-1cddf38a13db6213.rmeta --extern anstyle_query=/work/oxidecomputer/crucible/target/debug/deps/libanstyle_query-f6b83c0a54455ea0.rmeta --extern colorchoice=/work/oxidecomputer/crucible/target/debug/deps/libcolorchoice-88cb686aa85d1c3b.rmeta --extern utf8parse=/work/oxidecomputer/crucible/target/debug/deps/libutf8parse-7be01c1e08c14495.rmeta --cap-lints allow`
   Compiling num-bigint v0.2.6
     Running `rustc --crate-name num_bigint /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/num-bigint-0.2.6/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="std"' -C metadata=fb487451ba2d3918 -C extra-filename=-fb487451ba2d3918 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern num_integer=/work/oxidecomputer/crucible/target/debug/deps/libnum_integer-ada2de842cf2fef0.rmeta --extern num_traits=/work/oxidecomputer/crucible/target/debug/deps/libnum_traits-8e50de91aba3f8f9.rmeta --cap-lints allow --cfg has_i128`
   Compiling twox-hash v1.6.3
     Running `rustc --crate-name twox_hash --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/twox-hash-1.6.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="rand"' --cfg 'feature="std"' -C metadata=9f5dd4f7319ca539 -C extra-filename=-9f5dd4f7319ca539 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern cfg_if=/work/oxidecomputer/crucible/target/debug/deps/libcfg_if-a93276574f0edf39.rmeta --extern rand=/work/oxidecomputer/crucible/target/debug/deps/librand-1f91a9ea4aed49ee.rmeta --extern static_assertions=/work/oxidecomputer/crucible/target/debug/deps/libstatic_assertions-457e7de98121159f.rmeta --cap-lints allow`
   Compiling async-stream v0.3.5
     Running `rustc --crate-name async_stream --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/async-stream-0.3.5/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=0486f21173e73f9c -C extra-filename=-0486f21173e73f9c --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern async_stream_impl=/work/oxidecomputer/crucible/target/debug/deps/libasync_stream_impl-499322f076b313bc.so --extern futures_core=/work/oxidecomputer/crucible/target/debug/deps/libfutures_core-46c6e3a1b3966417.rmeta --extern pin_project_lite=/work/oxidecomputer/crucible/target/debug/deps/libpin_project_lite-c035e320730303c2.rmeta --cap-lints allow`
   Compiling bytes v1.4.0
     Running `rustc --crate-name bytes --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/bytes-1.4.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="serde"' --cfg 'feature="std"' -C metadata=64a8a55ef81e55dd -C extra-filename=-64a8a55ef81e55dd --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --cap-lints allow`
   Compiling serde_json v1.0.105
   Compiling indexmap v1.9.3
     Running `rustc --crate-name serde_json --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/serde_json-1.0.105/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=96027e7dd982d07a -C extra-filename=-96027e7dd982d07a --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern itoa=/work/oxidecomputer/crucible/target/debug/deps/libitoa-f0781104e344570e.rmeta --extern ryu=/work/oxidecomputer/crucible/target/debug/deps/libryu-062e5ac4087417b3.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --cap-lints allow --cfg limb_width_64`
     Running `rustc --crate-name indexmap --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/indexmap-1.9.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="serde"' --cfg 'feature="serde-1"' --cfg 'feature="std"' -C metadata=c2c8f74266eebb64 -C extra-filename=-c2c8f74266eebb64 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern hashbrown=/work/oxidecomputer/crucible/target/debug/deps/libhashbrown-3ad6614047c487f9.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --cap-lints allow --cfg has_std`
   Compiling chrono v0.4.26
     Running `rustc --crate-name chrono --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/chrono-0.4.26/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="clock"' --cfg 'feature="default"' --cfg 'feature="iana-time-zone"' --cfg 'feature="js-sys"' --cfg 'feature="oldtime"' --cfg 'feature="serde"' --cfg 'feature="std"' --cfg 'feature="time"' --cfg 'feature="wasm-bindgen"' --cfg 'feature="wasmbind"' --cfg 'feature="winapi"' -C metadata=0402a5e17dd99cdf -C extra-filename=-0402a5e17dd99cdf --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern iana_time_zone=/work/oxidecomputer/crucible/target/debug/deps/libiana_time_zone-3051bc3a975f54c9.rmeta --extern num_traits=/work/oxidecomputer/crucible/target/debug/deps/libnum_traits-8e50de91aba3f8f9.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern time=/work/oxidecomputer/crucible/target/debug/deps/libtime-fcbf6ea26d482f3a.rmeta --cap-lints allow`
   Compiling tokio v1.32.0
     Running `rustc --crate-name tokio --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.32.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="bytes"' --cfg 'feature="default"' --cfg 'feature="fs"' --cfg 'feature="full"' --cfg 'feature="io-std"' --cfg 'feature="io-util"' --cfg 'feature="libc"' --cfg 'feature="macros"' --cfg 'feature="mio"' --cfg 'feature="net"' --cfg 'feature="num_cpus"' --cfg 'feature="parking_lot"' --cfg 'feature="process"' --cfg 'feature="rt"' --cfg 'feature="rt-multi-thread"' --cfg 'feature="signal"' --cfg 'feature="signal-hook-registry"' --cfg 'feature="socket2"' --cfg 'feature="sync"' --cfg 'feature="test-util"' --cfg 'feature="time"' --cfg 'feature="tokio-macros"' --cfg 'feature="windows-sys"' -C metadata=cf190744403b2ee1 -C extra-filename=-cf190744403b2ee1 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern bytes=/work/oxidecomputer/crucible/target/debug/deps/libbytes-64a8a55ef81e55dd.rmeta --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --extern mio=/work/oxidecomputer/crucible/target/debug/deps/libmio-27a8136cf12de2bb.rmeta --extern num_cpus=/work/oxidecomputer/crucible/target/debug/deps/libnum_cpus-67a451bebfcc5086.rmeta --extern parking_lot=/work/oxidecomputer/crucible/target/debug/deps/libparking_lot-970d5c0acece447c.rmeta --extern pin_project_lite=/work/oxidecomputer/crucible/target/debug/deps/libpin_project_lite-c035e320730303c2.rmeta --extern signal_hook_registry=/work/oxidecomputer/crucible/target/debug/deps/libsignal_hook_registry-4d955479f235827e.rmeta --extern socket2=/work/oxidecomputer/crucible/target/debug/deps/libsocket2-b464b617227db85e.rmeta --extern tokio_macros=/work/oxidecomputer/crucible/target/debug/deps/libtokio_macros-129d27199cb07668.so --cap-lints allow`
   Compiling uuid v1.4.1
     Running `rustc --crate-name uuid --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/uuid-1.4.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="getrandom"' --cfg 'feature="rng"' --cfg 'feature="serde"' --cfg 'feature="std"' --cfg 'feature="v4"' -C metadata=7cc8e87b4149b49e -C extra-filename=-7cc8e87b4149b49e --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern getrandom=/work/oxidecomputer/crucible/target/debug/deps/libgetrandom-567199de146d617e.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --cap-lints allow`
   Compiling http v0.2.9
     Running `rustc --crate-name http --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/http-0.2.9/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=4526ab951518c907 -C extra-filename=-4526ab951518c907 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern bytes=/work/oxidecomputer/crucible/target/debug/deps/libbytes-64a8a55ef81e55dd.rmeta --extern fnv=/work/oxidecomputer/crucible/target/debug/deps/libfnv-5641130f60a8056b.rmeta --extern itoa=/work/oxidecomputer/crucible/target/debug/deps/libitoa-f0781104e344570e.rmeta --cap-lints allow`
   Compiling serde_urlencoded v0.7.1
     Running `rustc --crate-name serde_urlencoded --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/serde_urlencoded-0.7.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=779d3ac41d95557a -C extra-filename=-779d3ac41d95557a --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern form_urlencoded=/work/oxidecomputer/crucible/target/debug/deps/libform_urlencoded-ef731295a29c9150.rmeta --extern itoa=/work/oxidecomputer/crucible/target/debug/deps/libitoa-f0781104e344570e.rmeta --extern ryu=/work/oxidecomputer/crucible/target/debug/deps/libryu-062e5ac4087417b3.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --cap-lints allow`
   Compiling serde_tokenstream v0.2.0
     Running `rustc --crate-name serde_tokenstream --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/serde_tokenstream-0.2.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C metadata=2a83027836bfd0fb -C extra-filename=-2a83027836bfd0fb --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rmeta --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern syn=/work/oxidecomputer/crucible/target/debug/deps/libsyn-baedf68a9175a6da.rmeta --cap-lints allow`
   Compiling schemars v0.8.12
     Running `rustc --crate-name schemars --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/schemars-0.8.12/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="bytes"' --cfg 'feature="chrono"' --cfg 'feature="default"' --cfg 'feature="derive"' --cfg 'feature="schemars_derive"' --cfg 'feature="uuid1"' -C metadata=83d20014cee5d9b5 -C extra-filename=-83d20014cee5d9b5 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern bytes=/work/oxidecomputer/crucible/target/debug/deps/libbytes-64a8a55ef81e55dd.rmeta --extern chrono=/work/oxidecomputer/crucible/target/debug/deps/libchrono-0402a5e17dd99cdf.rmeta --extern dyn_clone=/work/oxidecomputer/crucible/target/debug/deps/libdyn_clone-6ca330b50294836a.rmeta --extern schemars_derive=/work/oxidecomputer/crucible/target/debug/deps/libschemars_derive-ca659e99c5534de4.so --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rmeta --extern uuid1=/work/oxidecomputer/crucible/target/debug/deps/libuuid-7cc8e87b4149b49e.rmeta --cap-lints allow --cfg std_atomic64 --cfg std_atomic`
   Compiling openapiv3 v1.0.2
   Compiling serde_yaml v0.9.17
     Running `rustc --crate-name openapiv3 --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/openapiv3-1.0.2/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="skip_serializing_defaults"' -C metadata=f84072cef6d0d68c -C extra-filename=-f84072cef6d0d68c --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern indexmap=/work/oxidecomputer/crucible/target/debug/deps/libindexmap-c2c8f74266eebb64.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rmeta --cap-lints allow`
     Running `rustc --crate-name serde_yaml --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/serde_yaml-0.9.17/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C metadata=c7db3f53bbf8134b -C extra-filename=-c7db3f53bbf8134b --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern indexmap=/work/oxidecomputer/crucible/target/debug/deps/libindexmap-c2c8f74266eebb64.rmeta --extern itoa=/work/oxidecomputer/crucible/target/debug/deps/libitoa-f0781104e344570e.rmeta --extern ryu=/work/oxidecomputer/crucible/target/debug/deps/libryu-062e5ac4087417b3.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern unsafe_libyaml=/work/oxidecomputer/crucible/target/debug/deps/libunsafe_libyaml-18c99c193ab0b0f5.rmeta --cap-lints allow`
   Compiling serde_spanned v0.6.3
     Running `rustc --crate-name serde_spanned --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/serde_spanned-0.6.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="serde"' -C metadata=92d13640a4d5d836 -C extra-filename=-92d13640a4d5d836 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --cap-lints allow`
   Compiling toml_datetime v0.6.3
     Running `rustc --crate-name toml_datetime --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/toml_datetime-0.6.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="serde"' -C metadata=c003080b79dfe49d -C extra-filename=-c003080b79dfe49d --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --cap-lints allow`
   Compiling http-body v0.4.4
     Running `rustc --crate-name http_body --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/http-body-0.4.4/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=bb1d69dd918c127f -C extra-filename=-bb1d69dd918c127f --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern bytes=/work/oxidecomputer/crucible/target/debug/deps/libbytes-64a8a55ef81e55dd.rmeta --extern http=/work/oxidecomputer/crucible/target/debug/deps/libhttp-4526ab951518c907.rmeta --extern pin_project_lite=/work/oxidecomputer/crucible/target/debug/deps/libpin_project_lite-c035e320730303c2.rmeta --cap-lints allow`
   Compiling toml_edit v0.19.12
     Running `rustc --crate-name toml_edit --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/toml_edit-0.19.12/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="serde"' -C metadata=29ed73c573391add -C extra-filename=-29ed73c573391add --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern indexmap=/work/oxidecomputer/crucible/target/debug/deps/libindexmap-32a62b7926f710bd.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern serde_spanned=/work/oxidecomputer/crucible/target/debug/deps/libserde_spanned-92d13640a4d5d836.rmeta --extern toml_datetime=/work/oxidecomputer/crucible/target/debug/deps/libtoml_datetime-c003080b79dfe49d.rmeta --extern winnow=/work/oxidecomputer/crucible/target/debug/deps/libwinnow-e34c187c773d92ef.rmeta --cap-lints allow`
   Compiling usdt-impl v0.3.5
     Running `rustc --crate-name usdt_impl --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/usdt-impl-0.3.5/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="asm"' -C metadata=aa84cb563c29877d -C extra-filename=-aa84cb563c29877d --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern byteorder=/work/oxidecomputer/crucible/target/debug/deps/libbyteorder-1306999913c8e1b3.rmeta --extern dof=/work/oxidecomputer/crucible/target/debug/deps/libdof-48f06b38719b0295.rmeta --extern dtrace_parser=/work/oxidecomputer/crucible/target/debug/deps/libdtrace_parser-647a421ab06e4ff3.rmeta --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rmeta --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rmeta --extern syn=/work/oxidecomputer/crucible/target/debug/deps/libsyn-837f9a049f77ca38.rmeta --extern thiserror=/work/oxidecomputer/crucible/target/debug/deps/libthiserror-07eca56f531a0e5d.rmeta --extern thread_id=/work/oxidecomputer/crucible/target/debug/deps/libthread_id-eaa0d9ff93152533.rmeta --cap-lints allow --cfg usdt_stable_asm --cfg usdt_backend_standard`
   Compiling typify-impl v0.0.13 (https://github.com/oxidecomputer/typify#92bfed8b)
     Running `rustc --crate-name typify_impl --edition=2021 /home/build/.cargo/git/checkouts/typify-288d5a84bbbe6a46/92bfed8/typify-impl/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=5385296f2ea50467 -C extra-filename=-5385296f2ea50467 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern heck=/work/oxidecomputer/crucible/target/debug/deps/libheck-29af75c938b110f7.rmeta --extern log=/work/oxidecomputer/crucible/target/debug/deps/liblog-930e200cffaa7930.rmeta --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rmeta --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rmeta --extern regress=/work/oxidecomputer/crucible/target/debug/deps/libregress-10da65958da1c830.rmeta --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rmeta --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rmeta --extern syn=/work/oxidecomputer/crucible/target/debug/deps/libsyn-baedf68a9175a6da.rmeta --extern thiserror=/work/oxidecomputer/crucible/target/debug/deps/libthiserror-07eca56f531a0e5d.rmeta --extern unicode_ident=/work/oxidecomputer/crucible/target/debug/deps/libunicode_ident-81e7752fff89e70f.rmeta --cap-lints allow`
   Compiling semver v1.0.18
     Running `rustc --crate-name semver --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/semver-1.0.18/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="serde"' --cfg 'feature="std"' -C metadata=8c1c5827befd93e7 -C extra-filename=-8c1c5827befd93e7 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --cap-lints allow`
   Compiling serde_tokenstream v0.1.6
     Running `rustc --crate-name serde_tokenstream --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/serde_tokenstream-0.1.6/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C metadata=a3f756c2f035b223 -C extra-filename=-a3f756c2f035b223 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern syn=/work/oxidecomputer/crucible/target/debug/deps/libsyn-837f9a049f77ca38.rmeta --cap-lints allow`
   Compiling slog-json v2.6.1
     Running `rustc --crate-name slog_json --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/slog-json-2.6.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' -C metadata=d8408f8f3a6dd5b7 -C extra-filename=-d8408f8f3a6dd5b7 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rmeta --extern slog=/work/oxidecomputer/crucible/target/debug/deps/libslog-84fd25666c3c26ee.rmeta --extern time=/work/oxidecomputer/crucible/target/debug/deps/libtime-9b604407a0d52f86.rmeta --cap-lints allow`
   Compiling usdt-macro v0.3.5
     Running `rustc --crate-name usdt_macro --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/usdt-macro-0.3.5/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type proc-macro --emit=dep-info,link -C prefer-dynamic -C embed-bitcode=no --cfg 'feature="asm"' -C metadata=0ea5e72aa4bb61c5 -C extra-filename=-0ea5e72aa4bb61c5 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern dtrace_parser=/work/oxidecomputer/crucible/target/debug/deps/libdtrace_parser-647a421ab06e4ff3.rlib --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rlib --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rlib --extern serde_tokenstream=/work/oxidecomputer/crucible/target/debug/deps/libserde_tokenstream-a3f756c2f035b223.rlib --extern syn=/work/oxidecomputer/crucible/target/debug/deps/libsyn-837f9a049f77ca38.rlib --extern usdt_impl=/work/oxidecomputer/crucible/target/debug/deps/libusdt_impl-aa84cb563c29877d.rlib --extern proc_macro --cap-lints allow`
   Compiling usdt-attr-macro v0.3.5
     Running `rustc --crate-name usdt_attr_macro --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/usdt-attr-macro-0.3.5/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type proc-macro --emit=dep-info,link -C prefer-dynamic -C embed-bitcode=no --cfg 'feature="asm"' -C metadata=1e49091236cd57b8 -C extra-filename=-1e49091236cd57b8 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern dtrace_parser=/work/oxidecomputer/crucible/target/debug/deps/libdtrace_parser-647a421ab06e4ff3.rlib --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rlib --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rlib --extern serde_tokenstream=/work/oxidecomputer/crucible/target/debug/deps/libserde_tokenstream-a3f756c2f035b223.rlib --extern syn=/work/oxidecomputer/crucible/target/debug/deps/libsyn-837f9a049f77ca38.rlib --extern usdt_impl=/work/oxidecomputer/crucible/target/debug/deps/libusdt_impl-aa84cb563c29877d.rlib --extern proc_macro --cap-lints allow`
   Compiling slog-bunyan v2.4.0
     Running `rustc --crate-name slog_bunyan --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/slog-bunyan-2.4.0/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=dce051a6775f1d99 -C extra-filename=-dce051a6775f1d99 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern hostname=/work/oxidecomputer/crucible/target/debug/deps/libhostname-4a0f8b1a56e5681a.rmeta --extern slog=/work/oxidecomputer/crucible/target/debug/deps/libslog-84fd25666c3c26ee.rmeta --extern slog_json=/work/oxidecomputer/crucible/target/debug/deps/libslog_json-d8408f8f3a6dd5b7.rmeta --extern time=/work/oxidecomputer/crucible/target/debug/deps/libtime-9b604407a0d52f86.rmeta --cap-lints allow`
   Compiling rustc_version v0.4.0
     Running `rustc --crate-name rustc_version --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/rustc_version-0.4.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C metadata=201ef6100eba532b -C extra-filename=-201ef6100eba532b --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern semver=/work/oxidecomputer/crucible/target/debug/deps/libsemver-8c1c5827befd93e7.rmeta --cap-lints allow`
   Compiling toml v0.7.6
     Running `rustc --crate-name toml --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/toml-0.7.6/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="display"' --cfg 'feature="parse"' -C metadata=de0eb3fcc3b95b5c -C extra-filename=-de0eb3fcc3b95b5c --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern serde_spanned=/work/oxidecomputer/crucible/target/debug/deps/libserde_spanned-92d13640a4d5d836.rmeta --extern toml_datetime=/work/oxidecomputer/crucible/target/debug/deps/libtoml_datetime-c003080b79dfe49d.rmeta --extern toml_edit=/work/oxidecomputer/crucible/target/debug/deps/libtoml_edit-29ed73c573391add.rmeta --cap-lints allow`
   Compiling darling_macro v0.20.1
     Running `rustc --crate-name darling_macro --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/darling_macro-0.20.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type proc-macro --emit=dep-info,link -C prefer-dynamic -C embed-bitcode=no -C metadata=9054a39daee42f78 -C extra-filename=-9054a39daee42f78 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern darling_core=/work/oxidecomputer/crucible/target/debug/deps/libdarling_core-9504cdbd254aa6a2.rlib --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rlib --extern syn=/work/oxidecomputer/crucible/target/debug/deps/libsyn-baedf68a9175a6da.rlib --extern proc_macro --cap-lints allow`
   Compiling camino v1.1.4
     Running `rustc --crate-name camino --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/camino-1.1.4/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="serde"' --cfg 'feature="serde1"' -C metadata=45f0f4a2c258f934 -C extra-filename=-45f0f4a2c258f934 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --cap-lints allow --cfg path_buf_capacity --cfg shrink_to --cfg try_reserve_2 --cfg path_buf_deref_mut`
   Compiling hex v0.4.3
     Running `rustc --crate-name hex --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/hex-0.4.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="alloc"' --cfg 'feature="default"' --cfg 'feature="serde"' --cfg 'feature="std"' -C metadata=6531b11cb72de3e5 -C extra-filename=-6531b11cb72de3e5 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --cap-lints allow`
   Compiling tokio-util v0.7.3
     Running `rustc --crate-name tokio_util --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-util-0.7.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="codec"' --cfg 'feature="default"' --cfg 'feature="io"' --cfg 'feature="tracing"' -C metadata=279b3765a2b5aad1 -C extra-filename=-279b3765a2b5aad1 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern bytes=/work/oxidecomputer/crucible/target/debug/deps/libbytes-64a8a55ef81e55dd.rmeta --extern futures_core=/work/oxidecomputer/crucible/target/debug/deps/libfutures_core-46c6e3a1b3966417.rmeta --extern futures_sink=/work/oxidecomputer/crucible/target/debug/deps/libfutures_sink-a6d6ed8a846c5f8a.rmeta --extern pin_project_lite=/work/oxidecomputer/crucible/target/debug/deps/libpin_project_lite-c035e320730303c2.rmeta --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rmeta --extern tracing=/work/oxidecomputer/crucible/target/debug/deps/libtracing-1de351a7f2b0ab55.rmeta --cap-lints allow`
   Compiling tokio-native-tls v0.3.0
     Running `rustc --crate-name tokio_native_tls --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-native-tls-0.3.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=f56aba82a642e205 -C extra-filename=-f56aba82a642e205 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern native_tls=/work/oxidecomputer/crucible/target/debug/deps/libnative_tls-320c05ab5bbd33c9.rmeta --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rmeta --cap-lints allow -L native=/usr/ssl-3/lib/amd64`
   Compiling usdt v0.3.5
     Running `rustc --crate-name usdt --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/usdt-0.3.5/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="asm"' --cfg 'feature="default"' --cfg 'feature="dtrace-parser"' -C metadata=86bb76e3b8fcea87 -C extra-filename=-86bb76e3b8fcea87 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern dtrace_parser=/work/oxidecomputer/crucible/target/debug/deps/libdtrace_parser-647a421ab06e4ff3.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern usdt_attr_macro=/work/oxidecomputer/crucible/target/debug/deps/libusdt_attr_macro-1e49091236cd57b8.so --extern usdt_impl=/work/oxidecomputer/crucible/target/debug/deps/libusdt_impl-aa84cb563c29877d.rmeta --extern usdt_macro=/work/oxidecomputer/crucible/target/debug/deps/libusdt_macro-0ea5e72aa4bb61c5.so --cap-lints allow`
   Compiling darling v0.20.1
     Running `rustc --crate-name darling --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/darling-0.20.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no --cfg 'feature="default"' --cfg 'feature="suggestions"' -C metadata=00ecdd47ee30dd62 -C extra-filename=-00ecdd47ee30dd62 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern darling_core=/work/oxidecomputer/crucible/target/debug/deps/libdarling_core-9504cdbd254aa6a2.rmeta --extern darling_macro=/work/oxidecomputer/crucible/target/debug/deps/libdarling_macro-9054a39daee42f78.so --cap-lints allow`
   Compiling sha2 v0.10.2
     Running `rustc --crate-name sha2 --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/sha2-0.10.2/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=9b09b4b286e2cb62 -C extra-filename=-9b09b4b286e2cb62 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern cfg_if=/work/oxidecomputer/crucible/target/debug/deps/libcfg_if-a93276574f0edf39.rmeta --extern cpufeatures=/work/oxidecomputer/crucible/target/debug/deps/libcpufeatures-6e25ef20a8fa218f.rmeta --extern digest=/work/oxidecomputer/crucible/target/debug/deps/libdigest-d0d34473efe105d1.rmeta --cap-lints allow`
   Compiling hmac v0.12.1
     Running `rustc --crate-name hmac --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/hmac-0.12.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=dbc6bea790b908e1 -C extra-filename=-dbc6bea790b908e1 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern digest=/work/oxidecomputer/crucible/target/debug/deps/libdigest-d0d34473efe105d1.rmeta --cap-lints allow`
   Compiling slog-dtrace v0.2.3
     Running `rustc --crate-name slog_dtrace --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/slog-dtrace-0.2.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=8024beacfb95325b -C extra-filename=-8024beacfb95325b --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern chrono=/work/oxidecomputer/crucible/target/debug/deps/libchrono-0402a5e17dd99cdf.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rmeta --extern slog=/work/oxidecomputer/crucible/target/debug/deps/libslog-84fd25666c3c26ee.rmeta --extern usdt=/work/oxidecomputer/crucible/target/debug/deps/libusdt-86bb76e3b8fcea87.rmeta --cap-lints allow`
   Compiling md-5 v0.10.1
     Running `rustc --crate-name md5 --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/md-5-0.10.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=161e8167d58df2d6 -C extra-filename=-161e8167d58df2d6 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern digest=/work/oxidecomputer/crucible/target/debug/deps/libdigest-d0d34473efe105d1.rmeta --cap-lints allow`
   Compiling h2 v0.3.20
     Running `rustc --crate-name h2 --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/h2-0.3.20/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=3e2d8390f23dd48a -C extra-filename=-3e2d8390f23dd48a --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern bytes=/work/oxidecomputer/crucible/target/debug/deps/libbytes-64a8a55ef81e55dd.rmeta --extern fnv=/work/oxidecomputer/crucible/target/debug/deps/libfnv-5641130f60a8056b.rmeta --extern futures_core=/work/oxidecomputer/crucible/target/debug/deps/libfutures_core-46c6e3a1b3966417.rmeta --extern futures_sink=/work/oxidecomputer/crucible/target/debug/deps/libfutures_sink-a6d6ed8a846c5f8a.rmeta --extern futures_util=/work/oxidecomputer/crucible/target/debug/deps/libfutures_util-b4da5d5433271d56.rmeta --extern http=/work/oxidecomputer/crucible/target/debug/deps/libhttp-4526ab951518c907.rmeta --extern indexmap=/work/oxidecomputer/crucible/target/debug/deps/libindexmap-c2c8f74266eebb64.rmeta --extern slab=/work/oxidecomputer/crucible/target/debug/deps/libslab-5b7c79e345d6363e.rmeta --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rmeta --extern tokio_util=/work/oxidecomputer/crucible/target/debug/deps/libtokio_util-279b3765a2b5aad1.rmeta --extern tracing=/work/oxidecomputer/crucible/target/debug/deps/libtracing-1de351a7f2b0ab55.rmeta --cap-lints allow`
   Compiling structmeta-derive v0.1.5
     Running `rustc --crate-name structmeta_derive --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/structmeta-derive-0.1.5/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type proc-macro --emit=dep-info,link -C prefer-dynamic -C embed-bitcode=no -C metadata=fab85507a465b1dd -C extra-filename=-fab85507a465b1dd --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rlib --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rlib --extern syn=/work/oxidecomputer/crucible/target/debug/deps/libsyn-837f9a049f77ca38.rlib --extern proc_macro --cap-lints allow`
   Compiling stringprep v0.1.2
     Running `rustc --crate-name stringprep /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/stringprep-0.1.2/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=ca9dbc365b4e987e -C extra-filename=-ca9dbc365b4e987e --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern unicode_bidi=/work/oxidecomputer/crucible/target/debug/deps/libunicode_bidi-2c825f5beb05a037.rmeta --extern unicode_normalization=/work/oxidecomputer/crucible/target/debug/deps/libunicode_normalization-3df8261a03d4248e.rmeta --cap-lints allow`
   Compiling base64 v0.13.1
     Running `rustc --crate-name base64 --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/base64-0.13.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=480956047b7063a4 -C extra-filename=-480956047b7063a4 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling atomic-waker v1.1.1
     Running `rustc --crate-name atomic_waker --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/atomic-waker-1.1.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=2b745687caafccb6 -C extra-filename=-2b745687caafccb6 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling clap_lex v0.5.0
     Running `rustc --crate-name clap_lex --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/clap_lex-0.5.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=cd6fc5d5239978f4 -C extra-filename=-cd6fc5d5239978f4 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling regex-syntax v0.6.28
     Running `rustc --crate-name regex_syntax --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/regex-syntax-0.6.28/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="unicode"' --cfg 'feature="unicode-age"' --cfg 'feature="unicode-bool"' --cfg 'feature="unicode-case"' --cfg 'feature="unicode-gencat"' --cfg 'feature="unicode-perl"' --cfg 'feature="unicode-script"' --cfg 'feature="unicode-segment"' -C metadata=b9593ef3338880de -C extra-filename=-b9593ef3338880de --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling clap_builder v4.4.0
     Running `rustc --crate-name clap_builder --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/clap_builder-4.4.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="color"' --cfg 'feature="env"' --cfg 'feature="error-context"' --cfg 'feature="help"' --cfg 'feature="std"' --cfg 'feature="suggestions"' --cfg 'feature="usage"' --cfg 'feature="wrap_help"' -C metadata=f6e4b2a8b78f1399 -C extra-filename=-f6e4b2a8b78f1399 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern anstream=/work/oxidecomputer/crucible/target/debug/deps/libanstream-ca70a88d33d17944.rmeta --extern anstyle=/work/oxidecomputer/crucible/target/debug/deps/libanstyle-b22d2b8ea1e30552.rmeta --extern clap_lex=/work/oxidecomputer/crucible/target/debug/deps/libclap_lex-cd6fc5d5239978f4.rmeta --extern strsim=/work/oxidecomputer/crucible/target/debug/deps/libstrsim-3314ed6444b5bb69.rmeta --extern terminal_size=/work/oxidecomputer/crucible/target/debug/deps/libterminal_size-0b293d2f7e4e73b7.rmeta --cap-lints allow`
   Compiling postgres-protocol v0.6.4
     Running `rustc --crate-name postgres_protocol --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/postgres-protocol-0.6.4/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=2544e50ad0920d44 -C extra-filename=-2544e50ad0920d44 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern base64=/work/oxidecomputer/crucible/target/debug/deps/libbase64-480956047b7063a4.rmeta --extern byteorder=/work/oxidecomputer/crucible/target/debug/deps/libbyteorder-1306999913c8e1b3.rmeta --extern bytes=/work/oxidecomputer/crucible/target/debug/deps/libbytes-64a8a55ef81e55dd.rmeta --extern fallible_iterator=/work/oxidecomputer/crucible/target/debug/deps/libfallible_iterator-2f475e2aff163c98.rmeta --extern hmac=/work/oxidecomputer/crucible/target/debug/deps/libhmac-dbc6bea790b908e1.rmeta --extern md5=/work/oxidecomputer/crucible/target/debug/deps/libmd5-161e8167d58df2d6.rmeta --extern memchr=/work/oxidecomputer/crucible/target/debug/deps/libmemchr-9611c546f9b73cca.rmeta --extern rand=/work/oxidecomputer/crucible/target/debug/deps/librand-1f91a9ea4aed49ee.rmeta --extern sha2=/work/oxidecomputer/crucible/target/debug/deps/libsha2-9b09b4b286e2cb62.rmeta --extern stringprep=/work/oxidecomputer/crucible/target/debug/deps/libstringprep-ca9dbc365b4e987e.rmeta --cap-lints allow`
   Compiling waitgroup v0.1.2
     Running `rustc --crate-name waitgroup --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/waitgroup-0.1.2/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=db859ead02bd709e -C extra-filename=-db859ead02bd709e --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern atomic_waker=/work/oxidecomputer/crucible/target/debug/deps/libatomic_waker-2b745687caafccb6.rmeta --cap-lints allow`
   Compiling structmeta v0.1.5
     Running `rustc --crate-name structmeta --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/structmeta-0.1.5/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C metadata=478a2e76026e2d21 -C extra-filename=-478a2e76026e2d21 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rmeta --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rmeta --extern structmeta_derive=/work/oxidecomputer/crucible/target/debug/deps/libstructmeta_derive-fab85507a465b1dd.so --extern syn=/work/oxidecomputer/crucible/target/debug/deps/libsyn-837f9a049f77ca38.rmeta --cap-lints allow`
   Compiling serde_with_macros v2.3.3
     Running `rustc --crate-name serde_with_macros --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/serde_with_macros-2.3.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type proc-macro --emit=dep-info,link -C prefer-dynamic -C embed-bitcode=no -C metadata=0356ef5a55ed4c76 -C extra-filename=-0356ef5a55ed4c76 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern darling=/work/oxidecomputer/crucible/target/debug/deps/libdarling-00ecdd47ee30dd62.rlib --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rlib --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rlib --extern syn=/work/oxidecomputer/crucible/target/debug/deps/libsyn-baedf68a9175a6da.rlib --extern proc_macro --cap-lints allow`
   Compiling dropshot_endpoint v0.9.1-dev (https://github.com/oxidecomputer/dropshot?branch=main#aca6de3c)
     Running `rustc --crate-name dropshot_endpoint --edition=2018 /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/aca6de3/dropshot_endpoint/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type proc-macro --emit=dep-info,link -C prefer-dynamic -C embed-bitcode=no -C metadata=1ff3a3dd0352c250 -C extra-filename=-1ff3a3dd0352c250 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rlib --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rlib --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rlib --extern serde_tokenstream=/work/oxidecomputer/crucible/target/debug/deps/libserde_tokenstream-2a83027836bfd0fb.rlib --extern syn=/work/oxidecomputer/crucible/target/debug/deps/libsyn-baedf68a9175a6da.rlib --extern proc_macro --cap-lints allow`
   Compiling num-rational v0.2.4
     Running `rustc --crate-name num_rational /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/num-rational-0.2.4/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="bigint"' --cfg 'feature="num-bigint"' --cfg 'feature="std"' -C metadata=ff1997054aaa4c62 -C extra-filename=-ff1997054aaa4c62 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern num_bigint=/work/oxidecomputer/crucible/target/debug/deps/libnum_bigint-fb487451ba2d3918.rmeta --extern num_integer=/work/oxidecomputer/crucible/target/debug/deps/libnum_integer-ada2de842cf2fef0.rmeta --extern num_traits=/work/oxidecomputer/crucible/target/debug/deps/libnum_traits-8e50de91aba3f8f9.rmeta --cap-lints allow --cfg has_i128 --cfg has_const_fn`
   Compiling hyper v0.14.27
     Running `rustc --crate-name hyper --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/hyper-0.14.27/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="client"' --cfg 'feature="default"' --cfg 'feature="full"' --cfg 'feature="h2"' --cfg 'feature="http1"' --cfg 'feature="http2"' --cfg 'feature="runtime"' --cfg 'feature="server"' --cfg 'feature="socket2"' --cfg 'feature="stream"' --cfg 'feature="tcp"' -C metadata=dad943d3b7cc33e9 -C extra-filename=-dad943d3b7cc33e9 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern bytes=/work/oxidecomputer/crucible/target/debug/deps/libbytes-64a8a55ef81e55dd.rmeta --extern futures_channel=/work/oxidecomputer/crucible/target/debug/deps/libfutures_channel-34a7a018f58dc5a7.rmeta --extern futures_core=/work/oxidecomputer/crucible/target/debug/deps/libfutures_core-46c6e3a1b3966417.rmeta --extern futures_util=/work/oxidecomputer/crucible/target/debug/deps/libfutures_util-b4da5d5433271d56.rmeta --extern h2=/work/oxidecomputer/crucible/target/debug/deps/libh2-3e2d8390f23dd48a.rmeta --extern http=/work/oxidecomputer/crucible/target/debug/deps/libhttp-4526ab951518c907.rmeta --extern http_body=/work/oxidecomputer/crucible/target/debug/deps/libhttp_body-bb1d69dd918c127f.rmeta --extern httparse=/work/oxidecomputer/crucible/target/debug/deps/libhttparse-59406412a39ce707.rmeta --extern httpdate=/work/oxidecomputer/crucible/target/debug/deps/libhttpdate-e3ef82a990113a54.rmeta --extern itoa=/work/oxidecomputer/crucible/target/debug/deps/libitoa-f0781104e344570e.rmeta --extern pin_project_lite=/work/oxidecomputer/crucible/target/debug/deps/libpin_project_lite-c035e320730303c2.rmeta --extern socket2=/work/oxidecomputer/crucible/target/debug/deps/libsocket2-3c3e3607c1c6d64e.rmeta --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rmeta --extern tower_service=/work/oxidecomputer/crucible/target/debug/deps/libtower_service-51da71f2ad5117ee.rmeta --extern tracing=/work/oxidecomputer/crucible/target/debug/deps/libtracing-1de351a7f2b0ab55.rmeta --extern want=/work/oxidecomputer/crucible/target/debug/deps/libwant-5737a0d118420ef7.rmeta --cap-lints allow`
   Compiling serde_path_to_error v0.1.14
     Running `rustc --crate-name serde_path_to_error --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/serde_path_to_error-0.1.14/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=f8fffad8b554a310 -C extra-filename=-f8fffad8b554a310 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern itoa=/work/oxidecomputer/crucible/target/debug/deps/libitoa-f0781104e344570e.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --cap-lints allow`
   Compiling rand_chacha v0.1.1
     Running `rustc --crate-name rand_chacha /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/rand_chacha-0.1.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=60a86e6d4f82a03e -C extra-filename=-60a86e6d4f82a03e --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern rand_core=/work/oxidecomputer/crucible/target/debug/deps/librand_core-588d7d54223a4bae.rmeta --cap-lints allow --cfg rustc_1_26`
   Compiling rand_pcg v0.1.2
     Running `rustc --crate-name rand_pcg /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/rand_pcg-0.1.2/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=64095680c19f1d75 -C extra-filename=-64095680c19f1d75 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern rand_core=/work/oxidecomputer/crucible/target/debug/deps/librand_core-cf5252fd1c1f594a.rmeta --cap-lints allow --cfg rustc_1_26`
   Compiling num-complex v0.2.4
     Running `rustc --crate-name num_complex /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/num-complex-0.2.4/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="std"' -C metadata=b0ad3f1350de6722 -C extra-filename=-b0ad3f1350de6722 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern num_traits=/work/oxidecomputer/crucible/target/debug/deps/libnum_traits-8e50de91aba3f8f9.rmeta --cap-lints allow --cfg has_i128 --cfg has_const_fn`
   Compiling num-iter v0.1.42
     Running `rustc --crate-name num_iter /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/num-iter-0.1.42/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="std"' -C metadata=ea5f21eb64e222e3 -C extra-filename=-ea5f21eb64e222e3 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern num_integer=/work/oxidecomputer/crucible/target/debug/deps/libnum_integer-ada2de842cf2fef0.rmeta --extern num_traits=/work/oxidecomputer/crucible/target/debug/deps/libnum_traits-8e50de91aba3f8f9.rmeta --cap-lints allow --cfg has_i128`
       Fresh paste v1.0.14
   Compiling rand_xorshift v0.1.1
     Running `rustc --crate-name rand_xorshift /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/rand_xorshift-0.1.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=516db51379fddd21 -C extra-filename=-516db51379fddd21 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern rand_core=/work/oxidecomputer/crucible/target/debug/deps/librand_core-588d7d54223a4bae.rmeta --cap-lints allow`
   Compiling rand_hc v0.1.0
     Running `rustc --crate-name rand_hc /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/rand_hc-0.1.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=e624ce70fff78cce -C extra-filename=-e624ce70fff78cce --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern rand_core=/work/oxidecomputer/crucible/target/debug/deps/librand_core-588d7d54223a4bae.rmeta --cap-lints allow`
   Compiling rand_isaac v0.1.1
     Running `rustc --crate-name rand_isaac /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/rand_isaac-0.1.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=2a69a705c11dd4e8 -C extra-filename=-2a69a705c11dd4e8 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern rand_core=/work/oxidecomputer/crucible/target/debug/deps/librand_core-588d7d54223a4bae.rmeta --cap-lints allow`
   Compiling rand_jitter v0.1.4
     Running `rustc --crate-name rand_jitter /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/rand_jitter-0.1.4/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="std"' -C metadata=4375ed928e3b252c -C extra-filename=-4375ed928e3b252c --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern rand_core=/work/oxidecomputer/crucible/target/debug/deps/librand_core-cf5252fd1c1f594a.rmeta --cap-lints allow`
   Compiling rand_os v0.1.3
     Running `rustc --crate-name rand_os /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/rand_os-0.1.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=5662c3e15991676f -C extra-filename=-5662c3e15991676f --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --extern rand_core=/work/oxidecomputer/crucible/target/debug/deps/librand_core-cf5252fd1c1f594a.rmeta --cap-lints allow`
   Compiling sha1 v0.10.5
     Running `rustc --crate-name sha1 --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/sha1-0.10.5/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=8fdbd7a715f3bef1 -C extra-filename=-8fdbd7a715f3bef1 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern cfg_if=/work/oxidecomputer/crucible/target/debug/deps/libcfg_if-a93276574f0edf39.rmeta --extern cpufeatures=/work/oxidecomputer/crucible/target/debug/deps/libcpufeatures-6e25ef20a8fa218f.rmeta --extern digest=/work/oxidecomputer/crucible/target/debug/deps/libdigest-d0d34473efe105d1.rmeta --cap-lints allow`
   Compiling phf_shared v0.10.0
     Running `rustc --crate-name phf_shared --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/phf_shared-0.10.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="std"' -C metadata=ded46b90978b786e -C extra-filename=-ded46b90978b786e --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern siphasher=/work/oxidecomputer/crucible/target/debug/deps/libsiphasher-2eb27360f66646f3.rmeta --cap-lints allow`
   Compiling strum_macros v0.25.2
     Running `rustc --crate-name strum_macros --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/strum_macros-0.25.2/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type proc-macro --emit=dep-info,link -C prefer-dynamic -C embed-bitcode=no -C metadata=bc907f623478289d -C extra-filename=-bc907f623478289d --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern heck=/work/oxidecomputer/crucible/target/debug/deps/libheck-29af75c938b110f7.rlib --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rlib --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rlib --extern rustversion=/work/oxidecomputer/crucible/target/debug/deps/librustversion-ae2ca065d7ce77a2.so --extern syn=/work/oxidecomputer/crucible/target/debug/deps/libsyn-baedf68a9175a6da.rlib --extern proc_macro --cap-lints allow`
   Compiling clap_derive v4.4.0
     Running `rustc --crate-name clap_derive --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/clap_derive-4.4.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type proc-macro --emit=dep-info,link -C prefer-dynamic -C embed-bitcode=no --cfg 'feature="default"' -C metadata=09787059b4668753 -C extra-filename=-09787059b4668753 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern heck=/work/oxidecomputer/crucible/target/debug/deps/libheck-29af75c938b110f7.rlib --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rlib --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rlib --extern syn=/work/oxidecomputer/crucible/target/debug/deps/libsyn-baedf68a9175a6da.rlib --extern proc_macro --cap-lints allow`
   Compiling debug-ignore v1.0.5
     Running `rustc --crate-name debug_ignore /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/debug-ignore-1.0.5/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=2303f500fcbc7093 -C extra-filename=-2303f500fcbc7093 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling fixedbitset v0.4.1
     Running `rustc --crate-name fixedbitset /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/fixedbitset-0.4.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=c5fa142e7e51f766 -C extra-filename=-c5fa142e7e51f766 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling either v1.6.1
     Running `rustc --crate-name either /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/either-1.6.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="use_std"' -C metadata=e075e5427abad613 -C extra-filename=-e075e5427abad613 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling phf v0.10.1
     Running `rustc --crate-name phf --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/phf-0.10.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=692e1d997f5b6ce1 -C extra-filename=-692e1d997f5b6ce1 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern phf_shared=/work/oxidecomputer/crucible/target/debug/deps/libphf_shared-ded46b90978b786e.rmeta --cap-lints allow`
   Compiling petgraph v0.6.3
     Running `rustc --crate-name petgraph --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/petgraph-0.6.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="graphmap"' --cfg 'feature="matrix_graph"' --cfg 'feature="serde"' --cfg 'feature="serde-1"' --cfg 'feature="serde_derive"' --cfg 'feature="stable_graph"' -C metadata=8e9556a1882bb1d5 -C extra-filename=-8e9556a1882bb1d5 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern fixedbitset=/work/oxidecomputer/crucible/target/debug/deps/libfixedbitset-c5fa142e7e51f766.rmeta --extern indexmap=/work/oxidecomputer/crucible/target/debug/deps/libindexmap-c2c8f74266eebb64.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern serde_derive=/work/oxidecomputer/crucible/target/debug/deps/libserde_derive-3441a1f9756a6d5b.so --cap-lints allow`
   Compiling rand v0.6.5
     Running `rustc --crate-name rand /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/rand-0.6.5/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="alloc"' --cfg 'feature="default"' --cfg 'feature="rand_os"' --cfg 'feature="std"' -C metadata=d54af7a747f291ab -C extra-filename=-d54af7a747f291ab --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --extern rand_chacha=/work/oxidecomputer/crucible/target/debug/deps/librand_chacha-60a86e6d4f82a03e.rmeta --extern rand_core=/work/oxidecomputer/crucible/target/debug/deps/librand_core-cf5252fd1c1f594a.rmeta --extern rand_hc=/work/oxidecomputer/crucible/target/debug/deps/librand_hc-e624ce70fff78cce.rmeta --extern rand_isaac=/work/oxidecomputer/crucible/target/debug/deps/librand_isaac-2a69a705c11dd4e8.rmeta --extern rand_jitter=/work/oxidecomputer/crucible/target/debug/deps/librand_jitter-4375ed928e3b252c.rmeta --extern rand_os=/work/oxidecomputer/crucible/target/debug/deps/librand_os-5662c3e15991676f.rmeta --extern rand_pcg=/work/oxidecomputer/crucible/target/debug/deps/librand_pcg-64095680c19f1d75.rmeta --extern rand_xorshift=/work/oxidecomputer/crucible/target/debug/deps/librand_xorshift-516db51379fddd21.rmeta --cap-lints allow --cfg rustc_1_25 --cfg rustc_1_26 --cfg rustc_1_27`
   Compiling parse-display-derive v0.7.0
     Running `rustc --crate-name parse_display_derive --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/parse-display-derive-0.7.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type proc-macro --emit=dep-info,link -C prefer-dynamic -C embed-bitcode=no -C metadata=95e2e5d955c96bcf -C extra-filename=-95e2e5d955c96bcf --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern once_cell=/work/oxidecomputer/crucible/target/debug/deps/libonce_cell-fdb5f9e769d1e589.rlib --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rlib --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rlib --extern regex=/work/oxidecomputer/crucible/target/debug/deps/libregex-f9e3a4eb3da387ce.rlib --extern regex_syntax=/work/oxidecomputer/crucible/target/debug/deps/libregex_syntax-b9593ef3338880de.rlib --extern structmeta=/work/oxidecomputer/crucible/target/debug/deps/libstructmeta-478a2e76026e2d21.rlib --extern syn=/work/oxidecomputer/crucible/target/debug/deps/libsyn-837f9a049f77ca38.rlib --extern proc_macro --cap-lints allow`
   Compiling hyper-tls v0.5.0
     Running `rustc --crate-name hyper_tls --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/hyper-tls-0.5.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=2dbf57f91f681e2c -C extra-filename=-2dbf57f91f681e2c --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern bytes=/work/oxidecomputer/crucible/target/debug/deps/libbytes-64a8a55ef81e55dd.rmeta --extern hyper=/work/oxidecomputer/crucible/target/debug/deps/libhyper-dad943d3b7cc33e9.rmeta --extern native_tls=/work/oxidecomputer/crucible/target/debug/deps/libnative_tls-320c05ab5bbd33c9.rmeta --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rmeta --extern tokio_native_tls=/work/oxidecomputer/crucible/target/debug/deps/libtokio_native_tls-f56aba82a642e205.rmeta --cap-lints allow -L native=/usr/ssl-3/lib/amd64`
   Compiling num v0.2.1
     Running `rustc --crate-name num /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/num-0.2.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="num-bigint"' --cfg 'feature="std"' -C metadata=1148753de5cf68fd -C extra-filename=-1148753de5cf68fd --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern num_bigint=/work/oxidecomputer/crucible/target/debug/deps/libnum_bigint-fb487451ba2d3918.rmeta --extern num_complex=/work/oxidecomputer/crucible/target/debug/deps/libnum_complex-b0ad3f1350de6722.rmeta --extern num_integer=/work/oxidecomputer/crucible/target/debug/deps/libnum_integer-ada2de842cf2fef0.rmeta --extern num_iter=/work/oxidecomputer/crucible/target/debug/deps/libnum_iter-ea5f21eb64e222e3.rmeta --extern num_rational=/work/oxidecomputer/crucible/target/debug/deps/libnum_rational-ff1997054aaa4c62.rmeta --extern num_traits=/work/oxidecomputer/crucible/target/debug/deps/libnum_traits-8e50de91aba3f8f9.rmeta --cap-lints allow`
   Compiling newtype_derive v0.1.6
     Running `rustc --crate-name newtype_derive /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/newtype_derive-0.1.6/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=fedf432d6b5251a4 -C extra-filename=-fedf432d6b5251a4 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow --cfg op_assign`
   Compiling serde_with v2.3.3
     Running `rustc --crate-name serde_with --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/serde_with-2.3.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="alloc"' --cfg 'feature="default"' --cfg 'feature="macros"' --cfg 'feature="std"' -C metadata=4f9ddd30b380d6cf -C extra-filename=-4f9ddd30b380d6cf --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern serde_with_macros=/work/oxidecomputer/crucible/target/debug/deps/libserde_with_macros-0356ef5a55ed4c76.so --cap-lints allow`
   Compiling strum v0.25.0
     Running `rustc --crate-name strum --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/strum-0.25.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="derive"' --cfg 'feature="std"' --cfg 'feature="strum_macros"' -C metadata=59ea3c6704348e58 -C extra-filename=-59ea3c6704348e58 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern strum_macros=/work/oxidecomputer/crucible/target/debug/deps/libstrum_macros-bc907f623478289d.so --cap-lints allow`
   Compiling postgres-types v0.2.3
     Running `rustc --crate-name postgres_types --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/postgres-types-0.2.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="chrono-04"' --cfg 'feature="uuid-1"' --cfg 'feature="with-chrono-0_4"' --cfg 'feature="with-uuid-1"' -C metadata=3d7a8ed523cd84fa -C extra-filename=-3d7a8ed523cd84fa --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern bytes=/work/oxidecomputer/crucible/target/debug/deps/libbytes-64a8a55ef81e55dd.rmeta --extern chrono_04=/work/oxidecomputer/crucible/target/debug/deps/libchrono-0402a5e17dd99cdf.rmeta --extern fallible_iterator=/work/oxidecomputer/crucible/target/debug/deps/libfallible_iterator-2f475e2aff163c98.rmeta --extern postgres_protocol=/work/oxidecomputer/crucible/target/debug/deps/libpostgres_protocol-2544e50ad0920d44.rmeta --extern uuid_1=/work/oxidecomputer/crucible/target/debug/deps/libuuid-7cc8e87b4149b49e.rmeta --cap-lints allow`
   Compiling proc-macro-crate v1.3.1
     Running `rustc --crate-name proc_macro_crate --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/proc-macro-crate-1.3.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C metadata=3935aefc56866ee2 -C extra-filename=-3935aefc56866ee2 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern once_cell=/work/oxidecomputer/crucible/target/debug/deps/libonce_cell-fdb5f9e769d1e589.rmeta --extern toml_edit=/work/oxidecomputer/crucible/target/debug/deps/libtoml_edit-29ed73c573391add.rmeta --cap-lints allow`
   Compiling instant v0.1.12
     Running `rustc --crate-name instant --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/instant-0.1.12/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=3f52fffb2a7c0a97 -C extra-filename=-3f52fffb2a7c0a97 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern cfg_if=/work/oxidecomputer/crucible/target/debug/deps/libcfg_if-a93276574f0edf39.rmeta --cap-lints allow`
   Compiling backoff v0.4.0
     Running `rustc --crate-name backoff --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/backoff-0.4.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="futures"' --cfg 'feature="futures-core"' --cfg 'feature="pin-project-lite"' --cfg 'feature="tokio"' --cfg 'feature="tokio_1"' -C metadata=2bc4a2fd075cf434 -C extra-filename=-2bc4a2fd075cf434 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern futures_core=/work/oxidecomputer/crucible/target/debug/deps/libfutures_core-46c6e3a1b3966417.rmeta --extern getrandom=/work/oxidecomputer/crucible/target/debug/deps/libgetrandom-567199de146d617e.rmeta --extern instant=/work/oxidecomputer/crucible/target/debug/deps/libinstant-3f52fffb2a7c0a97.rmeta --extern pin_project_lite=/work/oxidecomputer/crucible/target/debug/deps/libpin_project_lite-c035e320730303c2.rmeta --extern rand=/work/oxidecomputer/crucible/target/debug/deps/librand-1f91a9ea4aed49ee.rmeta --extern tokio_1=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rmeta --cap-lints allow`
   Compiling clap v4.4.0
     Running `rustc --crate-name clap --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/clap-4.4.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="color"' --cfg 'feature="default"' --cfg 'feature="derive"' --cfg 'feature="env"' --cfg 'feature="error-context"' --cfg 'feature="help"' --cfg 'feature="std"' --cfg 'feature="suggestions"' --cfg 'feature="usage"' --cfg 'feature="wrap_help"' -C metadata=49bc17aade028e79 -C extra-filename=-49bc17aade028e79 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern clap_builder=/work/oxidecomputer/crucible/target/debug/deps/libclap_builder-f6e4b2a8b78f1399.rmeta --extern clap_derive=/work/oxidecomputer/crucible/target/debug/deps/libclap_derive-09787059b4668753.so --extern once_cell=/work/oxidecomputer/crucible/target/debug/deps/libonce_cell-fdb5f9e769d1e589.rmeta --cap-lints allow`
   Compiling tokio-postgres v0.7.6
     Running `rustc --crate-name tokio_postgres --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-postgres-0.7.6/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="runtime"' --cfg 'feature="with-chrono-0_4"' --cfg 'feature="with-uuid-1"' -C metadata=5628b93feb58339b -C extra-filename=-5628b93feb58339b --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern async_trait=/work/oxidecomputer/crucible/target/debug/deps/libasync_trait-a300e84178ee0ad1.so --extern byteorder=/work/oxidecomputer/crucible/target/debug/deps/libbyteorder-1306999913c8e1b3.rmeta --extern bytes=/work/oxidecomputer/crucible/target/debug/deps/libbytes-64a8a55ef81e55dd.rmeta --extern fallible_iterator=/work/oxidecomputer/crucible/target/debug/deps/libfallible_iterator-2f475e2aff163c98.rmeta --extern futures=/work/oxidecomputer/crucible/target/debug/deps/libfutures-df761c89bfa71e54.rmeta --extern log=/work/oxidecomputer/crucible/target/debug/deps/liblog-930e200cffaa7930.rmeta --extern parking_lot=/work/oxidecomputer/crucible/target/debug/deps/libparking_lot-970d5c0acece447c.rmeta --extern percent_encoding=/work/oxidecomputer/crucible/target/debug/deps/libpercent_encoding-0000aebce3d30803.rmeta --extern phf=/work/oxidecomputer/crucible/target/debug/deps/libphf-692e1d997f5b6ce1.rmeta --extern pin_project_lite=/work/oxidecomputer/crucible/target/debug/deps/libpin_project_lite-c035e320730303c2.rmeta --extern postgres_protocol=/work/oxidecomputer/crucible/target/debug/deps/libpostgres_protocol-2544e50ad0920d44.rmeta --extern postgres_types=/work/oxidecomputer/crucible/target/debug/deps/libpostgres_types-3d7a8ed523cd84fa.rmeta --extern socket2=/work/oxidecomputer/crucible/target/debug/deps/libsocket2-3c3e3607c1c6d64e.rmeta --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rmeta --extern tokio_util=/work/oxidecomputer/crucible/target/debug/deps/libtokio_util-279b3765a2b5aad1.rmeta --cap-lints allow`
   Compiling steno v0.4.0
     Running `rustc --crate-name steno --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/steno-0.4.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=d1d3ce30296926ad -C extra-filename=-d1d3ce30296926ad --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rmeta --extern async_trait=/work/oxidecomputer/crucible/target/debug/deps/libasync_trait-a300e84178ee0ad1.so --extern chrono=/work/oxidecomputer/crucible/target/debug/deps/libchrono-0402a5e17dd99cdf.rmeta --extern futures=/work/oxidecomputer/crucible/target/debug/deps/libfutures-df761c89bfa71e54.rmeta --extern lazy_static=/work/oxidecomputer/crucible/target/debug/deps/liblazy_static-9c1a21fa7aad0259.rmeta --extern newtype_derive=/work/oxidecomputer/crucible/target/debug/deps/libnewtype_derive-fedf432d6b5251a4.rmeta --extern petgraph=/work/oxidecomputer/crucible/target/debug/deps/libpetgraph-8e9556a1882bb1d5.rmeta --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rmeta --extern slog=/work/oxidecomputer/crucible/target/debug/deps/libslog-84fd25666c3c26ee.rmeta --extern thiserror=/work/oxidecomputer/crucible/target/debug/deps/libthiserror-07eca56f531a0e5d.rmeta --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rmeta --extern uuid=/work/oxidecomputer/crucible/target/debug/deps/libuuid-7cc8e87b4149b49e.rmeta --cap-lints allow`
   Compiling parse-display v0.7.0
     Running `rustc --crate-name parse_display --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/parse-display-0.7.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="once_cell"' --cfg 'feature="regex"' --cfg 'feature="std"' -C metadata=34a1a5d52375b70b -C extra-filename=-34a1a5d52375b70b --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern once_cell=/work/oxidecomputer/crucible/target/debug/deps/libonce_cell-fdb5f9e769d1e589.rmeta --extern parse_display_derive=/work/oxidecomputer/crucible/target/debug/deps/libparse_display_derive-95e2e5d955c96bcf.so --extern regex=/work/oxidecomputer/crucible/target/debug/deps/libregex-f9e3a4eb3da387ce.rmeta --cap-lints allow`
   Compiling statistical v1.0.0
     Running `rustc --crate-name statistical /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/statistical-1.0.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=f2809857d7fe1c91 -C extra-filename=-f2809857d7fe1c91 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern num=/work/oxidecomputer/crucible/target/debug/deps/libnum-1148753de5cf68fd.rmeta --extern rand=/work/oxidecomputer/crucible/target/debug/deps/librand-d54af7a747f291ab.rmeta --cap-lints allow`
   Compiling serde_human_bytes v0.1.0 (http://github.com/oxidecomputer/serde_human_bytes?branch=main#0a097945)
     Running `rustc --crate-name serde_human_bytes --edition=2018 /home/build/.cargo/git/checkouts/serde_human_bytes-25cac76e2ef2d15c/0a09794/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=7f54d0fcbf9b36f3 -C extra-filename=-7f54d0fcbf9b36f3 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern hex=/work/oxidecomputer/crucible/target/debug/deps/libhex-6531b11cb72de3e5.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --cap-lints allow`
   Compiling num_enum_derive v0.7.0
     Running `rustc --crate-name num_enum_derive --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/num_enum_derive-0.7.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type proc-macro --emit=dep-info,link -C prefer-dynamic -C embed-bitcode=no --cfg 'feature="proc-macro-crate"' --cfg 'feature="std"' -C metadata=2bef02e01d5b06d3 -C extra-filename=-2bef02e01d5b06d3 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern proc_macro_crate=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro_crate-3935aefc56866ee2.rlib --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rlib --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rlib --extern syn=/work/oxidecomputer/crucible/target/debug/deps/libsyn-baedf68a9175a6da.rlib --extern proc_macro --cap-lints allow`
   Compiling camino-tempfile v1.0.2
     Running `rustc --crate-name camino_tempfile --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/camino-tempfile-1.0.2/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=38b6a8d85c9dc0c0 -C extra-filename=-38b6a8d85c9dc0c0 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern camino=/work/oxidecomputer/crucible/target/debug/deps/libcamino-45f0f4a2c258f934.rmeta --extern tempfile=/work/oxidecomputer/crucible/target/debug/deps/libtempfile-05cb6a5f7e86cb66.rmeta --cap-lints allow`
   Compiling ipnetwork v0.20.0
     Running `rustc --crate-name ipnetwork --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/ipnetwork-0.20.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="schemars"' --cfg 'feature="serde"' -C metadata=0e9e550a49db2c52 -C extra-filename=-0e9e550a49db2c52 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --cap-lints allow`
   Compiling macaddr v1.0.1
     Running `rustc --crate-name macaddr --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/macaddr-1.0.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="serde"' --cfg 'feature="serde_std"' --cfg 'feature="std"' -C metadata=98e89df75c36be48 -C extra-filename=-98e89df75c36be48 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --cap-lints allow`
   Compiling api_identity v0.1.0 (https://github.com/oxidecomputer/omicron?branch=main#4c05962d)
     Running `rustc --crate-name api_identity --edition=2021 /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/4c05962/api_identity/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type proc-macro --emit=dep-info,link -C prefer-dynamic -C embed-bitcode=no -C metadata=90d45ecc06c8f773 -C extra-filename=-90d45ecc06c8f773 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rlib --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rlib --extern syn=/work/oxidecomputer/crucible/target/debug/deps/libsyn-baedf68a9175a6da.rlib --extern proc_macro --cap-lints allow`
   Compiling base64ct v1.6.0
     Running `rustc --crate-name base64ct --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/base64ct-1.6.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="alloc"' --cfg 'feature="std"' -C metadata=5aa1f74fadba3334 -C extra-filename=-5aa1f74fadba3334 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling bincode v1.3.3
     Running `rustc --crate-name bincode /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/bincode-1.3.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=bcb925e8faac86cd -C extra-filename=-bcb925e8faac86cd --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --cap-lints allow`
   Compiling password-hash v0.5.0
     Running `rustc --crate-name password_hash --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/password-hash-0.5.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="alloc"' --cfg 'feature="default"' --cfg 'feature="rand_core"' --cfg 'feature="std"' -C metadata=6571709e5cce3ff3 -C extra-filename=-6571709e5cce3ff3 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern base64ct=/work/oxidecomputer/crucible/target/debug/deps/libbase64ct-5aa1f74fadba3334.rmeta --extern rand_core=/work/oxidecomputer/crucible/target/debug/deps/librand_core-e2870cc0e01c33c9.rmeta --extern subtle=/work/oxidecomputer/crucible/target/debug/deps/libsubtle-ee0c8d46ce57336b.rmeta --cap-lints allow`
   Compiling blake2 v0.10.6
     Running `rustc --crate-name blake2 --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/blake2-0.10.6/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=39c009971635667f -C extra-filename=-39c009971635667f --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern digest=/work/oxidecomputer/crucible/target/debug/deps/libdigest-d0d34473efe105d1.rmeta --cap-lints allow`
   Compiling inout v0.1.3
     Running `rustc --crate-name inout --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/inout-0.1.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=63af7c577909eb44 -C extra-filename=-63af7c577909eb44 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern generic_array=/work/oxidecomputer/crucible/target/debug/deps/libgeneric_array-bc31b01978a602e7.rmeta --cap-lints allow`
   Compiling cipher v0.4.3
     Running `rustc --crate-name cipher --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/cipher-0.4.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=fe5b83e4e2b73d72 -C extra-filename=-fe5b83e4e2b73d72 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern crypto_common=/work/oxidecomputer/crucible/target/debug/deps/libcrypto_common-0953bfc5dcef84b9.rmeta --extern inout=/work/oxidecomputer/crucible/target/debug/deps/libinout-63af7c577909eb44.rmeta --cap-lints allow`
   Compiling argon2 v0.5.0
     Running `rustc --crate-name argon2 --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/argon2-0.5.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="alloc"' --cfg 'feature="default"' --cfg 'feature="password-hash"' --cfg 'feature="rand"' --cfg 'feature="std"' -C metadata=8f03ab3a8d12bcba -C extra-filename=-8f03ab3a8d12bcba --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern base64ct=/work/oxidecomputer/crucible/target/debug/deps/libbase64ct-5aa1f74fadba3334.rmeta --extern blake2=/work/oxidecomputer/crucible/target/debug/deps/libblake2-39c009971635667f.rmeta --extern password_hash=/work/oxidecomputer/crucible/target/debug/deps/libpassword_hash-6571709e5cce3ff3.rmeta --cap-lints allow`
   Compiling oximeter-macro-impl v0.1.0 (https://github.com/oxidecomputer/omicron?branch=main#4c05962d)
     Running `rustc --crate-name oximeter_macro_impl --edition=2021 /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/4c05962/oximeter/oximeter-macro-impl/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type proc-macro --emit=dep-info,link -C prefer-dynamic -C embed-bitcode=no -C metadata=e4cc949eda20c416 -C extra-filename=-e4cc949eda20c416 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rlib --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rlib --extern syn=/work/oxidecomputer/crucible/target/debug/deps/libsyn-baedf68a9175a6da.rlib --extern proc_macro --cap-lints allow`
   Compiling omicron-passwords v0.1.0 (https://github.com/oxidecomputer/omicron?branch=main#4c05962d)
     Running `rustc --crate-name omicron_passwords --edition=2021 /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/4c05962/passwords/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=ac6e3a602e6ad041 -C extra-filename=-ac6e3a602e6ad041 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern argon2=/work/oxidecomputer/crucible/target/debug/deps/libargon2-8f03ab3a8d12bcba.rmeta --extern rand=/work/oxidecomputer/crucible/target/debug/deps/librand-1f91a9ea4aed49ee.rmeta --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern serde_with=/work/oxidecomputer/crucible/target/debug/deps/libserde_with-4f9ddd30b380d6cf.rmeta --extern thiserror=/work/oxidecomputer/crucible/target/debug/deps/libthiserror-07eca56f531a0e5d.rmeta --cap-lints allow`
   Compiling universal-hash v0.5.0
     Running `rustc --crate-name universal_hash --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/universal-hash-0.5.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=4588bd931f24d922 -C extra-filename=-4588bd931f24d922 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern crypto_common=/work/oxidecomputer/crucible/target/debug/deps/libcrypto_common-0953bfc5dcef84b9.rmeta --extern subtle=/work/oxidecomputer/crucible/target/debug/deps/libsubtle-ee0c8d46ce57336b.rmeta --cap-lints allow`
   Compiling num_enum v0.7.0
     Running `rustc --crate-name num_enum --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/num_enum-0.7.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=9cd7a6d9dcf1dd5a -C extra-filename=-9cd7a6d9dcf1dd5a --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern num_enum_derive=/work/oxidecomputer/crucible/target/debug/deps/libnum_enum_derive-2bef02e01d5b06d3.so --cap-lints allow`
   Compiling opaque-debug v0.3.0
     Running `rustc --crate-name opaque_debug --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/opaque-debug-0.3.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=bdee85e4e8a367bc -C extra-filename=-bdee85e4e8a367bc --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling polyval v0.6.0
   Compiling aes v0.8.2
     Running `rustc --crate-name polyval --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/polyval-0.6.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=e4b47d7c1f31f998 -C extra-filename=-e4b47d7c1f31f998 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern cfg_if=/work/oxidecomputer/crucible/target/debug/deps/libcfg_if-a93276574f0edf39.rmeta --extern cpufeatures=/work/oxidecomputer/crucible/target/debug/deps/libcpufeatures-6e25ef20a8fa218f.rmeta --extern opaque_debug=/work/oxidecomputer/crucible/target/debug/deps/libopaque_debug-bdee85e4e8a367bc.rmeta --extern universal_hash=/work/oxidecomputer/crucible/target/debug/deps/libuniversal_hash-4588bd931f24d922.rmeta --cap-lints allow`
     Running `rustc --crate-name aes --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/aes-0.8.2/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=e0592e8e1d03d860 -C extra-filename=-e0592e8e1d03d860 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern cfg_if=/work/oxidecomputer/crucible/target/debug/deps/libcfg_if-a93276574f0edf39.rmeta --extern cipher=/work/oxidecomputer/crucible/target/debug/deps/libcipher-fe5b83e4e2b73d72.rmeta --extern cpufeatures=/work/oxidecomputer/crucible/target/debug/deps/libcpufeatures-6e25ef20a8fa218f.rmeta --cap-lints allow`
   Compiling ctr v0.9.2
     Running `rustc --crate-name ctr --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/ctr-0.9.2/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=583e100d27882194 -C extra-filename=-583e100d27882194 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern cipher=/work/oxidecomputer/crucible/target/debug/deps/libcipher-fe5b83e4e2b73d72.rmeta --cap-lints allow`
   Compiling crucible-client-types v0.1.0 (/work/oxidecomputer/crucible/crucible-client-types)
     Running `rustc --crate-name crucible_client_types --edition=2021 crucible-client-types/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=bd54c4335d2370bd -C extra-filename=-bd54c4335d2370bd --out-dir /work/oxidecomputer/crucible/target/debug/deps -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern base64=/work/oxidecomputer/crucible/target/debug/deps/libbase64-c139bdd129e780ac.rmeta --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rmeta --extern uuid=/work/oxidecomputer/crucible/target/debug/deps/libuuid-7cc8e87b4149b49e.rmeta`
   Compiling aead v0.5.1
     Running `rustc --crate-name aead --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/aead-0.5.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="alloc"' --cfg 'feature="getrandom"' --cfg 'feature="rand_core"' -C metadata=f30da292094eb963 -C extra-filename=-f30da292094eb963 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern crypto_common=/work/oxidecomputer/crucible/target/debug/deps/libcrypto_common-0953bfc5dcef84b9.rmeta --extern generic_array=/work/oxidecomputer/crucible/target/debug/deps/libgeneric_array-bc31b01978a602e7.rmeta --cap-lints allow`
   Compiling zeroize v1.3.0
     Running `rustc --crate-name zeroize --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/zeroize-1.3.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=8e99cf4813483d58 -C extra-filename=-8e99cf4813483d58 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling itertools v0.11.0
     Running `rustc --crate-name itertools --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/itertools-0.11.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="use_alloc"' --cfg 'feature="use_std"' -C metadata=b06e69badd72e55c -C extra-filename=-b06e69badd72e55c --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern either=/work/oxidecomputer/crucible/target/debug/deps/libeither-e075e5427abad613.rmeta --cap-lints allow`
   Compiling async-recursion v1.0.5
     Running `rustc --crate-name async_recursion --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/async-recursion-1.0.5/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type proc-macro --emit=dep-info,link -C prefer-dynamic -C embed-bitcode=no -C metadata=ce9499495a1cb858 -C extra-filename=-ce9499495a1cb858 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rlib --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rlib --extern syn=/work/oxidecomputer/crucible/target/debug/deps/libsyn-baedf68a9175a6da.rlib --extern proc_macro --cap-lints allow`
   Compiling ringbuffer v0.14.2
     Running `rustc --crate-name ringbuffer --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/ringbuffer-0.14.2/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="alloc"' --cfg 'feature="default"' -C metadata=e91d75e4694e6351 -C extra-filename=-e91d75e4694e6351 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling aes-gcm-siv v0.11.1
     Running `rustc --crate-name aes_gcm_siv --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/aes-gcm-siv-0.11.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="aes"' --cfg 'feature="alloc"' --cfg 'feature="default"' --cfg 'feature="getrandom"' -C metadata=21495b616a07c9a4 -C extra-filename=-21495b616a07c9a4 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern aead=/work/oxidecomputer/crucible/target/debug/deps/libaead-f30da292094eb963.rmeta --extern aes=/work/oxidecomputer/crucible/target/debug/deps/libaes-e0592e8e1d03d860.rmeta --extern cipher=/work/oxidecomputer/crucible/target/debug/deps/libcipher-fe5b83e4e2b73d72.rmeta --extern ctr=/work/oxidecomputer/crucible/target/debug/deps/libctr-583e100d27882194.rmeta --extern polyval=/work/oxidecomputer/crucible/target/debug/deps/libpolyval-e4b47d7c1f31f998.rmeta --extern subtle=/work/oxidecomputer/crucible/target/debug/deps/libsubtle-ee0c8d46ce57336b.rmeta --extern zeroize=/work/oxidecomputer/crucible/target/debug/deps/libzeroize-8e99cf4813483d58.rmeta --cap-lints allow`
   Compiling unicode-segmentation v1.10.0
     Running `rustc --crate-name unicode_segmentation --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/unicode-segmentation-1.10.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=06176721b7b95955 -C extra-filename=-06176721b7b95955 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling console v0.15.5
     Running `rustc --crate-name console --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/console-0.15.5/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="ansi-parsing"' --cfg 'feature="default"' --cfg 'feature="unicode-width"' -C metadata=4236472a6e29ce0a -C extra-filename=-4236472a6e29ce0a --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern lazy_static=/work/oxidecomputer/crucible/target/debug/deps/liblazy_static-9c1a21fa7aad0259.rmeta --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --extern unicode_width=/work/oxidecomputer/crucible/target/debug/deps/libunicode_width-87c5262a4c4bb0e9.rmeta --cap-lints allow`
   Compiling opentelemetry_api v0.18.0
     Running `rustc --crate-name opentelemetry_api --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/opentelemetry_api-0.18.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="fnv"' --cfg 'feature="metrics"' --cfg 'feature="pin-project-lite"' --cfg 'feature="trace"' -C metadata=54be0f0e21b05b9c -C extra-filename=-54be0f0e21b05b9c --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern fnv=/work/oxidecomputer/crucible/target/debug/deps/libfnv-5641130f60a8056b.rmeta --extern futures_channel=/work/oxidecomputer/crucible/target/debug/deps/libfutures_channel-34a7a018f58dc5a7.rmeta --extern futures_util=/work/oxidecomputer/crucible/target/debug/deps/libfutures_util-b4da5d5433271d56.rmeta --extern indexmap=/work/oxidecomputer/crucible/target/debug/deps/libindexmap-c2c8f74266eebb64.rmeta --extern once_cell=/work/oxidecomputer/crucible/target/debug/deps/libonce_cell-fdb5f9e769d1e589.rmeta --extern pin_project_lite=/work/oxidecomputer/crucible/target/debug/deps/libpin_project_lite-c035e320730303c2.rmeta --extern thiserror=/work/oxidecomputer/crucible/target/debug/deps/libthiserror-07eca56f531a0e5d.rmeta --cap-lints allow`
   Compiling dashmap v5.2.0
     Running `rustc --crate-name dashmap --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dashmap-5.2.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="send_guard"' -C metadata=e6d3c88fce252f4a -C extra-filename=-e6d3c88fce252f4a --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern cfg_if=/work/oxidecomputer/crucible/target/debug/deps/libcfg_if-a93276574f0edf39.rmeta --extern num_cpus=/work/oxidecomputer/crucible/target/debug/deps/libnum_cpus-67a451bebfcc5086.rmeta --extern parking_lot=/work/oxidecomputer/crucible/target/debug/deps/libparking_lot-970d5c0acece447c.rmeta --cap-lints allow`
   Compiling typify-macro v0.0.13 (https://github.com/oxidecomputer/typify#92bfed8b)
     Running `rustc --crate-name typify_macro --edition=2021 /home/build/.cargo/git/checkouts/typify-288d5a84bbbe6a46/92bfed8/typify-macro/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type proc-macro --emit=dep-info,link -C prefer-dynamic -C embed-bitcode=no -C metadata=f2da351574d5abd0 -C extra-filename=-f2da351574d5abd0 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rlib --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rlib --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rlib --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rlib --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rlib --extern serde_tokenstream=/work/oxidecomputer/crucible/target/debug/deps/libserde_tokenstream-2a83027836bfd0fb.rlib --extern syn=/work/oxidecomputer/crucible/target/debug/deps/libsyn-baedf68a9175a6da.rlib --extern typify_impl=/work/oxidecomputer/crucible/target/debug/deps/libtypify_impl-5385296f2ea50467.rlib --extern proc_macro --cap-lints allow`
   Compiling newline-converter v0.3.0
     Running `rustc --crate-name newline_converter --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/newline-converter-0.3.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=54456d7e7e0ff19b -C extra-filename=-54456d7e7e0ff19b --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern unicode_segmentation=/work/oxidecomputer/crucible/target/debug/deps/libunicode_segmentation-06176721b7b95955.rmeta --cap-lints allow`
   Compiling num-derive v0.4.0
     Running `rustc --crate-name num_derive --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/num-derive-0.4.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type proc-macro --emit=dep-info,link -C prefer-dynamic -C embed-bitcode=no -C metadata=814c8a0a0a713cba -C extra-filename=-814c8a0a0a713cba --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rlib --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rlib --extern syn=/work/oxidecomputer/crucible/target/debug/deps/libsyn-baedf68a9175a6da.rlib --extern proc_macro --cap-lints allow`
   Compiling convert_case v0.5.0
     Running `rustc --crate-name convert_case --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/convert_case-0.5.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=1b3c1b1b6d16787f -C extra-filename=-1b3c1b1b6d16787f --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling similar v2.2.1
     Running `rustc --crate-name similar --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/similar-2.2.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="text"' -C metadata=c5a411995c7e1b53 -C extra-filename=-c5a411995c7e1b53 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling opentelemetry_sdk v0.18.0
     Running `rustc --crate-name opentelemetry_sdk --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/opentelemetry_sdk-0.18.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="async-trait"' --cfg 'feature="crossbeam-channel"' --cfg 'feature="dashmap"' --cfg 'feature="default"' --cfg 'feature="fnv"' --cfg 'feature="metrics"' --cfg 'feature="percent-encoding"' --cfg 'feature="rand"' --cfg 'feature="trace"' -C metadata=f1be95b27d4b99f6 -C extra-filename=-f1be95b27d4b99f6 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern async_trait=/work/oxidecomputer/crucible/target/debug/deps/libasync_trait-a300e84178ee0ad1.so --extern crossbeam_channel=/work/oxidecomputer/crucible/target/debug/deps/libcrossbeam_channel-19ff6fa486e51c63.rmeta --extern dashmap=/work/oxidecomputer/crucible/target/debug/deps/libdashmap-e6d3c88fce252f4a.rmeta --extern fnv=/work/oxidecomputer/crucible/target/debug/deps/libfnv-5641130f60a8056b.rmeta --extern futures_channel=/work/oxidecomputer/crucible/target/debug/deps/libfutures_channel-34a7a018f58dc5a7.rmeta --extern futures_executor=/work/oxidecomputer/crucible/target/debug/deps/libfutures_executor-61f53162107ffb32.rmeta --extern futures_util=/work/oxidecomputer/crucible/target/debug/deps/libfutures_util-b4da5d5433271d56.rmeta --extern once_cell=/work/oxidecomputer/crucible/target/debug/deps/libonce_cell-fdb5f9e769d1e589.rmeta --extern opentelemetry_api=/work/oxidecomputer/crucible/target/debug/deps/libopentelemetry_api-54be0f0e21b05b9c.rmeta --extern percent_encoding=/work/oxidecomputer/crucible/target/debug/deps/libpercent_encoding-0000aebce3d30803.rmeta --extern rand=/work/oxidecomputer/crucible/target/debug/deps/librand-1f91a9ea4aed49ee.rmeta --extern thiserror=/work/oxidecomputer/crucible/target/debug/deps/libthiserror-07eca56f531a0e5d.rmeta --cap-lints allow`
   Compiling expectorate v1.0.7
     Running `rustc --crate-name expectorate --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/expectorate-1.0.7/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=4e45b262baa473cc -C extra-filename=-4e45b262baa473cc --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern console=/work/oxidecomputer/crucible/target/debug/deps/libconsole-4236472a6e29ce0a.rmeta --extern newline_converter=/work/oxidecomputer/crucible/target/debug/deps/libnewline_converter-54456d7e7e0ff19b.rmeta --extern similar=/work/oxidecomputer/crucible/target/debug/deps/libsimilar-c5a411995c7e1b53.rmeta --cap-lints allow`
   Compiling openapi-lint v0.1.0 (https://github.com/oxidecomputer/openapi-lint?branch=main#9e3ada82)
     Running `rustc --crate-name openapi_lint --edition=2018 /home/build/.cargo/git/checkouts/openapi-lint-42a90d71bc44c8c6/9e3ada8/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=848bb99097a9a843 -C extra-filename=-848bb99097a9a843 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern convert_case=/work/oxidecomputer/crucible/target/debug/deps/libconvert_case-1b3c1b1b6d16787f.rmeta --extern indexmap=/work/oxidecomputer/crucible/target/debug/deps/libindexmap-c2c8f74266eebb64.rmeta --extern openapiv3=/work/oxidecomputer/crucible/target/debug/deps/libopenapiv3-f84072cef6d0d68c.rmeta --cap-lints allow`
   Compiling unicase v2.6.0
     Running `rustc --crate-name unicase /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/unicase-2.6.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=60e4f37ecb23afe1 -C extra-filename=-60e4f37ecb23afe1 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow --cfg __unicase__iter_cmp --cfg __unicase__default_hasher --cfg __unicase__const_fns --cfg __unicase__core_and_alloc`
   Compiling urlencoding v2.1.2
     Running `rustc --crate-name urlencoding --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/urlencoding-2.1.2/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=76c058d20df73e40 -C extra-filename=-76c058d20df73e40 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling overload v0.1.1
     Running `rustc --crate-name overload --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/overload-0.1.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=c7048b1d029106d7 -C extra-filename=-c7048b1d029106d7 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling nu-ansi-term v0.46.0
     Running `rustc --crate-name nu_ansi_term --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/nu-ansi-term-0.46.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=91641c4d652bb213 -C extra-filename=-91641c4d652bb213 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern overload=/work/oxidecomputer/crucible/target/debug/deps/liboverload-c7048b1d029106d7.rmeta --cap-lints allow`
   Compiling opentelemetry_api v0.20.0
     Running `rustc --crate-name opentelemetry_api --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/opentelemetry_api-0.20.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="pin-project-lite"' --cfg 'feature="trace"' -C metadata=04e099a48329a859 -C extra-filename=-04e099a48329a859 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern futures_channel=/work/oxidecomputer/crucible/target/debug/deps/libfutures_channel-34a7a018f58dc5a7.rmeta --extern futures_util=/work/oxidecomputer/crucible/target/debug/deps/libfutures_util-b4da5d5433271d56.rmeta --extern indexmap=/work/oxidecomputer/crucible/target/debug/deps/libindexmap-c2c8f74266eebb64.rmeta --extern once_cell=/work/oxidecomputer/crucible/target/debug/deps/libonce_cell-fdb5f9e769d1e589.rmeta --extern pin_project_lite=/work/oxidecomputer/crucible/target/debug/deps/libpin_project_lite-c035e320730303c2.rmeta --extern thiserror=/work/oxidecomputer/crucible/target/debug/deps/libthiserror-07eca56f531a0e5d.rmeta --extern urlencoding=/work/oxidecomputer/crucible/target/debug/deps/liburlencoding-76c058d20df73e40.rmeta --cap-lints allow`
   Compiling crucible-smf v0.0.0 (/work/oxidecomputer/crucible/smf)
     Running `rustc --crate-name crucible_smf --edition=2021 smf/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=e1a82c6f17385dc6 -C extra-filename=-e1a82c6f17385dc6 --out-dir /work/oxidecomputer/crucible/target/debug/deps -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --extern num_derive=/work/oxidecomputer/crucible/target/debug/deps/libnum_derive-814c8a0a0a713cba.so --extern num_traits=/work/oxidecomputer/crucible/target/debug/deps/libnum_traits-8e50de91aba3f8f9.rmeta --extern thiserror=/work/oxidecomputer/crucible/target/debug/deps/libthiserror-07eca56f531a0e5d.rmeta`
   Compiling opentelemetry v0.18.0
     Running `rustc --crate-name opentelemetry --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/opentelemetry-0.18.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="metrics"' --cfg 'feature="trace"' -C metadata=f41372bdd269a545 -C extra-filename=-f41372bdd269a545 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern opentelemetry_api=/work/oxidecomputer/crucible/target/debug/deps/libopentelemetry_api-54be0f0e21b05b9c.rmeta --extern opentelemetry_sdk=/work/oxidecomputer/crucible/target/debug/deps/libopentelemetry_sdk-f1be95b27d4b99f6.rmeta --cap-lints allow`
   Compiling mime_guess v2.0.4
     Running `rustc --crate-name build_script_build /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/mime_guess-2.0.4/build.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no --cfg 'feature="default"' --cfg 'feature="rev-mappings"' -C metadata=dbd0371f2a220632 -C extra-filename=-dbd0371f2a220632 --out-dir /work/oxidecomputer/crucible/target/debug/build/mime_guess-dbd0371f2a220632 -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern unicase=/work/oxidecomputer/crucible/target/debug/deps/libunicase-60e4f37ecb23afe1.rlib --cap-lints allow`
   Compiling tracing-log v0.1.3
     Running `rustc --crate-name tracing_log --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tracing-log-0.1.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="log-tracer"' --cfg 'feature="std"' -C metadata=d77683210f5d223d -C extra-filename=-d77683210f5d223d --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern lazy_static=/work/oxidecomputer/crucible/target/debug/deps/liblazy_static-9c1a21fa7aad0259.rmeta --extern log=/work/oxidecomputer/crucible/target/debug/deps/liblog-930e200cffaa7930.rmeta --extern tracing_core=/work/oxidecomputer/crucible/target/debug/deps/libtracing_core-adac59f754126e83.rmeta --cap-lints allow`
   Compiling sharded-slab v0.1.4
     Running `rustc --crate-name sharded_slab --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/sharded-slab-0.1.4/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=9ba6a384880b5495 -C extra-filename=-9ba6a384880b5495 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern lazy_static=/work/oxidecomputer/crucible/target/debug/deps/liblazy_static-9c1a21fa7aad0259.rmeta --cap-lints allow`
   Compiling ordered-float v3.7.0
     Running `rustc --crate-name ordered_float --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/ordered-float-3.7.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=c435708391037060 -C extra-filename=-c435708391037060 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern num_traits=/work/oxidecomputer/crucible/target/debug/deps/libnum_traits-8e50de91aba3f8f9.rmeta --cap-lints allow`
   Compiling ordered-float v1.1.1
     Running `rustc --crate-name ordered_float /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/ordered-float-1.1.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=b7d47a63e5afa582 -C extra-filename=-b7d47a63e5afa582 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern num_traits=/work/oxidecomputer/crucible/target/debug/deps/libnum_traits-8e50de91aba3f8f9.rmeta --cap-lints allow`
   Compiling threadpool v1.8.1
     Running `rustc --crate-name threadpool /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/threadpool-1.8.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=e4e99e14eaa48b3e -C extra-filename=-e4e99e14eaa48b3e --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern num_cpus=/work/oxidecomputer/crucible/target/debug/deps/libnum_cpus-67a451bebfcc5086.rmeta --cap-lints allow`
   Compiling integer-encoding v3.0.3
     Running `rustc --crate-name integer_encoding --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/integer-encoding-3.0.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=d6f198d882d03cc4 -C extra-filename=-d6f198d882d03cc4 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
     Running `/work/oxidecomputer/crucible/target/debug/build/mime_guess-dbd0371f2a220632/build-script-build`
   Compiling opentelemetry_sdk v0.20.0
     Running `rustc --crate-name opentelemetry_sdk --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/opentelemetry_sdk-0.20.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="async-trait"' --cfg 'feature="crossbeam-channel"' --cfg 'feature="default"' --cfg 'feature="percent-encoding"' --cfg 'feature="rand"' --cfg 'feature="trace"' -C metadata=6fef3b11a1793dc3 -C extra-filename=-6fef3b11a1793dc3 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern async_trait=/work/oxidecomputer/crucible/target/debug/deps/libasync_trait-a300e84178ee0ad1.so --extern crossbeam_channel=/work/oxidecomputer/crucible/target/debug/deps/libcrossbeam_channel-19ff6fa486e51c63.rmeta --extern futures_channel=/work/oxidecomputer/crucible/target/debug/deps/libfutures_channel-34a7a018f58dc5a7.rmeta --extern futures_executor=/work/oxidecomputer/crucible/target/debug/deps/libfutures_executor-61f53162107ffb32.rmeta --extern futures_util=/work/oxidecomputer/crucible/target/debug/deps/libfutures_util-b4da5d5433271d56.rmeta --extern once_cell=/work/oxidecomputer/crucible/target/debug/deps/libonce_cell-fdb5f9e769d1e589.rmeta --extern opentelemetry_api=/work/oxidecomputer/crucible/target/debug/deps/libopentelemetry_api-04e099a48329a859.rmeta --extern ordered_float=/work/oxidecomputer/crucible/target/debug/deps/libordered_float-c435708391037060.rmeta --extern percent_encoding=/work/oxidecomputer/crucible/target/debug/deps/libpercent_encoding-0000aebce3d30803.rmeta --extern rand=/work/oxidecomputer/crucible/target/debug/deps/librand-1f91a9ea4aed49ee.rmeta --extern thiserror=/work/oxidecomputer/crucible/target/debug/deps/libthiserror-07eca56f531a0e5d.rmeta --cap-lints allow`
   Compiling opentelemetry-semantic-conventions v0.10.0
     Running `rustc --crate-name opentelemetry_semantic_conventions --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/opentelemetry-semantic-conventions-0.10.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=18113e5418aec521 -C extra-filename=-18113e5418aec521 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern opentelemetry=/work/oxidecomputer/crucible/target/debug/deps/libopentelemetry-f41372bdd269a545.rmeta --cap-lints allow`
   Compiling thrift v0.16.0
     Running `rustc --crate-name thrift --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/thrift-0.16.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="log"' --cfg 'feature="server"' --cfg 'feature="threadpool"' -C metadata=1fac1fb5888ad036 -C extra-filename=-1fac1fb5888ad036 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern byteorder=/work/oxidecomputer/crucible/target/debug/deps/libbyteorder-1306999913c8e1b3.rmeta --extern integer_encoding=/work/oxidecomputer/crucible/target/debug/deps/libinteger_encoding-d6f198d882d03cc4.rmeta --extern log=/work/oxidecomputer/crucible/target/debug/deps/liblog-930e200cffaa7930.rmeta --extern ordered_float=/work/oxidecomputer/crucible/target/debug/deps/libordered_float-b7d47a63e5afa582.rmeta --extern threadpool=/work/oxidecomputer/crucible/target/debug/deps/libthreadpool-e4e99e14eaa48b3e.rmeta --cap-lints allow`
   Compiling tracing-subscriber v0.3.17
     Running `rustc --crate-name tracing_subscriber --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tracing-subscriber-0.3.17/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="alloc"' --cfg 'feature="ansi"' --cfg 'feature="default"' --cfg 'feature="fmt"' --cfg 'feature="nu-ansi-term"' --cfg 'feature="registry"' --cfg 'feature="sharded-slab"' --cfg 'feature="smallvec"' --cfg 'feature="std"' --cfg 'feature="thread_local"' --cfg 'feature="tracing-log"' -C metadata=e39dae5ba339bc78 -C extra-filename=-e39dae5ba339bc78 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern nu_ansi_term=/work/oxidecomputer/crucible/target/debug/deps/libnu_ansi_term-91641c4d652bb213.rmeta --extern sharded_slab=/work/oxidecomputer/crucible/target/debug/deps/libsharded_slab-9ba6a384880b5495.rmeta --extern smallvec=/work/oxidecomputer/crucible/target/debug/deps/libsmallvec-397f26bd8c84e528.rmeta --extern thread_local=/work/oxidecomputer/crucible/target/debug/deps/libthread_local-771d112d9219dc9f.rmeta --extern tracing_core=/work/oxidecomputer/crucible/target/debug/deps/libtracing_core-adac59f754126e83.rmeta --extern tracing_log=/work/oxidecomputer/crucible/target/debug/deps/libtracing_log-d77683210f5d223d.rmeta --cap-lints allow`
   Compiling phf_shared v0.11.1
     Running `rustc --crate-name phf_shared --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/phf_shared-0.11.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=f4c611c5b95ef91c -C extra-filename=-f4c611c5b95ef91c --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern siphasher=/work/oxidecomputer/crucible/target/debug/deps/libsiphasher-2eb27360f66646f3.rmeta --cap-lints allow`
   Compiling typify v0.0.13 (https://github.com/oxidecomputer/typify#92bfed8b)
     Running `rustc --crate-name typify --edition=2021 /home/build/.cargo/git/checkouts/typify-288d5a84bbbe6a46/92bfed8/typify/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="macro"' --cfg 'feature="typify-macro"' -C metadata=60072fbcea665837 -C extra-filename=-60072fbcea665837 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern typify_impl=/work/oxidecomputer/crucible/target/debug/deps/libtypify_impl-5385296f2ea50467.rmeta --extern typify_macro=/work/oxidecomputer/crucible/target/debug/deps/libtypify_macro-f2da351574d5abd0.so --cap-lints allow`
   Compiling progenitor-impl v0.3.0 (https://github.com/oxidecomputer/progenitor?branch=main#76716eea)
     Running `rustc --crate-name progenitor_impl --edition=2021 /home/build/.cargo/git/checkouts/progenitor-639bd64206ac5e43/76716ee/progenitor-impl/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=4eeb9d4c0f08c454 -C extra-filename=-4eeb9d4c0f08c454 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern getopts=/work/oxidecomputer/crucible/target/debug/deps/libgetopts-93a8419d37acce69.rmeta --extern heck=/work/oxidecomputer/crucible/target/debug/deps/libheck-29af75c938b110f7.rmeta --extern http=/work/oxidecomputer/crucible/target/debug/deps/libhttp-4526ab951518c907.rmeta --extern indexmap=/work/oxidecomputer/crucible/target/debug/deps/libindexmap-c2c8f74266eebb64.rmeta --extern openapiv3=/work/oxidecomputer/crucible/target/debug/deps/libopenapiv3-f84072cef6d0d68c.rmeta --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rmeta --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rmeta --extern regex=/work/oxidecomputer/crucible/target/debug/deps/libregex-f9e3a4eb3da387ce.rmeta --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rmeta --extern syn=/work/oxidecomputer/crucible/target/debug/deps/libsyn-baedf68a9175a6da.rmeta --extern thiserror=/work/oxidecomputer/crucible/target/debug/deps/libthiserror-07eca56f531a0e5d.rmeta --extern typify=/work/oxidecomputer/crucible/target/debug/deps/libtypify-60072fbcea665837.rmeta --extern unicode_ident=/work/oxidecomputer/crucible/target/debug/deps/libunicode_ident-81e7752fff89e70f.rmeta --cap-lints allow`
     Running `rustc --crate-name mime_guess /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/mime_guess-2.0.4/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="rev-mappings"' -C metadata=66974d6c31968dc2 -C extra-filename=-66974d6c31968dc2 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern mime=/work/oxidecomputer/crucible/target/debug/deps/libmime-ac14a9115eddd3c2.rmeta --extern unicase=/work/oxidecomputer/crucible/target/debug/deps/libunicase-60e4f37ecb23afe1.rmeta --cap-lints allow`
   Compiling opentelemetry v0.20.0
     Running `rustc --crate-name opentelemetry --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/opentelemetry-0.20.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="trace"' -C metadata=5524fd7817ad57fb -C extra-filename=-5524fd7817ad57fb --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern opentelemetry_api=/work/oxidecomputer/crucible/target/debug/deps/libopentelemetry_api-04e099a48329a859.rmeta --extern opentelemetry_sdk=/work/oxidecomputer/crucible/target/debug/deps/libopentelemetry_sdk-6fef3b11a1793dc3.rmeta --cap-lints allow`
   Compiling opentelemetry-jaeger v0.17.0
     Running `rustc --crate-name opentelemetry_jaeger --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/opentelemetry-jaeger-0.17.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' -C metadata=9ebf47742e5e063f -C extra-filename=-9ebf47742e5e063f --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern async_trait=/work/oxidecomputer/crucible/target/debug/deps/libasync_trait-a300e84178ee0ad1.so --extern futures=/work/oxidecomputer/crucible/target/debug/deps/libfutures-df761c89bfa71e54.rmeta --extern futures_executor=/work/oxidecomputer/crucible/target/debug/deps/libfutures_executor-61f53162107ffb32.rmeta --extern once_cell=/work/oxidecomputer/crucible/target/debug/deps/libonce_cell-fdb5f9e769d1e589.rmeta --extern opentelemetry=/work/oxidecomputer/crucible/target/debug/deps/libopentelemetry-f41372bdd269a545.rmeta --extern opentelemetry_semantic_conventions=/work/oxidecomputer/crucible/target/debug/deps/libopentelemetry_semantic_conventions-18113e5418aec521.rmeta --extern thiserror=/work/oxidecomputer/crucible/target/debug/deps/libthiserror-07eca56f531a0e5d.rmeta --extern thrift=/work/oxidecomputer/crucible/target/debug/deps/libthrift-1fac1fb5888ad036.rmeta --cap-lints allow`
   Compiling signal-hook v0.3.17
     Running `rustc --crate-name signal_hook --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/signal-hook-0.3.17/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="channel"' --cfg 'feature="default"' --cfg 'feature="iterator"' -C metadata=ae206b38743b6815 -C extra-filename=-ae206b38743b6815 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --extern signal_hook_registry=/work/oxidecomputer/crucible/target/debug/deps/libsignal_hook_registry-4d955479f235827e.rmeta --cap-lints allow`
   Compiling http-range v0.1.5
     Running `rustc --crate-name http_range --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/http-range-0.1.5/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=4885ae91fe1ea75f -C extra-filename=-4885ae91fe1ea75f --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling memoffset v0.6.5
     Running `rustc --crate-name memoffset /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/memoffset-0.6.5/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' -C metadata=508ecbf92309dc73 -C extra-filename=-508ecbf92309dc73 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow --cfg tuple_ty --cfg allow_clippy --cfg maybe_uninit --cfg doctests --cfg raw_ref_macros`
   Compiling tracing-opentelemetry v0.18.0
     Running `rustc --crate-name tracing_opentelemetry --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tracing-opentelemetry-0.18.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="metrics"' --cfg 'feature="tracing-log"' -C metadata=8f8cfc1900c3a663 -C extra-filename=-8f8cfc1900c3a663 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern once_cell=/work/oxidecomputer/crucible/target/debug/deps/libonce_cell-fdb5f9e769d1e589.rmeta --extern opentelemetry=/work/oxidecomputer/crucible/target/debug/deps/libopentelemetry-f41372bdd269a545.rmeta --extern tracing=/work/oxidecomputer/crucible/target/debug/deps/libtracing-1de351a7f2b0ab55.rmeta --extern tracing_core=/work/oxidecomputer/crucible/target/debug/deps/libtracing_core-adac59f754126e83.rmeta --extern tracing_log=/work/oxidecomputer/crucible/target/debug/deps/libtracing_log-d77683210f5d223d.rmeta --extern tracing_subscriber=/work/oxidecomputer/crucible/target/debug/deps/libtracing_subscriber-e39dae5ba339bc78.rmeta --cap-lints allow`
   Compiling hyper-staticfile v0.9.4
     Running `rustc --crate-name hyper_staticfile --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/hyper-staticfile-0.9.4/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=559b4389ef952563 -C extra-filename=-559b4389ef952563 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern futures_util=/work/oxidecomputer/crucible/target/debug/deps/libfutures_util-b4da5d5433271d56.rmeta --extern http=/work/oxidecomputer/crucible/target/debug/deps/libhttp-4526ab951518c907.rmeta --extern http_range=/work/oxidecomputer/crucible/target/debug/deps/libhttp_range-4885ae91fe1ea75f.rmeta --extern httpdate=/work/oxidecomputer/crucible/target/debug/deps/libhttpdate-e3ef82a990113a54.rmeta --extern hyper=/work/oxidecomputer/crucible/target/debug/deps/libhyper-dad943d3b7cc33e9.rmeta --extern mime_guess=/work/oxidecomputer/crucible/target/debug/deps/libmime_guess-66974d6c31968dc2.rmeta --extern percent_encoding=/work/oxidecomputer/crucible/target/debug/deps/libpercent_encoding-0000aebce3d30803.rmeta --extern rand=/work/oxidecomputer/crucible/target/debug/deps/librand-1f91a9ea4aed49ee.rmeta --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rmeta --extern url=/work/oxidecomputer/crucible/target/debug/deps/liburl-ff56943ab9066fdc.rmeta --cap-lints allow`
   Compiling phf_generator v0.11.1
     Running `rustc --crate-name phf_generator --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/phf_generator-0.11.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C metadata=41d436cb237437bf -C extra-filename=-41d436cb237437bf --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern phf_shared=/work/oxidecomputer/crucible/target/debug/deps/libphf_shared-f4c611c5b95ef91c.rmeta --extern rand=/work/oxidecomputer/crucible/target/debug/deps/librand-1f91a9ea4aed49ee.rmeta --cap-lints allow`
   Compiling phf_codegen v0.11.1
     Running `rustc --crate-name phf_codegen --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/phf_codegen-0.11.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C metadata=f8063558873b33a1 -C extra-filename=-f8063558873b33a1 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern phf_generator=/work/oxidecomputer/crucible/target/debug/deps/libphf_generator-41d436cb237437bf.rmeta --extern phf_shared=/work/oxidecomputer/crucible/target/debug/deps/libphf_shared-f4c611c5b95ef91c.rmeta --cap-lints allow`
   Compiling crossbeam-epoch v0.9.8
     Running `rustc --crate-name crossbeam_epoch --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/crossbeam-epoch-0.9.8/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="alloc"' --cfg 'feature="lazy_static"' --cfg 'feature="std"' -C metadata=44fc1cc817fcdd23 -C extra-filename=-44fc1cc817fcdd23 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern cfg_if=/work/oxidecomputer/crucible/target/debug/deps/libcfg_if-a93276574f0edf39.rmeta --extern crossbeam_utils=/work/oxidecomputer/crucible/target/debug/deps/libcrossbeam_utils-a1fb255bfa31483a.rmeta --extern lazy_static=/work/oxidecomputer/crucible/target/debug/deps/liblazy_static-9c1a21fa7aad0259.rmeta --extern memoffset=/work/oxidecomputer/crucible/target/debug/deps/libmemoffset-508ecbf92309dc73.rmeta --extern scopeguard=/work/oxidecomputer/crucible/target/debug/deps/libscopeguard-00b7ece4eb7b8e7e.rmeta --cap-lints allow --cfg crossbeam_const_fn_trait_bound`
   Compiling crossbeam-deque v0.8.1
     Running `rustc --crate-name crossbeam_deque --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/crossbeam-deque-0.8.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="crossbeam-epoch"' --cfg 'feature="crossbeam-utils"' --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=46df68c0a92e69f5 -C extra-filename=-46df68c0a92e69f5 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern cfg_if=/work/oxidecomputer/crucible/target/debug/deps/libcfg_if-a93276574f0edf39.rmeta --extern crossbeam_epoch=/work/oxidecomputer/crucible/target/debug/deps/libcrossbeam_epoch-44fc1cc817fcdd23.rmeta --extern crossbeam_utils=/work/oxidecomputer/crucible/target/debug/deps/libcrossbeam_utils-a1fb255bfa31483a.rmeta --cap-lints allow`
   Compiling terminfo v0.8.0
     Running `rustc --crate-name build_script_build --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/terminfo-0.8.0/build.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no -C metadata=bed6d7dcce5eac66 -C extra-filename=-bed6d7dcce5eac66 --out-dir /work/oxidecomputer/crucible/target/debug/build/terminfo-bed6d7dcce5eac66 -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern phf_codegen=/work/oxidecomputer/crucible/target/debug/deps/libphf_codegen-f8063558873b33a1.rlib --cap-lints allow`
   Compiling io-lifetimes v0.5.3
     Running `rustc --crate-name io_lifetimes --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/io-lifetimes-0.5.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=59f01ff60b1f438f -C extra-filename=-59f01ff60b1f438f --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling signal-hook-mio v0.2.3
     Running `rustc --crate-name signal_hook_mio --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/signal-hook-mio-0.2.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="mio-0_8"' --cfg 'feature="support-v0_8"' -C metadata=b78bc232ff08be19 -C extra-filename=-b78bc232ff08be19 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --extern mio_0_8=/work/oxidecomputer/crucible/target/debug/deps/libmio-27a8136cf12de2bb.rmeta --extern signal_hook=/work/oxidecomputer/crucible/target/debug/deps/libsignal_hook-ae206b38743b6815.rmeta --cap-lints allow`
   Compiling structmeta-derive v0.2.0
     Running `rustc --crate-name structmeta_derive --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/structmeta-derive-0.2.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type proc-macro --emit=dep-info,link -C prefer-dynamic -C embed-bitcode=no -C metadata=6868cfc37bbab5f5 -C extra-filename=-6868cfc37bbab5f5 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rlib --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rlib --extern syn=/work/oxidecomputer/crucible/target/debug/deps/libsyn-baedf68a9175a6da.rlib --extern proc_macro --cap-lints allow`
   Compiling vte_generate_state_changes v0.1.1
     Running `rustc --crate-name vte_generate_state_changes --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/vte_generate_state_changes-0.1.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type proc-macro --emit=dep-info,link -C prefer-dynamic -C embed-bitcode=no -C metadata=52ea4d6c5be5fc21 -C extra-filename=-52ea4d6c5be5fc21 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rlib --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rlib --extern proc_macro --cap-lints allow`
   Compiling subprocess v0.2.9
     Running `rustc --crate-name subprocess --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/subprocess-0.2.9/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=0acfc5c9b903588a -C extra-filename=-0acfc5c9b903588a --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --cap-lints allow`
   Compiling dirs-sys v0.3.7
     Running `rustc --crate-name dirs_sys /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dirs-sys-0.3.7/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=15239ab4fd89ac89 -C extra-filename=-15239ab4fd89ac89 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --cap-lints allow`
   Compiling wait-timeout v0.2.0
     Running `rustc --crate-name wait_timeout /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/wait-timeout-0.2.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=509e2becba2710d2 -C extra-filename=-509e2becba2710d2 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --cap-lints allow`
   Compiling adler v1.0.2
     Running `rustc --crate-name adler /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/adler-1.0.2/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=de7aee9f30a32373 -C extra-filename=-de7aee9f30a32373 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling quick-error v1.2.3
     Running `rustc --crate-name quick_error /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/quick-error-1.2.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=c8c4727ecc8bb914 -C extra-filename=-c8c4727ecc8bb914 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling minimal-lexical v0.2.1
     Running `rustc --crate-name minimal_lexical --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/minimal-lexical-0.2.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="std"' -C metadata=fec5e2c45ccf032b -C extra-filename=-fec5e2c45ccf032b --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling regex-automata v0.1.10
     Running `rustc --crate-name regex_automata /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/regex-automata-0.1.10/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=054d4c6c4feb4142 -C extra-filename=-054d4c6c4feb4142 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling bit-vec v0.6.3
     Running `rustc --crate-name bit_vec /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/bit-vec-0.6.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="std"' -C metadata=35546b1439b435ea -C extra-filename=-35546b1439b435ea --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling rusty-fork v0.3.0
     Running `rustc --crate-name rusty_fork --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/rusty-fork-0.3.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="timeout"' --cfg 'feature="wait-timeout"' -C metadata=20bbf553eecf6baa -C extra-filename=-20bbf553eecf6baa --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern fnv=/work/oxidecomputer/crucible/target/debug/deps/libfnv-5641130f60a8056b.rmeta --extern quick_error=/work/oxidecomputer/crucible/target/debug/deps/libquick_error-c8c4727ecc8bb914.rmeta --extern tempfile=/work/oxidecomputer/crucible/target/debug/deps/libtempfile-05cb6a5f7e86cb66.rmeta --extern wait_timeout=/work/oxidecomputer/crucible/target/debug/deps/libwait_timeout-509e2becba2710d2.rmeta --cap-lints allow`
   Compiling miniz_oxide v0.6.2
     Running `rustc --crate-name miniz_oxide --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/miniz_oxide-0.6.2/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="with-alloc"' -C metadata=f7bade0e2b1a9da4 -C extra-filename=-f7bade0e2b1a9da4 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern adler=/work/oxidecomputer/crucible/target/debug/deps/libadler-de7aee9f30a32373.rmeta --cap-lints allow`
   Compiling bstr v0.2.17
     Running `rustc --crate-name bstr --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/bstr-0.2.17/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="lazy_static"' --cfg 'feature="regex-automata"' --cfg 'feature="std"' --cfg 'feature="unicode"' -C metadata=f0785d87dddebfcd -C extra-filename=-f0785d87dddebfcd --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern lazy_static=/work/oxidecomputer/crucible/target/debug/deps/liblazy_static-9c1a21fa7aad0259.rmeta --extern memchr=/work/oxidecomputer/crucible/target/debug/deps/libmemchr-9611c546f9b73cca.rmeta --extern regex_automata=/work/oxidecomputer/crucible/target/debug/deps/libregex_automata-054d4c6c4feb4142.rmeta --cap-lints allow`
   Compiling bit-set v0.5.3
     Running `rustc --crate-name bit_set /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/bit-set-0.5.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=ec12e43f0b47413c -C extra-filename=-ec12e43f0b47413c --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern bit_vec=/work/oxidecomputer/crucible/target/debug/deps/libbit_vec-35546b1439b435ea.rmeta --cap-lints allow`
   Compiling nom v7.1.3
     Running `rustc --crate-name nom --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/nom-7.1.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="alloc"' --cfg 'feature="std"' -C metadata=7d7f045245d4ae57 -C extra-filename=-7d7f045245d4ae57 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern memchr=/work/oxidecomputer/crucible/target/debug/deps/libmemchr-9611c546f9b73cca.rmeta --extern minimal_lexical=/work/oxidecomputer/crucible/target/debug/deps/libminimal_lexical-fec5e2c45ccf032b.rmeta --cap-lints allow`
     Running `/work/oxidecomputer/crucible/target/debug/build/terminfo-bed6d7dcce5eac66/build-script-build`
   Compiling vte v0.11.1
     Running `rustc --crate-name vte --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/vte-0.11.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=66f6ba7b673381f9 -C extra-filename=-66f6ba7b673381f9 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern utf8parse=/work/oxidecomputer/crucible/target/debug/deps/libutf8parse-7be01c1e08c14495.rmeta --extern vte_generate_state_changes=/work/oxidecomputer/crucible/target/debug/deps/libvte_generate_state_changes-52ea4d6c5be5fc21.so --cap-lints allow`
   Compiling dirs v4.0.0
     Running `rustc --crate-name dirs /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dirs-4.0.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=b6e400c266a3b937 -C extra-filename=-b6e400c266a3b937 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern dirs_sys=/work/oxidecomputer/crucible/target/debug/deps/libdirs_sys-15239ab4fd89ac89.rmeta --cap-lints allow`
   Compiling rustix v0.33.4
     Running `rustc --crate-name rustix --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/rustix-0.33.4/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="io-lifetimes"' --cfg 'feature="std"' -C metadata=f2ea1fb682f42c58 -C extra-filename=-f2ea1fb682f42c58 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern bitflags=/work/oxidecomputer/crucible/target/debug/deps/libbitflags-b5bc15fb96af61fc.rmeta --extern errno=/work/oxidecomputer/crucible/target/debug/deps/liberrno-af3769d2acbbbf20.rmeta --extern io_lifetimes=/work/oxidecomputer/crucible/target/debug/deps/libio_lifetimes-59f01ff60b1f438f.rmeta --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --cap-lints allow --cfg libc`
   Compiling rayon-core v1.9.2
     Running `rustc --crate-name rayon_core --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/rayon-core-1.9.2/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=000ce3790ebea6a1 -C extra-filename=-000ce3790ebea6a1 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern crossbeam_channel=/work/oxidecomputer/crucible/target/debug/deps/libcrossbeam_channel-19ff6fa486e51c63.rmeta --extern crossbeam_deque=/work/oxidecomputer/crucible/target/debug/deps/libcrossbeam_deque-46df68c0a92e69f5.rmeta --extern crossbeam_utils=/work/oxidecomputer/crucible/target/debug/deps/libcrossbeam_utils-a1fb255bfa31483a.rmeta --extern num_cpus=/work/oxidecomputer/crucible/target/debug/deps/libnum_cpus-67a451bebfcc5086.rmeta --cap-lints allow`
   Compiling structmeta v0.2.0
     Running `rustc --crate-name structmeta --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/structmeta-0.2.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C metadata=3c927247a3e97db4 -C extra-filename=-3c927247a3e97db4 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rmeta --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rmeta --extern structmeta_derive=/work/oxidecomputer/crucible/target/debug/deps/libstructmeta_derive-6868cfc37bbab5f5.so --extern syn=/work/oxidecomputer/crucible/target/debug/deps/libsyn-baedf68a9175a6da.rmeta --cap-lints allow`
   Compiling crc32fast v1.3.2
     Running `rustc --crate-name crc32fast /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/crc32fast-1.3.2/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=bbced466bd6966b3 -C extra-filename=-bbced466bd6966b3 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern cfg_if=/work/oxidecomputer/crucible/target/debug/deps/libcfg_if-a93276574f0edf39.rmeta --cap-lints allow --cfg crc32fast_stdarchx86`
   Compiling phf v0.11.1
     Running `rustc --crate-name phf --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/phf-0.11.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=26f57b9bc310a574 -C extra-filename=-26f57b9bc310a574 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern phf_shared=/work/oxidecomputer/crucible/target/debug/deps/libphf_shared-f4c611c5b95ef91c.rmeta --cap-lints allow`
   Compiling tokio-stream v0.1.8
   Compiling rand_xorshift v0.3.0
     Running `rustc --crate-name tokio_stream --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-stream-0.1.8/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="time"' -C metadata=4c3a15ae8dcea990 -C extra-filename=-4c3a15ae8dcea990 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern futures_core=/work/oxidecomputer/crucible/target/debug/deps/libfutures_core-46c6e3a1b3966417.rmeta --extern pin_project_lite=/work/oxidecomputer/crucible/target/debug/deps/libpin_project_lite-c035e320730303c2.rmeta --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rmeta --cap-lints allow`
     Running `rustc --crate-name rand_xorshift --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/rand_xorshift-0.3.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=a099defc127d0ae4 -C extra-filename=-a099defc127d0ae4 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern rand_core=/work/oxidecomputer/crucible/target/debug/deps/librand_core-e2870cc0e01c33c9.rmeta --cap-lints allow`
   Compiling xattr v0.2.2
     Running `rustc --crate-name xattr /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/xattr-0.2.2/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="unsupported"' -C metadata=70407947526a7a4b -C extra-filename=-70407947526a7a4b --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --cap-lints allow`
   Compiling filetime v0.2.17
     Running `rustc --crate-name filetime --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/filetime-0.2.17/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=337368c6d4c995d8 -C extra-filename=-337368c6d4c995d8 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern cfg_if=/work/oxidecomputer/crucible/target/debug/deps/libcfg_if-a93276574f0edf39.rmeta --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --cap-lints allow`
   Compiling csv-core v0.1.10
     Running `rustc --crate-name csv_core --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/csv-core-0.1.10/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' -C metadata=6a47c452f1df15e2 -C extra-filename=-6a47c452f1df15e2 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern memchr=/work/oxidecomputer/crucible/target/debug/deps/libmemchr-9611c546f9b73cca.rmeta --cap-lints allow`
   Compiling unarray v0.1.4
     Running `rustc --crate-name unarray --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/unarray-0.1.4/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=bb988de2629d2530 -C extra-filename=-bb988de2629d2530 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling same-file v1.0.6
     Running `rustc --crate-name same_file --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/same-file-1.0.6/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=b7a6da1044557b8b -C extra-filename=-b7a6da1044557b8b --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling proptest v1.2.0
     Running `rustc --crate-name proptest --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/proptest-1.2.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="bit-set"' --cfg 'feature="break-dead-code"' --cfg 'feature="default"' --cfg 'feature="fork"' --cfg 'feature="lazy_static"' --cfg 'feature="regex-syntax"' --cfg 'feature="rusty-fork"' --cfg 'feature="std"' --cfg 'feature="tempfile"' --cfg 'feature="timeout"' -C metadata=327f7f2cf6858f27 -C extra-filename=-327f7f2cf6858f27 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern bit_set=/work/oxidecomputer/crucible/target/debug/deps/libbit_set-ec12e43f0b47413c.rmeta --extern bitflags=/work/oxidecomputer/crucible/target/debug/deps/libbitflags-b5bc15fb96af61fc.rmeta --extern byteorder=/work/oxidecomputer/crucible/target/debug/deps/libbyteorder-1306999913c8e1b3.rmeta --extern lazy_static=/work/oxidecomputer/crucible/target/debug/deps/liblazy_static-9c1a21fa7aad0259.rmeta --extern num_traits=/work/oxidecomputer/crucible/target/debug/deps/libnum_traits-8e50de91aba3f8f9.rmeta --extern rand=/work/oxidecomputer/crucible/target/debug/deps/librand-1f91a9ea4aed49ee.rmeta --extern rand_chacha=/work/oxidecomputer/crucible/target/debug/deps/librand_chacha-add466c063ef8725.rmeta --extern rand_xorshift=/work/oxidecomputer/crucible/target/debug/deps/librand_xorshift-a099defc127d0ae4.rmeta --extern regex_syntax=/work/oxidecomputer/crucible/target/debug/deps/libregex_syntax-b9593ef3338880de.rmeta --extern rusty_fork=/work/oxidecomputer/crucible/target/debug/deps/librusty_fork-20bbf553eecf6baa.rmeta --extern tempfile=/work/oxidecomputer/crucible/target/debug/deps/libtempfile-05cb6a5f7e86cb66.rmeta --extern unarray=/work/oxidecomputer/crucible/target/debug/deps/libunarray-bb988de2629d2530.rmeta --cap-lints allow`
   Compiling walkdir v2.3.2
     Running `rustc --crate-name walkdir --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/walkdir-2.3.2/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=5232f739d2ba1b5e -C extra-filename=-5232f739d2ba1b5e --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern same_file=/work/oxidecomputer/crucible/target/debug/deps/libsame_file-b7a6da1044557b8b.rmeta --cap-lints allow`
   Compiling csv v1.2.2
     Running `rustc --crate-name csv --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/csv-1.2.2/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=187f0e890389cec3 -C extra-filename=-187f0e890389cec3 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern csv_core=/work/oxidecomputer/crucible/target/debug/deps/libcsv_core-6a47c452f1df15e2.rmeta --extern itoa=/work/oxidecomputer/crucible/target/debug/deps/libitoa-f0781104e344570e.rmeta --extern ryu=/work/oxidecomputer/crucible/target/debug/deps/libryu-062e5ac4087417b3.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --cap-lints allow`
   Compiling tokio-test v0.4.2
     Running `rustc --crate-name tokio_test --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-test-0.4.2/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=12a28be646ff63e6 -C extra-filename=-12a28be646ff63e6 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern async_stream=/work/oxidecomputer/crucible/target/debug/deps/libasync_stream-0486f21173e73f9c.rmeta --extern bytes=/work/oxidecomputer/crucible/target/debug/deps/libbytes-64a8a55ef81e55dd.rmeta --extern futures_core=/work/oxidecomputer/crucible/target/debug/deps/libfutures_core-46c6e3a1b3966417.rmeta --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rmeta --extern tokio_stream=/work/oxidecomputer/crucible/target/debug/deps/libtokio_stream-4c3a15ae8dcea990.rmeta --cap-lints allow`
     Running `rustc --crate-name terminfo --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/terminfo-0.8.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=cd93b6cd14f79089 -C extra-filename=-cd93b6cd14f79089 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern dirs=/work/oxidecomputer/crucible/target/debug/deps/libdirs-b6e400c266a3b937.rmeta --extern fnv=/work/oxidecomputer/crucible/target/debug/deps/libfnv-5641130f60a8056b.rmeta --extern nom=/work/oxidecomputer/crucible/target/debug/deps/libnom-7d7f045245d4ae57.rmeta --extern phf=/work/oxidecomputer/crucible/target/debug/deps/libphf-26f57b9bc310a574.rmeta --cap-lints allow`
   Compiling fd-lock v3.0.4
     Running `rustc --crate-name fd_lock --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/fd-lock-3.0.4/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=dd6f5c85295045f7 -C extra-filename=-dd6f5c85295045f7 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern cfg_if=/work/oxidecomputer/crucible/target/debug/deps/libcfg_if-a93276574f0edf39.rmeta --extern rustix=/work/oxidecomputer/crucible/target/debug/deps/librustix-f2ea1fb682f42c58.rmeta --cap-lints allow`
   Compiling tar v0.4.38
     Running `rustc --crate-name tar --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tar-0.4.38/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="xattr"' -C metadata=b33bc6012d78be3d -C extra-filename=-b33bc6012d78be3d --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern filetime=/work/oxidecomputer/crucible/target/debug/deps/libfiletime-337368c6d4c995d8.rmeta --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --extern xattr=/work/oxidecomputer/crucible/target/debug/deps/libxattr-70407947526a7a4b.rmeta --cap-lints allow`
   Compiling strip-ansi-escapes v0.2.0
     Running `rustc --crate-name strip_ansi_escapes /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/strip-ansi-escapes-0.2.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=8090d3de2e6bf9be -C extra-filename=-8090d3de2e6bf9be --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern vte=/work/oxidecomputer/crucible/target/debug/deps/libvte-66f6ba7b673381f9.rmeta --cap-lints allow`
   Compiling rayon v1.5.2
     Running `rustc --crate-name rayon --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/rayon-1.5.2/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=f2d40ba22c8b185e -C extra-filename=-f2d40ba22c8b185e --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern crossbeam_deque=/work/oxidecomputer/crucible/target/debug/deps/libcrossbeam_deque-46df68c0a92e69f5.rmeta --extern either=/work/oxidecomputer/crucible/target/debug/deps/libeither-e075e5427abad613.rmeta --extern rayon_core=/work/oxidecomputer/crucible/target/debug/deps/librayon_core-000ce3790ebea6a1.rmeta --cap-lints allow --cfg has_step_by_rev --cfg has_min_const_generics --cfg has_control_flow`
   Compiling test-strategy v0.3.1
     Running `rustc --crate-name test_strategy --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/test-strategy-0.3.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type proc-macro --emit=dep-info,link -C prefer-dynamic -C embed-bitcode=no -C metadata=5eb6b90d55d9f739 -C extra-filename=-5eb6b90d55d9f739 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rlib --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rlib --extern structmeta=/work/oxidecomputer/crucible/target/debug/deps/libstructmeta-3c927247a3e97db4.rlib --extern syn=/work/oxidecomputer/crucible/target/debug/deps/libsyn-baedf68a9175a6da.rlib --extern proc_macro --cap-lints allow`
   Compiling flate2 v1.0.25
     Running `rustc --crate-name flate2 --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/flate2-1.0.25/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="miniz_oxide"' --cfg 'feature="rust_backend"' -C metadata=e91a1b496d4e6ad4 -C extra-filename=-e91a1b496d4e6ad4 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern crc32fast=/work/oxidecomputer/crucible/target/debug/deps/libcrc32fast-bbced466bd6966b3.rmeta --extern miniz_oxide=/work/oxidecomputer/crucible/target/debug/deps/libminiz_oxide-f7bade0e2b1a9da4.rmeta --cap-lints allow`
   Compiling httptest v0.15.4
     Running `rustc --crate-name httptest --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/httptest-0.15.4/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=174da737d96e2af6 -C extra-filename=-174da737d96e2af6 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern bstr=/work/oxidecomputer/crucible/target/debug/deps/libbstr-f0785d87dddebfcd.rmeta --extern bytes=/work/oxidecomputer/crucible/target/debug/deps/libbytes-64a8a55ef81e55dd.rmeta --extern crossbeam_channel=/work/oxidecomputer/crucible/target/debug/deps/libcrossbeam_channel-19ff6fa486e51c63.rmeta --extern form_urlencoded=/work/oxidecomputer/crucible/target/debug/deps/libform_urlencoded-ef731295a29c9150.rmeta --extern futures=/work/oxidecomputer/crucible/target/debug/deps/libfutures-df761c89bfa71e54.rmeta --extern http=/work/oxidecomputer/crucible/target/debug/deps/libhttp-4526ab951518c907.rmeta --extern hyper=/work/oxidecomputer/crucible/target/debug/deps/libhyper-dad943d3b7cc33e9.rmeta --extern log=/work/oxidecomputer/crucible/target/debug/deps/liblog-930e200cffaa7930.rmeta --extern once_cell=/work/oxidecomputer/crucible/target/debug/deps/libonce_cell-fdb5f9e769d1e589.rmeta --extern regex=/work/oxidecomputer/crucible/target/debug/deps/libregex-f9e3a4eb3da387ce.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rmeta --extern serde_urlencoded=/work/oxidecomputer/crucible/target/debug/deps/libserde_urlencoded-779d3ac41d95557a.rmeta --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rmeta --cap-lints allow`
   Compiling progenitor-macro v0.3.0 (https://github.com/oxidecomputer/progenitor?branch=main#76716eea)
     Running `rustc --crate-name progenitor_macro --edition=2021 /home/build/.cargo/git/checkouts/progenitor-639bd64206ac5e43/76716ee/progenitor-macro/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type proc-macro --emit=dep-info,link -C prefer-dynamic -C embed-bitcode=no -C metadata=4c13a8353939c841 -C extra-filename=-4c13a8353939c841 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern openapiv3=/work/oxidecomputer/crucible/target/debug/deps/libopenapiv3-f84072cef6d0d68c.rlib --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rlib --extern progenitor_impl=/work/oxidecomputer/crucible/target/debug/deps/libprogenitor_impl-4eeb9d4c0f08c454.rlib --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rlib --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rlib --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rlib --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rlib --extern serde_tokenstream=/work/oxidecomputer/crucible/target/debug/deps/libserde_tokenstream-2a83027836bfd0fb.rlib --extern serde_yaml=/work/oxidecomputer/crucible/target/debug/deps/libserde_yaml-c7db3f53bbf8134b.rlib --extern syn=/work/oxidecomputer/crucible/target/debug/deps/libsyn-baedf68a9175a6da.rlib --extern proc_macro --cap-lints allow`
   Compiling portable-atomic v1.4.1
     Running `rustc --crate-name portable_atomic --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/portable-atomic-1.4.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="fallback"' -C metadata=f0a1a94e9d6381ba -C extra-filename=-f0a1a94e9d6381ba --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow --cfg portable_atomic_llvm_16`
   Compiling crossterm v0.26.1
     Running `rustc --crate-name crossterm --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/crossterm-0.26.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="bracketed-paste"' --cfg 'feature="default"' --cfg 'feature="serde"' -C metadata=ddbacbe0f657f0ff -C extra-filename=-ddbacbe0f657f0ff --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern bitflags=/work/oxidecomputer/crucible/target/debug/deps/libbitflags-b5bc15fb96af61fc.rmeta --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --extern mio=/work/oxidecomputer/crucible/target/debug/deps/libmio-27a8136cf12de2bb.rmeta --extern parking_lot=/work/oxidecomputer/crucible/target/debug/deps/libparking_lot-970d5c0acece447c.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern signal_hook=/work/oxidecomputer/crucible/target/debug/deps/libsignal_hook-ae206b38743b6815.rmeta --extern signal_hook_mio=/work/oxidecomputer/crucible/target/debug/deps/libsignal_hook_mio-b78bc232ff08be19.rmeta --cap-lints allow`
   Compiling signal-hook-tokio v0.3.1
     Running `rustc --crate-name signal_hook_tokio --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/signal-hook-tokio-0.3.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="futures-core-0_3"' --cfg 'feature="futures-v0_3"' -C metadata=6a6b104c61918fa0 -C extra-filename=-6a6b104c61918fa0 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern futures_core_0_3=/work/oxidecomputer/crucible/target/debug/deps/libfutures_core-46c6e3a1b3966417.rmeta --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --extern signal_hook=/work/oxidecomputer/crucible/target/debug/deps/libsignal_hook-ae206b38743b6815.rmeta --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rmeta --cap-lints allow`
   Compiling which v4.4.0
     Running `rustc --crate-name which --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/which-4.4.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=23fb4550fe083323 -C extra-filename=-23fb4550fe083323 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern either=/work/oxidecomputer/crucible/target/debug/deps/libeither-e075e5427abad613.rmeta --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --cap-lints allow`
   Compiling itertools v0.10.5
     Running `rustc --crate-name itertools --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/itertools-0.10.5/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="use_alloc"' --cfg 'feature="use_std"' -C metadata=09aeacd112427d42 -C extra-filename=-09aeacd112427d42 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern either=/work/oxidecomputer/crucible/target/debug/deps/libeither-e075e5427abad613.rmeta --cap-lints allow`
   Compiling nu-ansi-term v0.49.0
     Running `rustc --crate-name nu_ansi_term --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/nu-ansi-term-0.49.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=cdbbf2d007fd1e63 -C extra-filename=-cdbbf2d007fd1e63 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling number_prefix v0.4.0
     Running `rustc --crate-name number_prefix /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/number_prefix-0.4.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=be3728a6ec19cda2 -C extra-filename=-be3728a6ec19cda2 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling utf8-width v0.1.6
     Running `rustc --crate-name utf8_width --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/utf8-width-0.1.6/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=7e583d5482ac364b -C extra-filename=-7e583d5482ac364b --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling indicatif v0.17.6
     Running `rustc --crate-name indicatif --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/indicatif-0.17.6/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="rayon"' --cfg 'feature="unicode-width"' -C metadata=297a26a70875006e -C extra-filename=-297a26a70875006e --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern console=/work/oxidecomputer/crucible/target/debug/deps/libconsole-4236472a6e29ce0a.rmeta --extern number_prefix=/work/oxidecomputer/crucible/target/debug/deps/libnumber_prefix-be3728a6ec19cda2.rmeta --extern portable_atomic=/work/oxidecomputer/crucible/target/debug/deps/libportable_atomic-f0a1a94e9d6381ba.rmeta --extern rayon=/work/oxidecomputer/crucible/target/debug/deps/librayon-f2d40ba22c8b185e.rmeta --extern unicode_width=/work/oxidecomputer/crucible/target/debug/deps/libunicode_width-87c5262a4c4bb0e9.rmeta --cap-lints allow`
   Compiling byte-unit v4.0.19
     Running `rustc --crate-name byte_unit --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/byte-unit-4.0.19/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="alloc"' --cfg 'feature="default"' --cfg 'feature="serde"' --cfg 'feature="std"' --cfg 'feature="u128"' -C metadata=02cb17c857e20dac -C extra-filename=-02cb17c857e20dac --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern utf8_width=/work/oxidecomputer/crucible/target/debug/deps/libutf8_width-7e583d5482ac364b.rmeta --cap-lints allow`
   Compiling crossterm v0.27.0
     Running `rustc --crate-name crossterm --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/crossterm-0.27.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="bracketed-paste"' --cfg 'feature="default"' --cfg 'feature="events"' --cfg 'feature="windows"' -C metadata=3c787fd4c4d4bc45 -C extra-filename=-3c787fd4c4d4bc45 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern bitflags=/work/oxidecomputer/crucible/target/debug/deps/libbitflags-109244799287a8c3.rmeta --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --extern mio=/work/oxidecomputer/crucible/target/debug/deps/libmio-27a8136cf12de2bb.rmeta --extern parking_lot=/work/oxidecomputer/crucible/target/debug/deps/libparking_lot-970d5c0acece447c.rmeta --extern signal_hook=/work/oxidecomputer/crucible/target/debug/deps/libsignal_hook-ae206b38743b6815.rmeta --extern signal_hook_mio=/work/oxidecomputer/crucible/target/debug/deps/libsignal_hook_mio-b78bc232ff08be19.rmeta --cap-lints allow`
   Compiling nbd v0.2.3
     Running `rustc --crate-name nbd /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/nbd-0.2.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=6a0c01a24def5e9a -C extra-filename=-6a0c01a24def5e9a --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern byteorder=/work/oxidecomputer/crucible/target/debug/deps/libbyteorder-1306999913c8e1b3.rmeta --cap-lints allow`
   Compiling crucible-integration-tests v0.1.0 (/work/oxidecomputer/crucible/integration_tests)
     Running `rustc --crate-name crucible_integration_tests --edition=2021 integration_tests/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=521d4724b4b30c4a -C extra-filename=-521d4724b4b30c4a --out-dir /work/oxidecomputer/crucible/target/debug/deps -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps`
     Running `rustc --crate-name info --edition=2021 smf/examples/info.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no -C debuginfo=2 -C metadata=434b2d59a81079b5 -C extra-filename=-434b2d59a81079b5 --out-dir /work/oxidecomputer/crucible/target/debug/examples -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern crucible_smf=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_smf-e1a82c6f17385dc6.rlib --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rlib --extern num_derive=/work/oxidecomputer/crucible/target/debug/deps/libnum_derive-814c8a0a0a713cba.so --extern num_traits=/work/oxidecomputer/crucible/target/debug/deps/libnum_traits-8e50de91aba3f8f9.rlib --extern thiserror=/work/oxidecomputer/crucible/target/debug/deps/libthiserror-07eca56f531a0e5d.rlib`
   Compiling clearscreen v2.0.1
     Running `rustc --crate-name clearscreen --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/clearscreen-2.0.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=5e923be7ef236a41 -C extra-filename=-5e923be7ef236a41 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern nix=/work/oxidecomputer/crucible/target/debug/deps/libnix-3635aff9412bf811.rmeta --extern terminfo=/work/oxidecomputer/crucible/target/debug/deps/libterminfo-cd93b6cd14f79089.rmeta --extern thiserror=/work/oxidecomputer/crucible/target/debug/deps/libthiserror-07eca56f531a0e5d.rmeta --extern which=/work/oxidecomputer/crucible/target/debug/deps/libwhich-23fb4550fe083323.rmeta --cap-lints allow`
   Compiling reedline v0.23.0
     Running `rustc --crate-name reedline --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/reedline-0.23.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=6e6244e0f6aa654d -C extra-filename=-6e6244e0f6aa654d --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern chrono=/work/oxidecomputer/crucible/target/debug/deps/libchrono-0402a5e17dd99cdf.rmeta --extern crossterm=/work/oxidecomputer/crucible/target/debug/deps/libcrossterm-ddbacbe0f657f0ff.rmeta --extern fd_lock=/work/oxidecomputer/crucible/target/debug/deps/libfd_lock-dd6f5c85295045f7.rmeta --extern itertools=/work/oxidecomputer/crucible/target/debug/deps/libitertools-09aeacd112427d42.rmeta --extern nu_ansi_term=/work/oxidecomputer/crucible/target/debug/deps/libnu_ansi_term-cdbbf2d007fd1e63.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern strip_ansi_escapes=/work/oxidecomputer/crucible/target/debug/deps/libstrip_ansi_escapes-8090d3de2e6bf9be.rmeta --extern strum=/work/oxidecomputer/crucible/target/debug/deps/libstrum-59ea3c6704348e58.rmeta --extern strum_macros=/work/oxidecomputer/crucible/target/debug/deps/libstrum_macros-bc907f623478289d.so --extern thiserror=/work/oxidecomputer/crucible/target/debug/deps/libthiserror-07eca56f531a0e5d.rmeta --extern unicode_segmentation=/work/oxidecomputer/crucible/target/debug/deps/libunicode_segmentation-06176721b7b95955.rmeta --extern unicode_width=/work/oxidecomputer/crucible/target/debug/deps/libunicode_width-87c5262a4c4bb0e9.rmeta --cap-lints allow`
     Running `rustc --crate-name crucible_smf --edition=2021 smf/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --emit=dep-info,link -C embed-bitcode=no -C debuginfo=2 --test -C metadata=6114df38a9482a0c -C extra-filename=-6114df38a9482a0c --out-dir /work/oxidecomputer/crucible/target/debug/deps -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rlib --extern num_derive=/work/oxidecomputer/crucible/target/debug/deps/libnum_derive-814c8a0a0a713cba.so --extern num_traits=/work/oxidecomputer/crucible/target/debug/deps/libnum_traits-8e50de91aba3f8f9.rlib --extern thiserror=/work/oxidecomputer/crucible/target/debug/deps/libthiserror-07eca56f531a0e5d.rlib`
     Running `rustc --crate-name crucible_client_types --edition=2021 crucible-client-types/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --emit=dep-info,link -C embed-bitcode=no -C debuginfo=2 --test -C metadata=5ba9f9d411803900 -C extra-filename=-5ba9f9d411803900 --out-dir /work/oxidecomputer/crucible/target/debug/deps -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern base64=/work/oxidecomputer/crucible/target/debug/deps/libbase64-c139bdd129e780ac.rlib --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rlib --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rlib --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rlib --extern uuid=/work/oxidecomputer/crucible/target/debug/deps/libuuid-7cc8e87b4149b49e.rlib`
     Running `rustc --crate-name ring --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/ring-0.16.20/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="alloc"' --cfg 'feature="default"' --cfg 'feature="dev_urandom_fallback"' --cfg 'feature="once_cell"' -C metadata=76ccf829b8b489e1 -C extra-filename=-76ccf829b8b489e1 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern once_cell=/work/oxidecomputer/crucible/target/debug/deps/libonce_cell-fdb5f9e769d1e589.rmeta --extern spin=/work/oxidecomputer/crucible/target/debug/deps/libspin-bfb6115ad3135235.rmeta --extern untrusted=/work/oxidecomputer/crucible/target/debug/deps/libuntrusted-4b93784238d33e58.rmeta --cap-lints allow -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out -l static=ring-core -l static=ring-test`
   Compiling rustls v0.21.6
     Running `/work/oxidecomputer/crucible/target/debug/build/rustls-0c105edc866f624d/build-script-build`
   Compiling rustls-webpki v0.101.4
     Running `rustc --crate-name webpki --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/rustls-webpki-0.101.4/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="alloc"' --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=34d764dbf1af1e62 -C extra-filename=-34d764dbf1af1e62 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern ring=/work/oxidecomputer/crucible/target/debug/deps/libring-76ccf829b8b489e1.rmeta --extern untrusted=/work/oxidecomputer/crucible/target/debug/deps/libuntrusted-4b93784238d33e58.rmeta --cap-lints allow -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out`
   Compiling sct v0.7.0
     Running `rustc --crate-name sct --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/sct-0.7.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=78bb43c10db32a31 -C extra-filename=-78bb43c10db32a31 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern ring=/work/oxidecomputer/crucible/target/debug/deps/libring-76ccf829b8b489e1.rmeta --extern untrusted=/work/oxidecomputer/crucible/target/debug/deps/libuntrusted-4b93784238d33e58.rmeta --cap-lints allow -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out`
     Running `rustc --crate-name rustls --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/rustls-0.21.6/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="dangerous_configuration"' --cfg 'feature="default"' --cfg 'feature="log"' --cfg 'feature="logging"' --cfg 'feature="tls12"' -C metadata=3df6867cfa5c4a0a -C extra-filename=-3df6867cfa5c4a0a --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern log=/work/oxidecomputer/crucible/target/debug/deps/liblog-930e200cffaa7930.rmeta --extern ring=/work/oxidecomputer/crucible/target/debug/deps/libring-76ccf829b8b489e1.rmeta --extern webpki=/work/oxidecomputer/crucible/target/debug/deps/libwebpki-34d764dbf1af1e62.rmeta --extern sct=/work/oxidecomputer/crucible/target/debug/deps/libsct-78bb43c10db32a31.rmeta --cap-lints allow -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out`
   Compiling tokio-rustls v0.24.1
     Running `rustc --crate-name tokio_rustls --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-rustls-0.24.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="logging"' --cfg 'feature="tls12"' -C metadata=eafe4ab74a176b7d -C extra-filename=-eafe4ab74a176b7d --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern rustls=/work/oxidecomputer/crucible/target/debug/deps/librustls-3df6867cfa5c4a0a.rmeta --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rmeta --cap-lints allow -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out`
   Compiling hyper-rustls v0.24.0
   Compiling dropshot v0.9.1-dev (https://github.com/oxidecomputer/dropshot?branch=main#aca6de3c)
     Running `rustc --crate-name hyper_rustls --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/hyper-rustls-0.24.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=9a83fdcd4675665e -C extra-filename=-9a83fdcd4675665e --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern http=/work/oxidecomputer/crucible/target/debug/deps/libhttp-4526ab951518c907.rmeta --extern hyper=/work/oxidecomputer/crucible/target/debug/deps/libhyper-dad943d3b7cc33e9.rmeta --extern rustls=/work/oxidecomputer/crucible/target/debug/deps/librustls-3df6867cfa5c4a0a.rmeta --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rmeta --extern tokio_rustls=/work/oxidecomputer/crucible/target/debug/deps/libtokio_rustls-eafe4ab74a176b7d.rmeta --cap-lints allow -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out`
     Running `rustc --crate-name dropshot --edition=2018 /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/aca6de3/dropshot/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="usdt"' --cfg 'feature="usdt-probes"' -C metadata=a49a4505c9c6b86f -C extra-filename=-a49a4505c9c6b86f --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern async_stream=/work/oxidecomputer/crucible/target/debug/deps/libasync_stream-0486f21173e73f9c.rmeta --extern async_trait=/work/oxidecomputer/crucible/target/debug/deps/libasync_trait-a300e84178ee0ad1.so --extern base64=/work/oxidecomputer/crucible/target/debug/deps/libbase64-c139bdd129e780ac.rmeta --extern bytes=/work/oxidecomputer/crucible/target/debug/deps/libbytes-64a8a55ef81e55dd.rmeta --extern camino=/work/oxidecomputer/crucible/target/debug/deps/libcamino-45f0f4a2c258f934.rmeta --extern chrono=/work/oxidecomputer/crucible/target/debug/deps/libchrono-0402a5e17dd99cdf.rmeta --extern debug_ignore=/work/oxidecomputer/crucible/target/debug/deps/libdebug_ignore-2303f500fcbc7093.rmeta --extern dropshot_endpoint=/work/oxidecomputer/crucible/target/debug/deps/libdropshot_endpoint-1ff3a3dd0352c250.so --extern form_urlencoded=/work/oxidecomputer/crucible/target/debug/deps/libform_urlencoded-ef731295a29c9150.rmeta --extern futures=/work/oxidecomputer/crucible/target/debug/deps/libfutures-df761c89bfa71e54.rmeta --extern hostname=/work/oxidecomputer/crucible/target/debug/deps/libhostname-4a0f8b1a56e5681a.rmeta --extern http=/work/oxidecomputer/crucible/target/debug/deps/libhttp-4526ab951518c907.rmeta --extern hyper=/work/oxidecomputer/crucible/target/debug/deps/libhyper-dad943d3b7cc33e9.rmeta --extern indexmap=/work/oxidecomputer/crucible/target/debug/deps/libindexmap-c2c8f74266eebb64.rmeta --extern openapiv3=/work/oxidecomputer/crucible/target/debug/deps/libopenapiv3-f84072cef6d0d68c.rmeta --extern paste=/work/oxidecomputer/crucible/target/debug/deps/libpaste-251489637fc3d2bc.so --extern percent_encoding=/work/oxidecomputer/crucible/target/debug/deps/libpercent_encoding-0000aebce3d30803.rmeta --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rmeta --extern rustls=/work/oxidecomputer/crucible/target/debug/deps/librustls-3df6867cfa5c4a0a.rmeta --extern rustls_pemfile=/work/oxidecomputer/crucible/target/debug/deps/librustls_pemfile-e52b2a6debfcae48.rmeta --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rmeta --extern serde_path_to_error=/work/oxidecomputer/crucible/target/debug/deps/libserde_path_to_error-f8fffad8b554a310.rmeta --extern serde_urlencoded=/work/oxidecomputer/crucible/target/debug/deps/libserde_urlencoded-779d3ac41d95557a.rmeta --extern sha1=/work/oxidecomputer/crucible/target/debug/deps/libsha1-8fdbd7a715f3bef1.rmeta --extern slog=/work/oxidecomputer/crucible/target/debug/deps/libslog-84fd25666c3c26ee.rmeta --extern slog_async=/work/oxidecomputer/crucible/target/debug/deps/libslog_async-03e743f036119074.rmeta --extern slog_bunyan=/work/oxidecomputer/crucible/target/debug/deps/libslog_bunyan-dce051a6775f1d99.rmeta --extern slog_json=/work/oxidecomputer/crucible/target/debug/deps/libslog_json-d8408f8f3a6dd5b7.rmeta --extern slog_term=/work/oxidecomputer/crucible/target/debug/deps/libslog_term-31c687431f6dd53c.rmeta --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rmeta --extern tokio_rustls=/work/oxidecomputer/crucible/target/debug/deps/libtokio_rustls-eafe4ab74a176b7d.rmeta --extern toml=/work/oxidecomputer/crucible/target/debug/deps/libtoml-de0eb3fcc3b95b5c.rmeta --extern usdt=/work/oxidecomputer/crucible/target/debug/deps/libusdt-86bb76e3b8fcea87.rmeta --extern uuid=/work/oxidecomputer/crucible/target/debug/deps/libuuid-7cc8e87b4149b49e.rmeta --extern waitgroup=/work/oxidecomputer/crucible/target/debug/deps/libwaitgroup-db859ead02bd709e.rmeta --cap-lints allow -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out`
     Running `rustc --crate-name libgit2_sys --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/libgit2-sys-0.15.2+1.6.4/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C metadata=8cfb4e998561bba5 -C extra-filename=-8cfb4e998561bba5 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --extern libz_sys=/work/oxidecomputer/crucible/target/debug/deps/liblibz_sys-a3111f279c2174e3.rmeta --cap-lints allow -L native=/work/oxidecomputer/crucible/target/debug/build/libgit2-sys-6120c1544d9c7fd8/out/build -l static=git2 -L native=/usr/lib/amd64 --cfg libgit2_vendored`
   Compiling reqwest v0.11.20
     Running `rustc --crate-name reqwest --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/reqwest-0.11.20/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="__rustls"' --cfg 'feature="__tls"' --cfg 'feature="blocking"' --cfg 'feature="default"' --cfg 'feature="default-tls"' --cfg 'feature="hyper-rustls"' --cfg 'feature="hyper-tls"' --cfg 'feature="json"' --cfg 'feature="native-tls-crate"' --cfg 'feature="rustls"' --cfg 'feature="rustls-pemfile"' --cfg 'feature="rustls-tls"' --cfg 'feature="rustls-tls-webpki-roots"' --cfg 'feature="serde_json"' --cfg 'feature="stream"' --cfg 'feature="tokio-native-tls"' --cfg 'feature="tokio-rustls"' --cfg 'feature="tokio-util"' --cfg 'feature="wasm-streams"' --cfg 'feature="webpki-roots"' -C metadata=6407fc4e9374ca8e -C extra-filename=-6407fc4e9374ca8e --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern base64=/work/oxidecomputer/crucible/target/debug/deps/libbase64-c139bdd129e780ac.rmeta --extern bytes=/work/oxidecomputer/crucible/target/debug/deps/libbytes-64a8a55ef81e55dd.rmeta --extern encoding_rs=/work/oxidecomputer/crucible/target/debug/deps/libencoding_rs-3255048793b3f7a6.rmeta --extern futures_core=/work/oxidecomputer/crucible/target/debug/deps/libfutures_core-46c6e3a1b3966417.rmeta --extern futures_util=/work/oxidecomputer/crucible/target/debug/deps/libfutures_util-b4da5d5433271d56.rmeta --extern h2=/work/oxidecomputer/crucible/target/debug/deps/libh2-3e2d8390f23dd48a.rmeta --extern http=/work/oxidecomputer/crucible/target/debug/deps/libhttp-4526ab951518c907.rmeta --extern http_body=/work/oxidecomputer/crucible/target/debug/deps/libhttp_body-bb1d69dd918c127f.rmeta --extern hyper=/work/oxidecomputer/crucible/target/debug/deps/libhyper-dad943d3b7cc33e9.rmeta --extern hyper_rustls=/work/oxidecomputer/crucible/target/debug/deps/libhyper_rustls-9a83fdcd4675665e.rmeta --extern hyper_tls=/work/oxidecomputer/crucible/target/debug/deps/libhyper_tls-2dbf57f91f681e2c.rmeta --extern ipnet=/work/oxidecomputer/crucible/target/debug/deps/libipnet-8b250db103a32779.rmeta --extern log=/work/oxidecomputer/crucible/target/debug/deps/liblog-930e200cffaa7930.rmeta --extern mime=/work/oxidecomputer/crucible/target/debug/deps/libmime-ac14a9115eddd3c2.rmeta --extern native_tls_crate=/work/oxidecomputer/crucible/target/debug/deps/libnative_tls-320c05ab5bbd33c9.rmeta --extern once_cell=/work/oxidecomputer/crucible/target/debug/deps/libonce_cell-fdb5f9e769d1e589.rmeta --extern percent_encoding=/work/oxidecomputer/crucible/target/debug/deps/libpercent_encoding-0000aebce3d30803.rmeta --extern pin_project_lite=/work/oxidecomputer/crucible/target/debug/deps/libpin_project_lite-c035e320730303c2.rmeta --extern rustls=/work/oxidecomputer/crucible/target/debug/deps/librustls-3df6867cfa5c4a0a.rmeta --extern rustls_pemfile=/work/oxidecomputer/crucible/target/debug/deps/librustls_pemfile-e52b2a6debfcae48.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rmeta --extern serde_urlencoded=/work/oxidecomputer/crucible/target/debug/deps/libserde_urlencoded-779d3ac41d95557a.rmeta --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rmeta --extern tokio_native_tls=/work/oxidecomputer/crucible/target/debug/deps/libtokio_native_tls-f56aba82a642e205.rmeta --extern tokio_rustls=/work/oxidecomputer/crucible/target/debug/deps/libtokio_rustls-eafe4ab74a176b7d.rmeta --extern tokio_util=/work/oxidecomputer/crucible/target/debug/deps/libtokio_util-279b3765a2b5aad1.rmeta --extern tower_service=/work/oxidecomputer/crucible/target/debug/deps/libtower_service-51da71f2ad5117ee.rmeta --extern url=/work/oxidecomputer/crucible/target/debug/deps/liburl-ff56943ab9066fdc.rmeta --extern webpki_roots=/work/oxidecomputer/crucible/target/debug/deps/libwebpki_roots-31272bd9a7615638.rmeta --cap-lints allow -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out -L native=/usr/ssl-3/lib/amd64`
   Compiling git2 v0.17.2
     Running `rustc --crate-name git2 --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/git2-0.17.2/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C metadata=c3276e37b1dd24e5 -C extra-filename=-c3276e37b1dd24e5 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern bitflags=/work/oxidecomputer/crucible/target/debug/deps/libbitflags-b5bc15fb96af61fc.rmeta --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --extern libgit2_sys=/work/oxidecomputer/crucible/target/debug/deps/liblibgit2_sys-8cfb4e998561bba5.rmeta --extern log=/work/oxidecomputer/crucible/target/debug/deps/liblog-930e200cffaa7930.rmeta --extern url=/work/oxidecomputer/crucible/target/debug/deps/liburl-ff56943ab9066fdc.rmeta --cap-lints allow -L native=/work/oxidecomputer/crucible/target/debug/build/libgit2-sys-6120c1544d9c7fd8/out/build -L native=/usr/lib/amd64`
   Compiling progenitor-client v0.3.0 (https://github.com/oxidecomputer/progenitor?branch=main#76716eea)
   Compiling omicron-zone-package v0.9.1
     Running `rustc --crate-name progenitor_client --edition=2021 /home/build/.cargo/git/checkouts/progenitor-639bd64206ac5e43/76716ee/progenitor-client/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=a49921826ff8ec2a -C extra-filename=-a49921826ff8ec2a --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern bytes=/work/oxidecomputer/crucible/target/debug/deps/libbytes-64a8a55ef81e55dd.rmeta --extern futures_core=/work/oxidecomputer/crucible/target/debug/deps/libfutures_core-46c6e3a1b3966417.rmeta --extern percent_encoding=/work/oxidecomputer/crucible/target/debug/deps/libpercent_encoding-0000aebce3d30803.rmeta --extern reqwest=/work/oxidecomputer/crucible/target/debug/deps/libreqwest-6407fc4e9374ca8e.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rmeta --extern serde_urlencoded=/work/oxidecomputer/crucible/target/debug/deps/libserde_urlencoded-779d3ac41d95557a.rmeta --cap-lints allow -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out -L native=/usr/ssl-3/lib/amd64`
     Running `rustc --crate-name omicron_zone_package --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/omicron-zone-package-0.9.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=8423a7bf8bd88040 -C extra-filename=-8423a7bf8bd88040 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rmeta --extern async_trait=/work/oxidecomputer/crucible/target/debug/deps/libasync_trait-a300e84178ee0ad1.so --extern chrono=/work/oxidecomputer/crucible/target/debug/deps/libchrono-0402a5e17dd99cdf.rmeta --extern filetime=/work/oxidecomputer/crucible/target/debug/deps/libfiletime-337368c6d4c995d8.rmeta --extern flate2=/work/oxidecomputer/crucible/target/debug/deps/libflate2-e91a1b496d4e6ad4.rmeta --extern futures_util=/work/oxidecomputer/crucible/target/debug/deps/libfutures_util-b4da5d5433271d56.rmeta --extern hex=/work/oxidecomputer/crucible/target/debug/deps/libhex-6531b11cb72de3e5.rmeta --extern reqwest=/work/oxidecomputer/crucible/target/debug/deps/libreqwest-6407fc4e9374ca8e.rmeta --extern ring=/work/oxidecomputer/crucible/target/debug/deps/libring-76ccf829b8b489e1.rmeta --extern semver=/work/oxidecomputer/crucible/target/debug/deps/libsemver-8c1c5827befd93e7.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern serde_derive=/work/oxidecomputer/crucible/target/debug/deps/libserde_derive-3441a1f9756a6d5b.so --extern tar=/work/oxidecomputer/crucible/target/debug/deps/libtar-b33bc6012d78be3d.rmeta --extern tempfile=/work/oxidecomputer/crucible/target/debug/deps/libtempfile-05cb6a5f7e86cb66.rmeta --extern thiserror=/work/oxidecomputer/crucible/target/debug/deps/libthiserror-07eca56f531a0e5d.rmeta --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rmeta --extern toml=/work/oxidecomputer/crucible/target/debug/deps/libtoml-de0eb3fcc3b95b5c.rmeta --extern walkdir=/work/oxidecomputer/crucible/target/debug/deps/libwalkdir-5232f739d2ba1b5e.rmeta --cap-lints allow -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out -L native=/usr/ssl-3/lib/amd64`
   Compiling progenitor v0.3.0 (https://github.com/oxidecomputer/progenitor?branch=main#76716eea)
     Running `rustc --crate-name progenitor --edition=2021 /home/build/.cargo/git/checkouts/progenitor-639bd64206ac5e43/76716ee/progenitor/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=0319e0dfd841f493 -C extra-filename=-0319e0dfd841f493 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern progenitor_client=/work/oxidecomputer/crucible/target/debug/deps/libprogenitor_client-a49921826ff8ec2a.rmeta --extern progenitor_impl=/work/oxidecomputer/crucible/target/debug/deps/libprogenitor_impl-4eeb9d4c0f08c454.rmeta --extern progenitor_macro=/work/oxidecomputer/crucible/target/debug/deps/libprogenitor_macro-4c13a8353939c841.so --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rmeta --cap-lints allow -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out -L native=/usr/ssl-3/lib/amd64`
   Compiling repair-client v0.0.1 (/work/oxidecomputer/crucible/repair-client)
   Compiling dsc-client v0.0.1 (/work/oxidecomputer/crucible/dsc-client)
     Running `rustc --crate-name repair_client --edition=2021 repair-client/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=1452d56087b6ccb7 -C extra-filename=-1452d56087b6ccb7 --out-dir /work/oxidecomputer/crucible/target/debug/deps -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rmeta --extern chrono=/work/oxidecomputer/crucible/target/debug/deps/libchrono-0402a5e17dd99cdf.rmeta --extern percent_encoding=/work/oxidecomputer/crucible/target/debug/deps/libpercent_encoding-0000aebce3d30803.rmeta --extern progenitor=/work/oxidecomputer/crucible/target/debug/deps/libprogenitor-0319e0dfd841f493.rmeta --extern reqwest=/work/oxidecomputer/crucible/target/debug/deps/libreqwest-6407fc4e9374ca8e.rmeta --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rmeta -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out -L native=/usr/ssl-3/lib/amd64`
     Running `rustc --crate-name dsc_client --edition=2021 dsc-client/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=15b0c81fa833cf0f -C extra-filename=-15b0c81fa833cf0f --out-dir /work/oxidecomputer/crucible/target/debug/deps -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rmeta --extern percent_encoding=/work/oxidecomputer/crucible/target/debug/deps/libpercent_encoding-0000aebce3d30803.rmeta --extern progenitor=/work/oxidecomputer/crucible/target/debug/deps/libprogenitor-0319e0dfd841f493.rmeta --extern reqwest=/work/oxidecomputer/crucible/target/debug/deps/libreqwest-6407fc4e9374ca8e.rmeta --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rmeta -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out -L native=/usr/ssl-3/lib/amd64`
   Compiling crucible-pantry-client v0.0.1 (/work/oxidecomputer/crucible/pantry-client)
     Running `rustc --crate-name crucible_pantry_client --edition=2021 pantry-client/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=ccb9ddeebb23cea2 -C extra-filename=-ccb9ddeebb23cea2 --out-dir /work/oxidecomputer/crucible/target/debug/deps -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rmeta --extern chrono=/work/oxidecomputer/crucible/target/debug/deps/libchrono-0402a5e17dd99cdf.rmeta --extern percent_encoding=/work/oxidecomputer/crucible/target/debug/deps/libpercent_encoding-0000aebce3d30803.rmeta --extern progenitor=/work/oxidecomputer/crucible/target/debug/deps/libprogenitor-0319e0dfd841f493.rmeta --extern reqwest=/work/oxidecomputer/crucible/target/debug/deps/libreqwest-6407fc4e9374ca8e.rmeta --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rmeta --extern uuid=/work/oxidecomputer/crucible/target/debug/deps/libuuid-7cc8e87b4149b49e.rmeta -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out -L native=/usr/ssl-3/lib/amd64`
   Compiling omicron-common v0.1.0 (https://github.com/oxidecomputer/omicron?branch=main#4c05962d)
     Running `rustc --crate-name omicron_common --edition=2021 /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/4c05962/common/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=52df1fff8b36d94c -C extra-filename=-52df1fff8b36d94c --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rmeta --extern api_identity=/work/oxidecomputer/crucible/target/debug/deps/libapi_identity-90d45ecc06c8f773.so --extern async_trait=/work/oxidecomputer/crucible/target/debug/deps/libasync_trait-a300e84178ee0ad1.so --extern backoff=/work/oxidecomputer/crucible/target/debug/deps/libbackoff-2bc4a2fd075cf434.rmeta --extern camino=/work/oxidecomputer/crucible/target/debug/deps/libcamino-45f0f4a2c258f934.rmeta --extern camino_tempfile=/work/oxidecomputer/crucible/target/debug/deps/libcamino_tempfile-38b6a8d85c9dc0c0.rmeta --extern chrono=/work/oxidecomputer/crucible/target/debug/deps/libchrono-0402a5e17dd99cdf.rmeta --extern dropshot=/work/oxidecomputer/crucible/target/debug/deps/libdropshot-a49a4505c9c6b86f.rmeta --extern futures=/work/oxidecomputer/crucible/target/debug/deps/libfutures-df761c89bfa71e54.rmeta --extern hex=/work/oxidecomputer/crucible/target/debug/deps/libhex-6531b11cb72de3e5.rmeta --extern http=/work/oxidecomputer/crucible/target/debug/deps/libhttp-4526ab951518c907.rmeta --extern hyper=/work/oxidecomputer/crucible/target/debug/deps/libhyper-dad943d3b7cc33e9.rmeta --extern ipnetwork=/work/oxidecomputer/crucible/target/debug/deps/libipnetwork-0e9e550a49db2c52.rmeta --extern lazy_static=/work/oxidecomputer/crucible/target/debug/deps/liblazy_static-9c1a21fa7aad0259.rmeta --extern macaddr=/work/oxidecomputer/crucible/target/debug/deps/libmacaddr-98e89df75c36be48.rmeta --extern parse_display=/work/oxidecomputer/crucible/target/debug/deps/libparse_display-34a1a5d52375b70b.rmeta --extern progenitor=/work/oxidecomputer/crucible/target/debug/deps/libprogenitor-0319e0dfd841f493.rmeta --extern rand=/work/oxidecomputer/crucible/target/debug/deps/librand-1f91a9ea4aed49ee.rmeta --extern reqwest=/work/oxidecomputer/crucible/target/debug/deps/libreqwest-6407fc4e9374ca8e.rmeta --extern ring=/work/oxidecomputer/crucible/target/debug/deps/libring-76ccf829b8b489e1.rmeta --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rmeta --extern semver=/work/oxidecomputer/crucible/target/debug/deps/libsemver-8c1c5827befd93e7.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern serde_derive=/work/oxidecomputer/crucible/target/debug/deps/libserde_derive-3441a1f9756a6d5b.so --extern serde_human_bytes=/work/oxidecomputer/crucible/target/debug/deps/libserde_human_bytes-7f54d0fcbf9b36f3.rmeta --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rmeta --extern serde_with=/work/oxidecomputer/crucible/target/debug/deps/libserde_with-4f9ddd30b380d6cf.rmeta --extern slog=/work/oxidecomputer/crucible/target/debug/deps/libslog-84fd25666c3c26ee.rmeta --extern steno=/work/oxidecomputer/crucible/target/debug/deps/libsteno-d1d3ce30296926ad.rmeta --extern strum=/work/oxidecomputer/crucible/target/debug/deps/libstrum-59ea3c6704348e58.rmeta --extern thiserror=/work/oxidecomputer/crucible/target/debug/deps/libthiserror-07eca56f531a0e5d.rmeta --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rmeta --extern tokio_postgres=/work/oxidecomputer/crucible/target/debug/deps/libtokio_postgres-5628b93feb58339b.rmeta --extern toml=/work/oxidecomputer/crucible/target/debug/deps/libtoml-de0eb3fcc3b95b5c.rmeta --extern uuid=/work/oxidecomputer/crucible/target/debug/deps/libuuid-7cc8e87b4149b49e.rmeta --cap-lints allow -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out -L native=/usr/ssl-3/lib/amd64`
   Compiling crucible-control-client v0.0.1 (/work/oxidecomputer/crucible/control-client)
     Running `rustc --crate-name crucible_control_client --edition=2021 control-client/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=3d0142c7d3790e17 -C extra-filename=-3d0142c7d3790e17 --out-dir /work/oxidecomputer/crucible/target/debug/deps -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rmeta --extern percent_encoding=/work/oxidecomputer/crucible/target/debug/deps/libpercent_encoding-0000aebce3d30803.rmeta --extern progenitor=/work/oxidecomputer/crucible/target/debug/deps/libprogenitor-0319e0dfd841f493.rmeta --extern reqwest=/work/oxidecomputer/crucible/target/debug/deps/libreqwest-6407fc4e9374ca8e.rmeta --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rmeta -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out -L native=/usr/ssl-3/lib/amd64`
   Compiling vergen v8.2.4
     Running `rustc --crate-name vergen --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/vergen-8.2.4/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no --cfg 'feature="cargo"' --cfg 'feature="default"' --cfg 'feature="git"' --cfg 'feature="git2"' --cfg 'feature="git2-rs"' --cfg 'feature="rustc"' --cfg 'feature="rustc_version"' --cfg 'feature="time"' -C metadata=e75c33287bd2547c -C extra-filename=-e75c33287bd2547c --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rmeta --extern git2_rs=/work/oxidecomputer/crucible/target/debug/deps/libgit2-c3276e37b1dd24e5.rmeta --extern rustc_version=/work/oxidecomputer/crucible/target/debug/deps/librustc_version-201ef6100eba532b.rmeta --extern time=/work/oxidecomputer/crucible/target/debug/deps/libtime-9b604407a0d52f86.rmeta --cap-lints allow -L native=/work/oxidecomputer/crucible/target/debug/build/libgit2-sys-6120c1544d9c7fd8/out/build -L native=/usr/lib/amd64 --cfg stable --cfg msrv`
   Compiling crucible-agent-client v0.0.1 (/work/oxidecomputer/crucible/agent-client)
     Running `rustc --crate-name crucible_agent_client --edition=2021 agent-client/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=86e1c18945d61be3 -C extra-filename=-86e1c18945d61be3 --out-dir /work/oxidecomputer/crucible/target/debug/deps -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rmeta --extern chrono=/work/oxidecomputer/crucible/target/debug/deps/libchrono-0402a5e17dd99cdf.rmeta --extern percent_encoding=/work/oxidecomputer/crucible/target/debug/deps/libpercent_encoding-0000aebce3d30803.rmeta --extern progenitor=/work/oxidecomputer/crucible/target/debug/deps/libprogenitor-0319e0dfd841f493.rmeta --extern reqwest=/work/oxidecomputer/crucible/target/debug/deps/libreqwest-6407fc4e9374ca8e.rmeta --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rmeta -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out -L native=/usr/ssl-3/lib/amd64`
   Compiling crucible-common v0.0.1 (/work/oxidecomputer/crucible/common)
     Running `rustc --crate-name build_script_build --edition=2021 common/build.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no -C metadata=e7c30bee09b7ef3b -C extra-filename=-e7c30bee09b7ef3b --out-dir /work/oxidecomputer/crucible/target/debug/build/crucible-common-e7c30bee09b7ef3b -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern vergen=/work/oxidecomputer/crucible/target/debug/deps/libvergen-e75c33287bd2547c.rlib -L native=/work/oxidecomputer/crucible/target/debug/build/libgit2-sys-6120c1544d9c7fd8/out/build -L native=/usr/lib/amd64`
     Running `/work/oxidecomputer/crucible/target/debug/build/crucible-common-e7c30bee09b7ef3b/build-script-build`
     Running `rustc --crate-name crucible_common --edition=2021 common/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=c45901e9152d33f4 -C extra-filename=-c45901e9152d33f4 --out-dir /work/oxidecomputer/crucible/target/debug/deps -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rmeta --extern atty=/work/oxidecomputer/crucible/target/debug/deps/libatty-bfb6a2cdc762f7c4.rmeta --extern nix=/work/oxidecomputer/crucible/target/debug/deps/libnix-3635aff9412bf811.rmeta --extern rusqlite=/work/oxidecomputer/crucible/target/debug/deps/librusqlite-e4d2316a88b06837.rmeta --extern rustls_pemfile=/work/oxidecomputer/crucible/target/debug/deps/librustls_pemfile-e52b2a6debfcae48.rmeta --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rmeta --extern slog=/work/oxidecomputer/crucible/target/debug/deps/libslog-84fd25666c3c26ee.rmeta --extern slog_async=/work/oxidecomputer/crucible/target/debug/deps/libslog_async-03e743f036119074.rmeta --extern slog_bunyan=/work/oxidecomputer/crucible/target/debug/deps/libslog_bunyan-dce051a6775f1d99.rmeta --extern slog_dtrace=/work/oxidecomputer/crucible/target/debug/deps/libslog_dtrace-8024beacfb95325b.rmeta --extern slog_term=/work/oxidecomputer/crucible/target/debug/deps/libslog_term-31c687431f6dd53c.rmeta --extern tempfile=/work/oxidecomputer/crucible/target/debug/deps/libtempfile-05cb6a5f7e86cb66.rmeta --extern thiserror=/work/oxidecomputer/crucible/target/debug/deps/libthiserror-07eca56f531a0e5d.rmeta --extern tokio_rustls=/work/oxidecomputer/crucible/target/debug/deps/libtokio_rustls-eafe4ab74a176b7d.rmeta --extern toml=/work/oxidecomputer/crucible/target/debug/deps/libtoml-de0eb3fcc3b95b5c.rmeta --extern twox_hash=/work/oxidecomputer/crucible/target/debug/deps/libtwox_hash-9f5dd4f7319ca539.rmeta --extern uuid=/work/oxidecomputer/crucible/target/debug/deps/libuuid-7cc8e87b4149b49e.rmeta -L native=/usr/lib/amd64 -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out`
     Running `rustc --crate-name crucible_common --edition=2021 common/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --emit=dep-info,link -C embed-bitcode=no -C debuginfo=2 --test -C metadata=9a32809bdbdf85c4 -C extra-filename=-9a32809bdbdf85c4 --out-dir /work/oxidecomputer/crucible/target/debug/deps -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rlib --extern atty=/work/oxidecomputer/crucible/target/debug/deps/libatty-bfb6a2cdc762f7c4.rlib --extern nix=/work/oxidecomputer/crucible/target/debug/deps/libnix-3635aff9412bf811.rlib --extern rusqlite=/work/oxidecomputer/crucible/target/debug/deps/librusqlite-e4d2316a88b06837.rlib --extern rustls_pemfile=/work/oxidecomputer/crucible/target/debug/deps/librustls_pemfile-e52b2a6debfcae48.rlib --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rlib --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rlib --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rlib --extern slog=/work/oxidecomputer/crucible/target/debug/deps/libslog-84fd25666c3c26ee.rlib --extern slog_async=/work/oxidecomputer/crucible/target/debug/deps/libslog_async-03e743f036119074.rlib --extern slog_bunyan=/work/oxidecomputer/crucible/target/debug/deps/libslog_bunyan-dce051a6775f1d99.rlib --extern slog_dtrace=/work/oxidecomputer/crucible/target/debug/deps/libslog_dtrace-8024beacfb95325b.rlib --extern slog_term=/work/oxidecomputer/crucible/target/debug/deps/libslog_term-31c687431f6dd53c.rlib --extern tempfile=/work/oxidecomputer/crucible/target/debug/deps/libtempfile-05cb6a5f7e86cb66.rlib --extern thiserror=/work/oxidecomputer/crucible/target/debug/deps/libthiserror-07eca56f531a0e5d.rlib --extern tokio_rustls=/work/oxidecomputer/crucible/target/debug/deps/libtokio_rustls-eafe4ab74a176b7d.rlib --extern toml=/work/oxidecomputer/crucible/target/debug/deps/libtoml-de0eb3fcc3b95b5c.rlib --extern twox_hash=/work/oxidecomputer/crucible/target/debug/deps/libtwox_hash-9f5dd4f7319ca539.rlib --extern uuid=/work/oxidecomputer/crucible/target/debug/deps/libuuid-7cc8e87b4149b49e.rlib -L native=/usr/lib/amd64 -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out`
   Compiling crucible-protocol v0.0.0 (/work/oxidecomputer/crucible/protocol)
     Running `rustc --crate-name crucible_protocol --edition=2021 protocol/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=0682e169c907a102 -C extra-filename=-0682e169c907a102 --out-dir /work/oxidecomputer/crucible/target/debug/deps -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rmeta --extern bincode=/work/oxidecomputer/crucible/target/debug/deps/libbincode-bcb925e8faac86cd.rmeta --extern bytes=/work/oxidecomputer/crucible/target/debug/deps/libbytes-64a8a55ef81e55dd.rmeta --extern clap=/work/oxidecomputer/crucible/target/debug/deps/libclap-49bc17aade028e79.rmeta --extern crucible_common=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_common-c45901e9152d33f4.rmeta --extern futures=/work/oxidecomputer/crucible/target/debug/deps/libfutures-df761c89bfa71e54.rmeta --extern num_enum=/work/oxidecomputer/crucible/target/debug/deps/libnum_enum-9cd7a6d9dcf1dd5a.rmeta --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern statistical=/work/oxidecomputer/crucible/target/debug/deps/libstatistical-f2809857d7fe1c91.rmeta --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rmeta --extern tokio_util=/work/oxidecomputer/crucible/target/debug/deps/libtokio_util-279b3765a2b5aad1.rmeta --extern uuid=/work/oxidecomputer/crucible/target/debug/deps/libuuid-7cc8e87b4149b49e.rmeta -L native=/usr/lib/amd64 -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out`
   Compiling dsc v0.1.0 (/work/oxidecomputer/crucible/dsc)
     Running `rustc --crate-name crucible_agent_client --edition=2021 agent-client/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --emit=dep-info,link -C embed-bitcode=no -C debuginfo=2 --test -C metadata=6f3900e8033b57ec -C extra-filename=-6f3900e8033b57ec --out-dir /work/oxidecomputer/crucible/target/debug/deps -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rlib --extern chrono=/work/oxidecomputer/crucible/target/debug/deps/libchrono-0402a5e17dd99cdf.rlib --extern percent_encoding=/work/oxidecomputer/crucible/target/debug/deps/libpercent_encoding-0000aebce3d30803.rlib --extern progenitor=/work/oxidecomputer/crucible/target/debug/deps/libprogenitor-0319e0dfd841f493.rlib --extern reqwest=/work/oxidecomputer/crucible/target/debug/deps/libreqwest-6407fc4e9374ca8e.rlib --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rlib --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rlib --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rlib -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out -L native=/usr/ssl-3/lib/amd64`
     Running `rustc --crate-name dsc --edition=2021 dsc/src/main.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --emit=dep-info,link -C embed-bitcode=no -C debuginfo=2 --test -C metadata=2b80980cbc3bac2c -C extra-filename=-2b80980cbc3bac2c --out-dir /work/oxidecomputer/crucible/target/debug/deps -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rlib --extern byte_unit=/work/oxidecomputer/crucible/target/debug/deps/libbyte_unit-02cb17c857e20dac.rlib --extern clap=/work/oxidecomputer/crucible/target/debug/deps/libclap-49bc17aade028e79.rlib --extern csv=/work/oxidecomputer/crucible/target/debug/deps/libcsv-187f0e890389cec3.rlib --extern dropshot=/work/oxidecomputer/crucible/target/debug/deps/libdropshot-a49a4505c9c6b86f.rlib --extern dsc_client=/work/oxidecomputer/crucible/target/debug/deps/libdsc_client-15b0c81fa833cf0f.rlib --extern expectorate=/work/oxidecomputer/crucible/target/debug/deps/libexpectorate-4e45b262baa473cc.rlib --extern openapi_lint=/work/oxidecomputer/crucible/target/debug/deps/libopenapi_lint-848bb99097a9a843.rlib --extern openapiv3=/work/oxidecomputer/crucible/target/debug/deps/libopenapiv3-f84072cef6d0d68c.rlib --extern rand=/work/oxidecomputer/crucible/target/debug/deps/librand-1f91a9ea4aed49ee.rlib --extern rand_chacha=/work/oxidecomputer/crucible/target/debug/deps/librand_chacha-add466c063ef8725.rlib --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rlib --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rlib --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rlib --extern statistical=/work/oxidecomputer/crucible/target/debug/deps/libstatistical-f2809857d7fe1c91.rlib --extern tempfile=/work/oxidecomputer/crucible/target/debug/deps/libtempfile-05cb6a5f7e86cb66.rlib --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rlib -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out -L native=/usr/ssl-3/lib/amd64`
     Running `rustc --crate-name dsc_client --edition=2021 dsc-client/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --emit=dep-info,link -C embed-bitcode=no -C debuginfo=2 --test -C metadata=f8a5b497695371e1 -C extra-filename=-f8a5b497695371e1 --out-dir /work/oxidecomputer/crucible/target/debug/deps -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rlib --extern percent_encoding=/work/oxidecomputer/crucible/target/debug/deps/libpercent_encoding-0000aebce3d30803.rlib --extern progenitor=/work/oxidecomputer/crucible/target/debug/deps/libprogenitor-0319e0dfd841f493.rlib --extern reqwest=/work/oxidecomputer/crucible/target/debug/deps/libreqwest-6407fc4e9374ca8e.rlib --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rlib --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rlib --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rlib -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out -L native=/usr/ssl-3/lib/amd64`
     Running `rustc --crate-name crucible_pantry_client --edition=2021 pantry-client/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --emit=dep-info,link -C embed-bitcode=no -C debuginfo=2 --test -C metadata=8a27f01eb086219e -C extra-filename=-8a27f01eb086219e --out-dir /work/oxidecomputer/crucible/target/debug/deps -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rlib --extern chrono=/work/oxidecomputer/crucible/target/debug/deps/libchrono-0402a5e17dd99cdf.rlib --extern percent_encoding=/work/oxidecomputer/crucible/target/debug/deps/libpercent_encoding-0000aebce3d30803.rlib --extern progenitor=/work/oxidecomputer/crucible/target/debug/deps/libprogenitor-0319e0dfd841f493.rlib --extern reqwest=/work/oxidecomputer/crucible/target/debug/deps/libreqwest-6407fc4e9374ca8e.rlib --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rlib --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rlib --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rlib --extern uuid=/work/oxidecomputer/crucible/target/debug/deps/libuuid-7cc8e87b4149b49e.rlib -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out -L native=/usr/ssl-3/lib/amd64`
   Compiling nexus-client v0.1.0 (https://github.com/oxidecomputer/omicron?branch=main#4c05962d)
     Running `rustc --crate-name nexus_client --edition=2021 /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/4c05962/nexus-client/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=fd6034b144d15fe8 -C extra-filename=-fd6034b144d15fe8 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern chrono=/work/oxidecomputer/crucible/target/debug/deps/libchrono-0402a5e17dd99cdf.rmeta --extern futures=/work/oxidecomputer/crucible/target/debug/deps/libfutures-df761c89bfa71e54.rmeta --extern ipnetwork=/work/oxidecomputer/crucible/target/debug/deps/libipnetwork-0e9e550a49db2c52.rmeta --extern omicron_common=/work/oxidecomputer/crucible/target/debug/deps/libomicron_common-52df1fff8b36d94c.rmeta --extern omicron_passwords=/work/oxidecomputer/crucible/target/debug/deps/libomicron_passwords-ac6e3a602e6ad041.rmeta --extern progenitor=/work/oxidecomputer/crucible/target/debug/deps/libprogenitor-0319e0dfd841f493.rmeta --extern regress=/work/oxidecomputer/crucible/target/debug/deps/libregress-10da65958da1c830.rmeta --extern reqwest=/work/oxidecomputer/crucible/target/debug/deps/libreqwest-6407fc4e9374ca8e.rmeta --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rmeta --extern slog=/work/oxidecomputer/crucible/target/debug/deps/libslog-84fd25666c3c26ee.rmeta --extern uuid=/work/oxidecomputer/crucible/target/debug/deps/libuuid-7cc8e87b4149b49e.rmeta --cap-lints allow -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out -L native=/usr/ssl-3/lib/amd64`
   Compiling oximeter v0.1.0 (https://github.com/oxidecomputer/omicron?branch=main#4c05962d)
     Running `rustc --crate-name oximeter --edition=2021 /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/4c05962/oximeter/oximeter/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=90ae047d6b643e4e -C extra-filename=-90ae047d6b643e4e --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern bytes=/work/oxidecomputer/crucible/target/debug/deps/libbytes-64a8a55ef81e55dd.rmeta --extern chrono=/work/oxidecomputer/crucible/target/debug/deps/libchrono-0402a5e17dd99cdf.rmeta --extern num_traits=/work/oxidecomputer/crucible/target/debug/deps/libnum_traits-8e50de91aba3f8f9.rmeta --extern omicron_common=/work/oxidecomputer/crucible/target/debug/deps/libomicron_common-52df1fff8b36d94c.rmeta --extern oximeter_macro_impl=/work/oxidecomputer/crucible/target/debug/deps/liboximeter_macro_impl-e4cc949eda20c416.so --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern thiserror=/work/oxidecomputer/crucible/target/debug/deps/libthiserror-07eca56f531a0e5d.rmeta --extern uuid=/work/oxidecomputer/crucible/target/debug/deps/libuuid-7cc8e87b4149b49e.rmeta --cap-lints allow -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out -L native=/usr/ssl-3/lib/amd64`
     Running `rustc --crate-name crucible_protocol --edition=2021 protocol/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --emit=dep-info,link -C embed-bitcode=no -C debuginfo=2 --test -C metadata=c776b78ce4b42bf6 -C extra-filename=-c776b78ce4b42bf6 --out-dir /work/oxidecomputer/crucible/target/debug/deps -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rlib --extern bincode=/work/oxidecomputer/crucible/target/debug/deps/libbincode-bcb925e8faac86cd.rlib --extern bytes=/work/oxidecomputer/crucible/target/debug/deps/libbytes-64a8a55ef81e55dd.rlib --extern clap=/work/oxidecomputer/crucible/target/debug/deps/libclap-49bc17aade028e79.rlib --extern crucible_common=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_common-c45901e9152d33f4.rlib --extern futures=/work/oxidecomputer/crucible/target/debug/deps/libfutures-df761c89bfa71e54.rlib --extern num_enum=/work/oxidecomputer/crucible/target/debug/deps/libnum_enum-9cd7a6d9dcf1dd5a.rlib --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rlib --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rlib --extern statistical=/work/oxidecomputer/crucible/target/debug/deps/libstatistical-f2809857d7fe1c91.rlib --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rlib --extern tokio_util=/work/oxidecomputer/crucible/target/debug/deps/libtokio_util-279b3765a2b5aad1.rlib --extern uuid=/work/oxidecomputer/crucible/target/debug/deps/libuuid-7cc8e87b4149b49e.rlib -L native=/usr/lib/amd64 -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out`
   Compiling crucible-package v0.1.0 (/work/oxidecomputer/crucible/package)
     Running `rustc --crate-name crucible_package --edition=2021 package/src/main.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --emit=dep-info,link -C embed-bitcode=no -C debuginfo=2 --test -C metadata=e8ff0170d25e0da5 -C extra-filename=-e8ff0170d25e0da5 --out-dir /work/oxidecomputer/crucible/target/debug/deps -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rlib --extern omicron_zone_package=/work/oxidecomputer/crucible/target/debug/deps/libomicron_zone_package-8423a7bf8bd88040.rlib --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rlib -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out -L native=/usr/ssl-3/lib/amd64`
     Running `rustc --crate-name crucible_control_client --edition=2021 control-client/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --emit=dep-info,link -C embed-bitcode=no -C debuginfo=2 --test -C metadata=d0a58354872d46d9 -C extra-filename=-d0a58354872d46d9 --out-dir /work/oxidecomputer/crucible/target/debug/deps -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rlib --extern percent_encoding=/work/oxidecomputer/crucible/target/debug/deps/libpercent_encoding-0000aebce3d30803.rlib --extern progenitor=/work/oxidecomputer/crucible/target/debug/deps/libprogenitor-0319e0dfd841f493.rlib --extern reqwest=/work/oxidecomputer/crucible/target/debug/deps/libreqwest-6407fc4e9374ca8e.rlib --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rlib --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rlib --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rlib -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out -L native=/usr/ssl-3/lib/amd64`
     Running `rustc --crate-name repair_client --edition=2021 repair-client/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --emit=dep-info,link -C embed-bitcode=no -C debuginfo=2 --test -C metadata=5353c8de97b4615f -C extra-filename=-5353c8de97b4615f --out-dir /work/oxidecomputer/crucible/target/debug/deps -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rlib --extern chrono=/work/oxidecomputer/crucible/target/debug/deps/libchrono-0402a5e17dd99cdf.rlib --extern percent_encoding=/work/oxidecomputer/crucible/target/debug/deps/libpercent_encoding-0000aebce3d30803.rlib --extern progenitor=/work/oxidecomputer/crucible/target/debug/deps/libprogenitor-0319e0dfd841f493.rlib --extern reqwest=/work/oxidecomputer/crucible/target/debug/deps/libreqwest-6407fc4e9374ca8e.rlib --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rlib --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rlib --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rlib -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out -L native=/usr/ssl-3/lib/amd64`
     Running `rustc --crate-name crucible_protocol --edition=2021 protocol/src/main.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --emit=dep-info,link -C embed-bitcode=no -C debuginfo=2 --test -C metadata=d81e6562be2ffe77 -C extra-filename=-d81e6562be2ffe77 --out-dir /work/oxidecomputer/crucible/target/debug/deps -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rlib --extern bincode=/work/oxidecomputer/crucible/target/debug/deps/libbincode-bcb925e8faac86cd.rlib --extern bytes=/work/oxidecomputer/crucible/target/debug/deps/libbytes-64a8a55ef81e55dd.rlib --extern clap=/work/oxidecomputer/crucible/target/debug/deps/libclap-49bc17aade028e79.rlib --extern crucible_common=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_common-c45901e9152d33f4.rlib --extern crucible_protocol=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_protocol-0682e169c907a102.rlib --extern futures=/work/oxidecomputer/crucible/target/debug/deps/libfutures-df761c89bfa71e54.rlib --extern num_enum=/work/oxidecomputer/crucible/target/debug/deps/libnum_enum-9cd7a6d9dcf1dd5a.rlib --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rlib --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rlib --extern statistical=/work/oxidecomputer/crucible/target/debug/deps/libstatistical-f2809857d7fe1c91.rlib --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rlib --extern tokio_util=/work/oxidecomputer/crucible/target/debug/deps/libtokio_util-279b3765a2b5aad1.rlib --extern uuid=/work/oxidecomputer/crucible/target/debug/deps/libuuid-7cc8e87b4149b49e.rlib -L native=/usr/lib/amd64 -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out`
   Compiling crucible-agent v0.0.1 (/work/oxidecomputer/crucible/agent)
     Running `rustc --crate-name crucible_agent --edition=2021 agent/src/main.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --emit=dep-info,link -C embed-bitcode=no -C debuginfo=2 --test -C metadata=acbf118b39db417b -C extra-filename=-acbf118b39db417b --out-dir /work/oxidecomputer/crucible/target/debug/deps -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rlib --extern chrono=/work/oxidecomputer/crucible/target/debug/deps/libchrono-0402a5e17dd99cdf.rlib --extern clap=/work/oxidecomputer/crucible/target/debug/deps/libclap-49bc17aade028e79.rlib --extern crucible_common=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_common-c45901e9152d33f4.rlib --extern crucible_smf=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_smf-e1a82c6f17385dc6.rlib --extern dropshot=/work/oxidecomputer/crucible/target/debug/deps/libdropshot-a49a4505c9c6b86f.rlib --extern expectorate=/work/oxidecomputer/crucible/target/debug/deps/libexpectorate-4e45b262baa473cc.rlib --extern futures=/work/oxidecomputer/crucible/target/debug/deps/libfutures-df761c89bfa71e54.rlib --extern http=/work/oxidecomputer/crucible/target/debug/deps/libhttp-4526ab951518c907.rlib --extern hyper=/work/oxidecomputer/crucible/target/debug/deps/libhyper-dad943d3b7cc33e9.rlib --extern omicron_common=/work/oxidecomputer/crucible/target/debug/deps/libomicron_common-52df1fff8b36d94c.rlib --extern openapi_lint=/work/oxidecomputer/crucible/target/debug/deps/libopenapi_lint-848bb99097a9a843.rlib --extern openapiv3=/work/oxidecomputer/crucible/target/debug/deps/libopenapiv3-f84072cef6d0d68c.rlib --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rlib --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rlib --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rlib --extern slog=/work/oxidecomputer/crucible/target/debug/deps/libslog-84fd25666c3c26ee.rlib --extern slog_term=/work/oxidecomputer/crucible/target/debug/deps/libslog_term-31c687431f6dd53c.rlib --extern subprocess=/work/oxidecomputer/crucible/target/debug/deps/libsubprocess-0acfc5c9b903588a.rlib --extern tempfile=/work/oxidecomputer/crucible/target/debug/deps/libtempfile-05cb6a5f7e86cb66.rlib --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rlib --extern uuid=/work/oxidecomputer/crucible/target/debug/deps/libuuid-7cc8e87b4149b49e.rlib -L native=/usr/lib/amd64 -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out -L native=/usr/ssl-3/lib/amd64`
   Compiling oximeter-producer v0.1.0 (https://github.com/oxidecomputer/omicron?branch=main#4c05962d)
     Running `rustc --crate-name oximeter_producer --edition=2021 /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/4c05962/oximeter/producer/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=5dc4f732e258486e -C extra-filename=-5dc4f732e258486e --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern chrono=/work/oxidecomputer/crucible/target/debug/deps/libchrono-0402a5e17dd99cdf.rmeta --extern dropshot=/work/oxidecomputer/crucible/target/debug/deps/libdropshot-a49a4505c9c6b86f.rmeta --extern nexus_client=/work/oxidecomputer/crucible/target/debug/deps/libnexus_client-fd6034b144d15fe8.rmeta --extern omicron_common=/work/oxidecomputer/crucible/target/debug/deps/libomicron_common-52df1fff8b36d94c.rmeta --extern oximeter=/work/oxidecomputer/crucible/target/debug/deps/liboximeter-90ae047d6b643e4e.rmeta --extern reqwest=/work/oxidecomputer/crucible/target/debug/deps/libreqwest-6407fc4e9374ca8e.rmeta --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern slog=/work/oxidecomputer/crucible/target/debug/deps/libslog-84fd25666c3c26ee.rmeta --extern slog_dtrace=/work/oxidecomputer/crucible/target/debug/deps/libslog_dtrace-8024beacfb95325b.rmeta --extern thiserror=/work/oxidecomputer/crucible/target/debug/deps/libthiserror-07eca56f531a0e5d.rmeta --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rmeta --extern uuid=/work/oxidecomputer/crucible/target/debug/deps/libuuid-7cc8e87b4149b49e.rmeta --cap-lints allow -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out -L native=/usr/ssl-3/lib/amd64`
   Compiling crucible v0.0.1 (/work/oxidecomputer/crucible/upstairs)
     Running `rustc --crate-name crucible --edition=2021 upstairs/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=41ca439abdc23695 -C extra-filename=-41ca439abdc23695 --out-dir /work/oxidecomputer/crucible/target/debug/deps -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern aes_gcm_siv=/work/oxidecomputer/crucible/target/debug/deps/libaes_gcm_siv-21495b616a07c9a4.rmeta --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rmeta --extern async_recursion=/work/oxidecomputer/crucible/target/debug/deps/libasync_recursion-ce9499495a1cb858.so --extern async_trait=/work/oxidecomputer/crucible/target/debug/deps/libasync_trait-a300e84178ee0ad1.so --extern base64=/work/oxidecomputer/crucible/target/debug/deps/libbase64-c139bdd129e780ac.rmeta --extern bytes=/work/oxidecomputer/crucible/target/debug/deps/libbytes-64a8a55ef81e55dd.rmeta --extern chrono=/work/oxidecomputer/crucible/target/debug/deps/libchrono-0402a5e17dd99cdf.rmeta --extern crucible_client_types=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_client_types-bd54c4335d2370bd.rmeta --extern crucible_common=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_common-c45901e9152d33f4.rmeta --extern crucible_protocol=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_protocol-0682e169c907a102.rmeta --extern dropshot=/work/oxidecomputer/crucible/target/debug/deps/libdropshot-a49a4505c9c6b86f.rmeta --extern futures=/work/oxidecomputer/crucible/target/debug/deps/libfutures-df761c89bfa71e54.rmeta --extern futures_core=/work/oxidecomputer/crucible/target/debug/deps/libfutures_core-46c6e3a1b3966417.rmeta --extern itertools=/work/oxidecomputer/crucible/target/debug/deps/libitertools-b06e69badd72e55c.rmeta --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --extern omicron_common=/work/oxidecomputer/crucible/target/debug/deps/libomicron_common-52df1fff8b36d94c.rmeta --extern oximeter=/work/oxidecomputer/crucible/target/debug/deps/liboximeter-90ae047d6b643e4e.rmeta --extern oximeter_producer=/work/oxidecomputer/crucible/target/debug/deps/liboximeter_producer-5dc4f732e258486e.rmeta --extern rand=/work/oxidecomputer/crucible/target/debug/deps/librand-1f91a9ea4aed49ee.rmeta --extern rand_chacha=/work/oxidecomputer/crucible/target/debug/deps/librand_chacha-add466c063ef8725.rmeta --extern reqwest=/work/oxidecomputer/crucible/target/debug/deps/libreqwest-6407fc4e9374ca8e.rmeta --extern ringbuffer=/work/oxidecomputer/crucible/target/debug/deps/libringbuffer-e91d75e4694e6351.rmeta --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rmeta --extern slog=/work/oxidecomputer/crucible/target/debug/deps/libslog-84fd25666c3c26ee.rmeta --extern slog_async=/work/oxidecomputer/crucible/target/debug/deps/libslog_async-03e743f036119074.rmeta --extern slog_dtrace=/work/oxidecomputer/crucible/target/debug/deps/libslog_dtrace-8024beacfb95325b.rmeta --extern slog_term=/work/oxidecomputer/crucible/target/debug/deps/libslog_term-31c687431f6dd53c.rmeta --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rmeta --extern tokio_rustls=/work/oxidecomputer/crucible/target/debug/deps/libtokio_rustls-eafe4ab74a176b7d.rmeta --extern tokio_util=/work/oxidecomputer/crucible/target/debug/deps/libtokio_util-279b3765a2b5aad1.rmeta --extern toml=/work/oxidecomputer/crucible/target/debug/deps/libtoml-de0eb3fcc3b95b5c.rmeta --extern tracing=/work/oxidecomputer/crucible/target/debug/deps/libtracing-1de351a7f2b0ab55.rmeta --extern usdt=/work/oxidecomputer/crucible/target/debug/deps/libusdt-86bb76e3b8fcea87.rmeta --extern uuid=/work/oxidecomputer/crucible/target/debug/deps/libuuid-7cc8e87b4149b49e.rmeta -L native=/usr/lib/amd64 -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out -L native=/usr/ssl-3/lib/amd64`
     Running `rustc --crate-name crucible --edition=2021 upstairs/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --emit=dep-info,link -C embed-bitcode=no -C debuginfo=2 --test -C metadata=53d074fabbf363e8 -C extra-filename=-53d074fabbf363e8 --out-dir /work/oxidecomputer/crucible/target/debug/deps -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern aes_gcm_siv=/work/oxidecomputer/crucible/target/debug/deps/libaes_gcm_siv-21495b616a07c9a4.rlib --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rlib --extern async_recursion=/work/oxidecomputer/crucible/target/debug/deps/libasync_recursion-ce9499495a1cb858.so --extern async_trait=/work/oxidecomputer/crucible/target/debug/deps/libasync_trait-a300e84178ee0ad1.so --extern base64=/work/oxidecomputer/crucible/target/debug/deps/libbase64-c139bdd129e780ac.rlib --extern bytes=/work/oxidecomputer/crucible/target/debug/deps/libbytes-64a8a55ef81e55dd.rlib --extern chrono=/work/oxidecomputer/crucible/target/debug/deps/libchrono-0402a5e17dd99cdf.rlib --extern crucible_client_types=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_client_types-bd54c4335d2370bd.rlib --extern crucible_common=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_common-c45901e9152d33f4.rlib --extern crucible_protocol=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_protocol-0682e169c907a102.rlib --extern dropshot=/work/oxidecomputer/crucible/target/debug/deps/libdropshot-a49a4505c9c6b86f.rlib --extern expectorate=/work/oxidecomputer/crucible/target/debug/deps/libexpectorate-4e45b262baa473cc.rlib --extern futures=/work/oxidecomputer/crucible/target/debug/deps/libfutures-df761c89bfa71e54.rlib --extern futures_core=/work/oxidecomputer/crucible/target/debug/deps/libfutures_core-46c6e3a1b3966417.rlib --extern itertools=/work/oxidecomputer/crucible/target/debug/deps/libitertools-b06e69badd72e55c.rlib --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rlib --extern omicron_common=/work/oxidecomputer/crucible/target/debug/deps/libomicron_common-52df1fff8b36d94c.rlib --extern openapi_lint=/work/oxidecomputer/crucible/target/debug/deps/libopenapi_lint-848bb99097a9a843.rlib --extern openapiv3=/work/oxidecomputer/crucible/target/debug/deps/libopenapiv3-f84072cef6d0d68c.rlib --extern oximeter=/work/oxidecomputer/crucible/target/debug/deps/liboximeter-90ae047d6b643e4e.rlib --extern oximeter_producer=/work/oxidecomputer/crucible/target/debug/deps/liboximeter_producer-5dc4f732e258486e.rlib --extern proptest=/work/oxidecomputer/crucible/target/debug/deps/libproptest-327f7f2cf6858f27.rlib --extern rand=/work/oxidecomputer/crucible/target/debug/deps/librand-1f91a9ea4aed49ee.rlib --extern rand_chacha=/work/oxidecomputer/crucible/target/debug/deps/librand_chacha-add466c063ef8725.rlib --extern reqwest=/work/oxidecomputer/crucible/target/debug/deps/libreqwest-6407fc4e9374ca8e.rlib --extern ringbuffer=/work/oxidecomputer/crucible/target/debug/deps/libringbuffer-e91d75e4694e6351.rlib --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rlib --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rlib --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rlib --extern slog=/work/oxidecomputer/crucible/target/debug/deps/libslog-84fd25666c3c26ee.rlib --extern slog_async=/work/oxidecomputer/crucible/target/debug/deps/libslog_async-03e743f036119074.rlib --extern slog_dtrace=/work/oxidecomputer/crucible/target/debug/deps/libslog_dtrace-8024beacfb95325b.rlib --extern slog_term=/work/oxidecomputer/crucible/target/debug/deps/libslog_term-31c687431f6dd53c.rlib --extern tempfile=/work/oxidecomputer/crucible/target/debug/deps/libtempfile-05cb6a5f7e86cb66.rlib --extern test_strategy=/work/oxidecomputer/crucible/target/debug/deps/libtest_strategy-5eb6b90d55d9f739.so --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rlib --extern tokio_rustls=/work/oxidecomputer/crucible/target/debug/deps/libtokio_rustls-eafe4ab74a176b7d.rlib --extern tokio_test=/work/oxidecomputer/crucible/target/debug/deps/libtokio_test-12a28be646ff63e6.rlib --extern tokio_util=/work/oxidecomputer/crucible/target/debug/deps/libtokio_util-279b3765a2b5aad1.rlib --extern toml=/work/oxidecomputer/crucible/target/debug/deps/libtoml-de0eb3fcc3b95b5c.rlib --extern tracing=/work/oxidecomputer/crucible/target/debug/deps/libtracing-1de351a7f2b0ab55.rlib --extern usdt=/work/oxidecomputer/crucible/target/debug/deps/libusdt-86bb76e3b8fcea87.rlib --extern uuid=/work/oxidecomputer/crucible/target/debug/deps/libuuid-7cc8e87b4149b49e.rlib -L native=/usr/lib/amd64 -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out -L native=/usr/ssl-3/lib/amd64`
   Compiling crucible-downstairs v0.0.1 (/work/oxidecomputer/crucible/downstairs)
   Compiling crucible-pantry v0.0.1 (/work/oxidecomputer/crucible/pantry)
     Running `rustc --crate-name crucible_downstairs --edition=2021 downstairs/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' -C metadata=6276be71be5284a4 -C extra-filename=-6276be71be5284a4 --out-dir /work/oxidecomputer/crucible/target/debug/deps -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rmeta --extern bincode=/work/oxidecomputer/crucible/target/debug/deps/libbincode-bcb925e8faac86cd.rmeta --extern bytes=/work/oxidecomputer/crucible/target/debug/deps/libbytes-64a8a55ef81e55dd.rmeta --extern chrono=/work/oxidecomputer/crucible/target/debug/deps/libchrono-0402a5e17dd99cdf.rmeta --extern clap=/work/oxidecomputer/crucible/target/debug/deps/libclap-49bc17aade028e79.rmeta --extern crucible=/work/oxidecomputer/crucible/target/debug/deps/libcrucible-41ca439abdc23695.rmeta --extern crucible_common=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_common-c45901e9152d33f4.rmeta --extern crucible_protocol=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_protocol-0682e169c907a102.rmeta --extern dropshot=/work/oxidecomputer/crucible/target/debug/deps/libdropshot-a49a4505c9c6b86f.rmeta --extern futures=/work/oxidecomputer/crucible/target/debug/deps/libfutures-df761c89bfa71e54.rmeta --extern futures_core=/work/oxidecomputer/crucible/target/debug/deps/libfutures_core-46c6e3a1b3966417.rmeta --extern hex=/work/oxidecomputer/crucible/target/debug/deps/libhex-6531b11cb72de3e5.rmeta --extern http=/work/oxidecomputer/crucible/target/debug/deps/libhttp-4526ab951518c907.rmeta --extern hyper=/work/oxidecomputer/crucible/target/debug/deps/libhyper-dad943d3b7cc33e9.rmeta --extern hyper_staticfile=/work/oxidecomputer/crucible/target/debug/deps/libhyper_staticfile-559b4389ef952563.rmeta --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --extern mime_guess=/work/oxidecomputer/crucible/target/debug/deps/libmime_guess-66974d6c31968dc2.rmeta --extern nix=/work/oxidecomputer/crucible/target/debug/deps/libnix-3635aff9412bf811.rmeta --extern omicron_common=/work/oxidecomputer/crucible/target/debug/deps/libomicron_common-52df1fff8b36d94c.rmeta --extern opentelemetry=/work/oxidecomputer/crucible/target/debug/deps/libopentelemetry-5524fd7817ad57fb.rmeta --extern opentelemetry_jaeger=/work/oxidecomputer/crucible/target/debug/deps/libopentelemetry_jaeger-9ebf47742e5e063f.rmeta --extern oximeter=/work/oxidecomputer/crucible/target/debug/deps/liboximeter-90ae047d6b643e4e.rmeta --extern oximeter_producer=/work/oxidecomputer/crucible/target/debug/deps/liboximeter_producer-5dc4f732e258486e.rmeta --extern rand=/work/oxidecomputer/crucible/target/debug/deps/librand-1f91a9ea4aed49ee.rmeta --extern repair_client=/work/oxidecomputer/crucible/target/debug/deps/librepair_client-1452d56087b6ccb7.rmeta --extern reqwest=/work/oxidecomputer/crucible/target/debug/deps/libreqwest-6407fc4e9374ca8e.rmeta --extern ringbuffer=/work/oxidecomputer/crucible/target/debug/deps/libringbuffer-e91d75e4694e6351.rmeta --extern rusqlite=/work/oxidecomputer/crucible/target/debug/deps/librusqlite-e4d2316a88b06837.rmeta --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rmeta --extern sha2=/work/oxidecomputer/crucible/target/debug/deps/libsha2-9b09b4b286e2cb62.rmeta --extern slog=/work/oxidecomputer/crucible/target/debug/deps/libslog-84fd25666c3c26ee.rmeta --extern slog_async=/work/oxidecomputer/crucible/target/debug/deps/libslog_async-03e743f036119074.rmeta --extern slog_dtrace=/work/oxidecomputer/crucible/target/debug/deps/libslog_dtrace-8024beacfb95325b.rmeta --extern slog_term=/work/oxidecomputer/crucible/target/debug/deps/libslog_term-31c687431f6dd53c.rmeta --extern statistical=/work/oxidecomputer/crucible/target/debug/deps/libstatistical-f2809857d7fe1c91.rmeta --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rmeta --extern tokio_rustls=/work/oxidecomputer/crucible/target/debug/deps/libtokio_rustls-eafe4ab74a176b7d.rmeta --extern tokio_util=/work/oxidecomputer/crucible/target/debug/deps/libtokio_util-279b3765a2b5aad1.rmeta --extern toml=/work/oxidecomputer/crucible/target/debug/deps/libtoml-de0eb3fcc3b95b5c.rmeta --extern tracing=/work/oxidecomputer/crucible/target/debug/deps/libtracing-1de351a7f2b0ab55.rmeta --extern tracing_opentelemetry=/work/oxidecomputer/crucible/target/debug/deps/libtracing_opentelemetry-8f8cfc1900c3a663.rmeta --extern tracing_subscriber=/work/oxidecomputer/crucible/target/debug/deps/libtracing_subscriber-e39dae5ba339bc78.rmeta --extern usdt=/work/oxidecomputer/crucible/target/debug/deps/libusdt-86bb76e3b8fcea87.rmeta --extern uuid=/work/oxidecomputer/crucible/target/debug/deps/libuuid-7cc8e87b4149b49e.rmeta -L native=/usr/lib/amd64 -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out -L native=/usr/ssl-3/lib/amd64`
     Running `rustc --crate-name crucible_pantry --edition=2021 pantry/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=fe633af5059fe3a7 -C extra-filename=-fe633af5059fe3a7 --out-dir /work/oxidecomputer/crucible/target/debug/deps -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rmeta --extern base64=/work/oxidecomputer/crucible/target/debug/deps/libbase64-c139bdd129e780ac.rmeta --extern chrono=/work/oxidecomputer/crucible/target/debug/deps/libchrono-0402a5e17dd99cdf.rmeta --extern clap=/work/oxidecomputer/crucible/target/debug/deps/libclap-49bc17aade028e79.rmeta --extern crucible=/work/oxidecomputer/crucible/target/debug/deps/libcrucible-41ca439abdc23695.rmeta --extern crucible_common=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_common-c45901e9152d33f4.rmeta --extern crucible_smf=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_smf-e1a82c6f17385dc6.rmeta --extern dropshot=/work/oxidecomputer/crucible/target/debug/deps/libdropshot-a49a4505c9c6b86f.rmeta --extern futures=/work/oxidecomputer/crucible/target/debug/deps/libfutures-df761c89bfa71e54.rmeta --extern hex=/work/oxidecomputer/crucible/target/debug/deps/libhex-6531b11cb72de3e5.rmeta --extern http=/work/oxidecomputer/crucible/target/debug/deps/libhttp-4526ab951518c907.rmeta --extern hyper=/work/oxidecomputer/crucible/target/debug/deps/libhyper-dad943d3b7cc33e9.rmeta --extern omicron_common=/work/oxidecomputer/crucible/target/debug/deps/libomicron_common-52df1fff8b36d94c.rmeta --extern reqwest=/work/oxidecomputer/crucible/target/debug/deps/libreqwest-6407fc4e9374ca8e.rmeta --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rmeta --extern sha2=/work/oxidecomputer/crucible/target/debug/deps/libsha2-9b09b4b286e2cb62.rmeta --extern slog=/work/oxidecomputer/crucible/target/debug/deps/libslog-84fd25666c3c26ee.rmeta --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rmeta --extern uuid=/work/oxidecomputer/crucible/target/debug/deps/libuuid-7cc8e87b4149b49e.rmeta -L native=/usr/lib/amd64 -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out -L native=/usr/ssl-3/lib/amd64`
   Compiling crutest v0.1.0 (/work/oxidecomputer/crucible/crutest)
   Compiling measure-iops v0.0.1 (/work/oxidecomputer/crucible/measure_iops)
   Compiling crucible-nbd-server v0.1.0 (/work/oxidecomputer/crucible/nbd_server)
     Running `rustc --crate-name crucible_pantry --edition=2021 pantry/src/main.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --emit=dep-info,link -C embed-bitcode=no -C debuginfo=2 --test -C metadata=b51bc30f7a0cbfa5 -C extra-filename=-b51bc30f7a0cbfa5 --out-dir /work/oxidecomputer/crucible/target/debug/deps -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rlib --extern base64=/work/oxidecomputer/crucible/target/debug/deps/libbase64-c139bdd129e780ac.rlib --extern chrono=/work/oxidecomputer/crucible/target/debug/deps/libchrono-0402a5e17dd99cdf.rlib --extern clap=/work/oxidecomputer/crucible/target/debug/deps/libclap-49bc17aade028e79.rlib --extern crucible=/work/oxidecomputer/crucible/target/debug/deps/libcrucible-41ca439abdc23695.rlib --extern crucible_common=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_common-c45901e9152d33f4.rlib --extern crucible_pantry=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_pantry-fe633af5059fe3a7.rlib --extern crucible_smf=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_smf-e1a82c6f17385dc6.rlib --extern dropshot=/work/oxidecomputer/crucible/target/debug/deps/libdropshot-a49a4505c9c6b86f.rlib --extern expectorate=/work/oxidecomputer/crucible/target/debug/deps/libexpectorate-4e45b262baa473cc.rlib --extern futures=/work/oxidecomputer/crucible/target/debug/deps/libfutures-df761c89bfa71e54.rlib --extern hex=/work/oxidecomputer/crucible/target/debug/deps/libhex-6531b11cb72de3e5.rlib --extern http=/work/oxidecomputer/crucible/target/debug/deps/libhttp-4526ab951518c907.rlib --extern hyper=/work/oxidecomputer/crucible/target/debug/deps/libhyper-dad943d3b7cc33e9.rlib --extern omicron_common=/work/oxidecomputer/crucible/target/debug/deps/libomicron_common-52df1fff8b36d94c.rlib --extern openapi_lint=/work/oxidecomputer/crucible/target/debug/deps/libopenapi_lint-848bb99097a9a843.rlib --extern openapiv3=/work/oxidecomputer/crucible/target/debug/deps/libopenapiv3-f84072cef6d0d68c.rlib --extern reqwest=/work/oxidecomputer/crucible/target/debug/deps/libreqwest-6407fc4e9374ca8e.rlib --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rlib --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rlib --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rlib --extern sha2=/work/oxidecomputer/crucible/target/debug/deps/libsha2-9b09b4b286e2cb62.rlib --extern slog=/work/oxidecomputer/crucible/target/debug/deps/libslog-84fd25666c3c26ee.rlib --extern subprocess=/work/oxidecomputer/crucible/target/debug/deps/libsubprocess-0acfc5c9b903588a.rlib --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rlib --extern uuid=/work/oxidecomputer/crucible/target/debug/deps/libuuid-7cc8e87b4149b49e.rlib -L native=/usr/lib/amd64 -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out -L native=/usr/ssl-3/lib/amd64`
     Running `rustc --crate-name crutest --edition=2021 crutest/src/main.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --emit=dep-info,link -C embed-bitcode=no -C debuginfo=2 --test -C metadata=af78e92d646e2d06 -C extra-filename=-af78e92d646e2d06 --out-dir /work/oxidecomputer/crucible/target/debug/deps -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rlib --extern bincode=/work/oxidecomputer/crucible/target/debug/deps/libbincode-bcb925e8faac86cd.rlib --extern bytes=/work/oxidecomputer/crucible/target/debug/deps/libbytes-64a8a55ef81e55dd.rlib --extern clap=/work/oxidecomputer/crucible/target/debug/deps/libclap-49bc17aade028e79.rlib --extern crossterm=/work/oxidecomputer/crucible/target/debug/deps/libcrossterm-3c787fd4c4d4bc45.rlib --extern crucible=/work/oxidecomputer/crucible/target/debug/deps/libcrucible-41ca439abdc23695.rlib --extern crucible_common=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_common-c45901e9152d33f4.rlib --extern crucible_protocol=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_protocol-0682e169c907a102.rlib --extern csv=/work/oxidecomputer/crucible/target/debug/deps/libcsv-187f0e890389cec3.rlib --extern dropshot=/work/oxidecomputer/crucible/target/debug/deps/libdropshot-a49a4505c9c6b86f.rlib --extern dsc_client=/work/oxidecomputer/crucible/target/debug/deps/libdsc_client-15b0c81fa833cf0f.rlib --extern futures=/work/oxidecomputer/crucible/target/debug/deps/libfutures-df761c89bfa71e54.rlib --extern futures_core=/work/oxidecomputer/crucible/target/debug/deps/libfutures_core-46c6e3a1b3966417.rlib --extern indicatif=/work/oxidecomputer/crucible/target/debug/deps/libindicatif-297a26a70875006e.rlib --extern omicron_common=/work/oxidecomputer/crucible/target/debug/deps/libomicron_common-52df1fff8b36d94c.rlib --extern oximeter=/work/oxidecomputer/crucible/target/debug/deps/liboximeter-90ae047d6b643e4e.rlib --extern oximeter_producer=/work/oxidecomputer/crucible/target/debug/deps/liboximeter_producer-5dc4f732e258486e.rlib --extern rand=/work/oxidecomputer/crucible/target/debug/deps/librand-1f91a9ea4aed49ee.rlib --extern rand_chacha=/work/oxidecomputer/crucible/target/debug/deps/librand_chacha-add466c063ef8725.rlib --extern reedline=/work/oxidecomputer/crucible/target/debug/deps/libreedline-6e6244e0f6aa654d.rlib --extern ringbuffer=/work/oxidecomputer/crucible/target/debug/deps/libringbuffer-e91d75e4694e6351.rlib --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rlib --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rlib --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rlib --extern signal_hook=/work/oxidecomputer/crucible/target/debug/deps/libsignal_hook-ae206b38743b6815.rlib --extern signal_hook_tokio=/work/oxidecomputer/crucible/target/debug/deps/libsignal_hook_tokio-6a6b104c61918fa0.rlib --extern statistical=/work/oxidecomputer/crucible/target/debug/deps/libstatistical-f2809857d7fe1c91.rlib --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rlib --extern tokio_util=/work/oxidecomputer/crucible/target/debug/deps/libtokio_util-279b3765a2b5aad1.rlib --extern toml=/work/oxidecomputer/crucible/target/debug/deps/libtoml-de0eb3fcc3b95b5c.rlib --extern uuid=/work/oxidecomputer/crucible/target/debug/deps/libuuid-7cc8e87b4149b49e.rlib -L native=/usr/lib/amd64 -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out -L native=/usr/ssl-3/lib/amd64`
     Running `rustc --crate-name measure_iops --edition=2021 measure_iops/src/main.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --emit=dep-info,link -C embed-bitcode=no -C debuginfo=2 --test -C metadata=cbdca99bf515defe -C extra-filename=-cbdca99bf515defe --out-dir /work/oxidecomputer/crucible/target/debug/deps -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rlib --extern bytes=/work/oxidecomputer/crucible/target/debug/deps/libbytes-64a8a55ef81e55dd.rlib --extern clap=/work/oxidecomputer/crucible/target/debug/deps/libclap-49bc17aade028e79.rlib --extern crucible=/work/oxidecomputer/crucible/target/debug/deps/libcrucible-41ca439abdc23695.rlib --extern crucible_common=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_common-c45901e9152d33f4.rlib --extern rand=/work/oxidecomputer/crucible/target/debug/deps/librand-1f91a9ea4aed49ee.rlib --extern statistical=/work/oxidecomputer/crucible/target/debug/deps/libstatistical-f2809857d7fe1c91.rlib --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rlib --extern uuid=/work/oxidecomputer/crucible/target/debug/deps/libuuid-7cc8e87b4149b49e.rlib -L native=/usr/lib/amd64 -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out -L native=/usr/ssl-3/lib/amd64`
     Running `rustc --crate-name crucible_nbd_server --edition=2021 nbd_server/src/main.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --emit=dep-info,link -C embed-bitcode=no -C debuginfo=2 --test -C metadata=8c1612631a1669fd -C extra-filename=-8c1612631a1669fd --out-dir /work/oxidecomputer/crucible/target/debug/deps -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rlib --extern bytes=/work/oxidecomputer/crucible/target/debug/deps/libbytes-64a8a55ef81e55dd.rlib --extern clap=/work/oxidecomputer/crucible/target/debug/deps/libclap-49bc17aade028e79.rlib --extern crucible=/work/oxidecomputer/crucible/target/debug/deps/libcrucible-41ca439abdc23695.rlib --extern crucible_common=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_common-c45901e9152d33f4.rlib --extern crucible_protocol=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_protocol-0682e169c907a102.rlib --extern futures=/work/oxidecomputer/crucible/target/debug/deps/libfutures-df761c89bfa71e54.rlib --extern futures_core=/work/oxidecomputer/crucible/target/debug/deps/libfutures_core-46c6e3a1b3966417.rlib --extern nbd=/work/oxidecomputer/crucible/target/debug/deps/libnbd-6a0c01a24def5e9a.rlib --extern ringbuffer=/work/oxidecomputer/crucible/target/debug/deps/libringbuffer-e91d75e4694e6351.rlib --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rlib --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rlib --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rlib --extern tokio_util=/work/oxidecomputer/crucible/target/debug/deps/libtokio_util-279b3765a2b5aad1.rlib --extern toml=/work/oxidecomputer/crucible/target/debug/deps/libtoml-de0eb3fcc3b95b5c.rlib -L native=/usr/lib/amd64 -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out -L native=/usr/ssl-3/lib/amd64`
     Running `rustc --crate-name crucible_pantry --edition=2021 pantry/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --emit=dep-info,link -C embed-bitcode=no -C debuginfo=2 --test -C metadata=8e1bf492bfe90e8c -C extra-filename=-8e1bf492bfe90e8c --out-dir /work/oxidecomputer/crucible/target/debug/deps -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rlib --extern base64=/work/oxidecomputer/crucible/target/debug/deps/libbase64-c139bdd129e780ac.rlib --extern chrono=/work/oxidecomputer/crucible/target/debug/deps/libchrono-0402a5e17dd99cdf.rlib --extern clap=/work/oxidecomputer/crucible/target/debug/deps/libclap-49bc17aade028e79.rlib --extern crucible=/work/oxidecomputer/crucible/target/debug/deps/libcrucible-41ca439abdc23695.rlib --extern crucible_common=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_common-c45901e9152d33f4.rlib --extern crucible_smf=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_smf-e1a82c6f17385dc6.rlib --extern dropshot=/work/oxidecomputer/crucible/target/debug/deps/libdropshot-a49a4505c9c6b86f.rlib --extern expectorate=/work/oxidecomputer/crucible/target/debug/deps/libexpectorate-4e45b262baa473cc.rlib --extern futures=/work/oxidecomputer/crucible/target/debug/deps/libfutures-df761c89bfa71e54.rlib --extern hex=/work/oxidecomputer/crucible/target/debug/deps/libhex-6531b11cb72de3e5.rlib --extern http=/work/oxidecomputer/crucible/target/debug/deps/libhttp-4526ab951518c907.rlib --extern hyper=/work/oxidecomputer/crucible/target/debug/deps/libhyper-dad943d3b7cc33e9.rlib --extern omicron_common=/work/oxidecomputer/crucible/target/debug/deps/libomicron_common-52df1fff8b36d94c.rlib --extern openapi_lint=/work/oxidecomputer/crucible/target/debug/deps/libopenapi_lint-848bb99097a9a843.rlib --extern openapiv3=/work/oxidecomputer/crucible/target/debug/deps/libopenapiv3-f84072cef6d0d68c.rlib --extern reqwest=/work/oxidecomputer/crucible/target/debug/deps/libreqwest-6407fc4e9374ca8e.rlib --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rlib --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rlib --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rlib --extern sha2=/work/oxidecomputer/crucible/target/debug/deps/libsha2-9b09b4b286e2cb62.rlib --extern slog=/work/oxidecomputer/crucible/target/debug/deps/libslog-84fd25666c3c26ee.rlib --extern subprocess=/work/oxidecomputer/crucible/target/debug/deps/libsubprocess-0acfc5c9b903588a.rlib --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rlib --extern uuid=/work/oxidecomputer/crucible/target/debug/deps/libuuid-7cc8e87b4149b49e.rlib -L native=/usr/lib/amd64 -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out -L native=/usr/ssl-3/lib/amd64`
   Compiling cmon v0.1.0 (/work/oxidecomputer/crucible/cmon)
     Running `rustc --crate-name cmon --edition=2021 cmon/src/main.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --emit=dep-info,link -C embed-bitcode=no -C debuginfo=2 --test -C metadata=2a9909624d24c98d -C extra-filename=-2a9909624d24c98d --out-dir /work/oxidecomputer/crucible/target/debug/deps -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern clap=/work/oxidecomputer/crucible/target/debug/deps/libclap-49bc17aade028e79.rlib --extern clearscreen=/work/oxidecomputer/crucible/target/debug/deps/libclearscreen-5e923be7ef236a41.rlib --extern crucible=/work/oxidecomputer/crucible/target/debug/deps/libcrucible-41ca439abdc23695.rlib --extern crucible_control_client=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_control_client-3d0142c7d3790e17.rlib --extern crucible_protocol=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_protocol-0682e169c907a102.rlib --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rlib --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rlib -L native=/usr/lib/amd64 -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out -L native=/usr/ssl-3/lib/amd64`
     Running `rustc --crate-name crucible_downstairs --edition=2021 downstairs/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --emit=dep-info,link -C embed-bitcode=no -C debuginfo=2 --test --cfg 'feature="default"' -C metadata=dce67baac661a5f4 -C extra-filename=-dce67baac661a5f4 --out-dir /work/oxidecomputer/crucible/target/debug/deps -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rlib --extern bincode=/work/oxidecomputer/crucible/target/debug/deps/libbincode-bcb925e8faac86cd.rlib --extern bytes=/work/oxidecomputer/crucible/target/debug/deps/libbytes-64a8a55ef81e55dd.rlib --extern chrono=/work/oxidecomputer/crucible/target/debug/deps/libchrono-0402a5e17dd99cdf.rlib --extern clap=/work/oxidecomputer/crucible/target/debug/deps/libclap-49bc17aade028e79.rlib --extern crucible=/work/oxidecomputer/crucible/target/debug/deps/libcrucible-41ca439abdc23695.rlib --extern crucible_common=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_common-c45901e9152d33f4.rlib --extern crucible_protocol=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_protocol-0682e169c907a102.rlib --extern dropshot=/work/oxidecomputer/crucible/target/debug/deps/libdropshot-a49a4505c9c6b86f.rlib --extern expectorate=/work/oxidecomputer/crucible/target/debug/deps/libexpectorate-4e45b262baa473cc.rlib --extern futures=/work/oxidecomputer/crucible/target/debug/deps/libfutures-df761c89bfa71e54.rlib --extern futures_core=/work/oxidecomputer/crucible/target/debug/deps/libfutures_core-46c6e3a1b3966417.rlib --extern hex=/work/oxidecomputer/crucible/target/debug/deps/libhex-6531b11cb72de3e5.rlib --extern http=/work/oxidecomputer/crucible/target/debug/deps/libhttp-4526ab951518c907.rlib --extern hyper=/work/oxidecomputer/crucible/target/debug/deps/libhyper-dad943d3b7cc33e9.rlib --extern hyper_staticfile=/work/oxidecomputer/crucible/target/debug/deps/libhyper_staticfile-559b4389ef952563.rlib --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rlib --extern mime_guess=/work/oxidecomputer/crucible/target/debug/deps/libmime_guess-66974d6c31968dc2.rlib --extern nix=/work/oxidecomputer/crucible/target/debug/deps/libnix-3635aff9412bf811.rlib --extern omicron_common=/work/oxidecomputer/crucible/target/debug/deps/libomicron_common-52df1fff8b36d94c.rlib --extern openapi_lint=/work/oxidecomputer/crucible/target/debug/deps/libopenapi_lint-848bb99097a9a843.rlib --extern openapiv3=/work/oxidecomputer/crucible/target/debug/deps/libopenapiv3-f84072cef6d0d68c.rlib --extern opentelemetry=/work/oxidecomputer/crucible/target/debug/deps/libopentelemetry-5524fd7817ad57fb.rlib --extern opentelemetry_jaeger=/work/oxidecomputer/crucible/target/debug/deps/libopentelemetry_jaeger-9ebf47742e5e063f.rlib --extern oximeter=/work/oxidecomputer/crucible/target/debug/deps/liboximeter-90ae047d6b643e4e.rlib --extern oximeter_producer=/work/oxidecomputer/crucible/target/debug/deps/liboximeter_producer-5dc4f732e258486e.rlib --extern rand=/work/oxidecomputer/crucible/target/debug/deps/librand-1f91a9ea4aed49ee.rlib --extern rand_chacha=/work/oxidecomputer/crucible/target/debug/deps/librand_chacha-add466c063ef8725.rlib --extern repair_client=/work/oxidecomputer/crucible/target/debug/deps/librepair_client-1452d56087b6ccb7.rlib --extern reqwest=/work/oxidecomputer/crucible/target/debug/deps/libreqwest-6407fc4e9374ca8e.rlib --extern ringbuffer=/work/oxidecomputer/crucible/target/debug/deps/libringbuffer-e91d75e4694e6351.rlib --extern rusqlite=/work/oxidecomputer/crucible/target/debug/deps/librusqlite-e4d2316a88b06837.rlib --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rlib --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rlib --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rlib --extern sha2=/work/oxidecomputer/crucible/target/debug/deps/libsha2-9b09b4b286e2cb62.rlib --extern slog=/work/oxidecomputer/crucible/target/debug/deps/libslog-84fd25666c3c26ee.rlib --extern slog_async=/work/oxidecomputer/crucible/target/debug/deps/libslog_async-03e743f036119074.rlib --extern slog_dtrace=/work/oxidecomputer/crucible/target/debug/deps/libslog_dtrace-8024beacfb95325b.rlib --extern slog_term=/work/oxidecomputer/crucible/target/debug/deps/libslog_term-31c687431f6dd53c.rlib --extern statistical=/work/oxidecomputer/crucible/target/debug/deps/libstatistical-f2809857d7fe1c91.rlib --extern tempfile=/work/oxidecomputer/crucible/target/debug/deps/libtempfile-05cb6a5f7e86cb66.rlib --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rlib --extern tokio_rustls=/work/oxidecomputer/crucible/target/debug/deps/libtokio_rustls-eafe4ab74a176b7d.rlib --extern tokio_util=/work/oxidecomputer/crucible/target/debug/deps/libtokio_util-279b3765a2b5aad1.rlib --extern toml=/work/oxidecomputer/crucible/target/debug/deps/libtoml-de0eb3fcc3b95b5c.rlib --extern tracing=/work/oxidecomputer/crucible/target/debug/deps/libtracing-1de351a7f2b0ab55.rlib --extern tracing_opentelemetry=/work/oxidecomputer/crucible/target/debug/deps/libtracing_opentelemetry-8f8cfc1900c3a663.rlib --extern tracing_subscriber=/work/oxidecomputer/crucible/target/debug/deps/libtracing_subscriber-e39dae5ba339bc78.rlib --extern usdt=/work/oxidecomputer/crucible/target/debug/deps/libusdt-86bb76e3b8fcea87.rlib --extern uuid=/work/oxidecomputer/crucible/target/debug/deps/libuuid-7cc8e87b4149b49e.rlib -L native=/usr/lib/amd64 -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out -L native=/usr/ssl-3/lib/amd64`
   Compiling crudd v0.1.0 (/work/oxidecomputer/crucible/crudd)
     Running `rustc --crate-name crudd --edition=2021 crudd/src/main.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --emit=dep-info,link -C embed-bitcode=no -C debuginfo=2 --test -C metadata=3e9b00990c25260e -C extra-filename=-3e9b00990c25260e --out-dir /work/oxidecomputer/crucible/target/debug/deps -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rlib --extern bytes=/work/oxidecomputer/crucible/target/debug/deps/libbytes-64a8a55ef81e55dd.rlib --extern clap=/work/oxidecomputer/crucible/target/debug/deps/libclap-49bc17aade028e79.rlib --extern crucible=/work/oxidecomputer/crucible/target/debug/deps/libcrucible-41ca439abdc23695.rlib --extern crucible_common=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_common-c45901e9152d33f4.rlib --extern crucible_protocol=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_protocol-0682e169c907a102.rlib --extern futures=/work/oxidecomputer/crucible/target/debug/deps/libfutures-df761c89bfa71e54.rlib --extern futures_core=/work/oxidecomputer/crucible/target/debug/deps/libfutures_core-46c6e3a1b3966417.rlib --extern ringbuffer=/work/oxidecomputer/crucible/target/debug/deps/libringbuffer-e91d75e4694e6351.rlib --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rlib --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rlib --extern signal_hook=/work/oxidecomputer/crucible/target/debug/deps/libsignal_hook-ae206b38743b6815.rlib --extern signal_hook_tokio=/work/oxidecomputer/crucible/target/debug/deps/libsignal_hook_tokio-6a6b104c61918fa0.rlib --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rlib --extern tokio_util=/work/oxidecomputer/crucible/target/debug/deps/libtokio_util-279b3765a2b5aad1.rlib --extern toml=/work/oxidecomputer/crucible/target/debug/deps/libtoml-de0eb3fcc3b95b5c.rlib -L native=/usr/lib/amd64 -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out -L native=/usr/ssl-3/lib/amd64`
   Compiling crucible-hammer v0.1.0 (/work/oxidecomputer/crucible/hammer)
     Running `rustc --crate-name crucible_hammer --edition=2021 hammer/src/main.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --emit=dep-info,link -C embed-bitcode=no -C debuginfo=2 --test -C metadata=9622fb9be260fb45 -C extra-filename=-9622fb9be260fb45 --out-dir /work/oxidecomputer/crucible/target/debug/deps -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rlib --extern bytes=/work/oxidecomputer/crucible/target/debug/deps/libbytes-64a8a55ef81e55dd.rlib --extern clap=/work/oxidecomputer/crucible/target/debug/deps/libclap-49bc17aade028e79.rlib --extern crucible=/work/oxidecomputer/crucible/target/debug/deps/libcrucible-41ca439abdc23695.rlib --extern crucible_common=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_common-c45901e9152d33f4.rlib --extern opentelemetry=/work/oxidecomputer/crucible/target/debug/deps/libopentelemetry-5524fd7817ad57fb.rlib --extern opentelemetry_jaeger=/work/oxidecomputer/crucible/target/debug/deps/libopentelemetry_jaeger-9ebf47742e5e063f.rlib --extern rand=/work/oxidecomputer/crucible/target/debug/deps/librand-1f91a9ea4aed49ee.rlib --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rlib --extern tracing_opentelemetry=/work/oxidecomputer/crucible/target/debug/deps/libtracing_opentelemetry-8f8cfc1900c3a663.rlib --extern tracing_subscriber=/work/oxidecomputer/crucible/target/debug/deps/libtracing_subscriber-e39dae5ba339bc78.rlib --extern uuid=/work/oxidecomputer/crucible/target/debug/deps/libuuid-7cc8e87b4149b49e.rlib -L native=/usr/lib/amd64 -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out -L native=/usr/ssl-3/lib/amd64`
     Running `rustc --crate-name crucible_integration_tests --edition=2021 integration_tests/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --emit=dep-info,link -C embed-bitcode=no -C debuginfo=2 --test -C metadata=8902d603847d3610 -C extra-filename=-8902d603847d3610 --out-dir /work/oxidecomputer/crucible/target/debug/deps -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rlib --extern base64=/work/oxidecomputer/crucible/target/debug/deps/libbase64-c139bdd129e780ac.rlib --extern bytes=/work/oxidecomputer/crucible/target/debug/deps/libbytes-64a8a55ef81e55dd.rlib --extern crucible=/work/oxidecomputer/crucible/target/debug/deps/libcrucible-41ca439abdc23695.rlib --extern crucible_client_types=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_client_types-bd54c4335d2370bd.rlib --extern crucible_downstairs=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_downstairs-6276be71be5284a4.rlib --extern crucible_pantry=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_pantry-fe633af5059fe3a7.rlib --extern crucible_pantry_client=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_pantry_client-ccb9ddeebb23cea2.rlib --extern futures=/work/oxidecomputer/crucible/target/debug/deps/libfutures-df761c89bfa71e54.rlib --extern futures_core=/work/oxidecomputer/crucible/target/debug/deps/libfutures_core-46c6e3a1b3966417.rlib --extern hex=/work/oxidecomputer/crucible/target/debug/deps/libhex-6531b11cb72de3e5.rlib --extern httptest=/work/oxidecomputer/crucible/target/debug/deps/libhttptest-174da737d96e2af6.rlib --extern rand=/work/oxidecomputer/crucible/target/debug/deps/librand-1f91a9ea4aed49ee.rlib --extern reqwest=/work/oxidecomputer/crucible/target/debug/deps/libreqwest-6407fc4e9374ca8e.rlib --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rlib --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rlib --extern sha2=/work/oxidecomputer/crucible/target/debug/deps/libsha2-9b09b4b286e2cb62.rlib --extern slog=/work/oxidecomputer/crucible/target/debug/deps/libslog-84fd25666c3c26ee.rlib --extern slog_async=/work/oxidecomputer/crucible/target/debug/deps/libslog_async-03e743f036119074.rlib --extern slog_dtrace=/work/oxidecomputer/crucible/target/debug/deps/libslog_dtrace-8024beacfb95325b.rlib --extern slog_term=/work/oxidecomputer/crucible/target/debug/deps/libslog_term-31c687431f6dd53c.rlib --extern tempfile=/work/oxidecomputer/crucible/target/debug/deps/libtempfile-05cb6a5f7e86cb66.rlib --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rlib --extern uuid=/work/oxidecomputer/crucible/target/debug/deps/libuuid-7cc8e87b4149b49e.rlib -L native=/usr/lib/amd64 -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out -L native=/usr/ssl-3/lib/amd64`
     Running `rustc --crate-name crucible_downstairs --edition=2021 downstairs/src/main.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --emit=dep-info,link -C embed-bitcode=no -C debuginfo=2 --test --cfg 'feature="default"' -C metadata=3ed9735920c1592d -C extra-filename=-3ed9735920c1592d --out-dir /work/oxidecomputer/crucible/target/debug/deps -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rlib --extern bincode=/work/oxidecomputer/crucible/target/debug/deps/libbincode-bcb925e8faac86cd.rlib --extern bytes=/work/oxidecomputer/crucible/target/debug/deps/libbytes-64a8a55ef81e55dd.rlib --extern chrono=/work/oxidecomputer/crucible/target/debug/deps/libchrono-0402a5e17dd99cdf.rlib --extern clap=/work/oxidecomputer/crucible/target/debug/deps/libclap-49bc17aade028e79.rlib --extern crucible=/work/oxidecomputer/crucible/target/debug/deps/libcrucible-41ca439abdc23695.rlib --extern crucible_common=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_common-c45901e9152d33f4.rlib --extern crucible_downstairs=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_downstairs-6276be71be5284a4.rlib --extern crucible_protocol=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_protocol-0682e169c907a102.rlib --extern dropshot=/work/oxidecomputer/crucible/target/debug/deps/libdropshot-a49a4505c9c6b86f.rlib --extern expectorate=/work/oxidecomputer/crucible/target/debug/deps/libexpectorate-4e45b262baa473cc.rlib --extern futures=/work/oxidecomputer/crucible/target/debug/deps/libfutures-df761c89bfa71e54.rlib --extern futures_core=/work/oxidecomputer/crucible/target/debug/deps/libfutures_core-46c6e3a1b3966417.rlib --extern hex=/work/oxidecomputer/crucible/target/debug/deps/libhex-6531b11cb72de3e5.rlib --extern http=/work/oxidecomputer/crucible/target/debug/deps/libhttp-4526ab951518c907.rlib --extern hyper=/work/oxidecomputer/crucible/target/debug/deps/libhyper-dad943d3b7cc33e9.rlib --extern hyper_staticfile=/work/oxidecomputer/crucible/target/debug/deps/libhyper_staticfile-559b4389ef952563.rlib --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rlib --extern mime_guess=/work/oxidecomputer/crucible/target/debug/deps/libmime_guess-66974d6c31968dc2.rlib --extern nix=/work/oxidecomputer/crucible/target/debug/deps/libnix-3635aff9412bf811.rlib --extern omicron_common=/work/oxidecomputer/crucible/target/debug/deps/libomicron_common-52df1fff8b36d94c.rlib --extern openapi_lint=/work/oxidecomputer/crucible/target/debug/deps/libopenapi_lint-848bb99097a9a843.rlib --extern openapiv3=/work/oxidecomputer/crucible/target/debug/deps/libopenapiv3-f84072cef6d0d68c.rlib --extern opentelemetry=/work/oxidecomputer/crucible/target/debug/deps/libopentelemetry-5524fd7817ad57fb.rlib --extern opentelemetry_jaeger=/work/oxidecomputer/crucible/target/debug/deps/libopentelemetry_jaeger-9ebf47742e5e063f.rlib --extern oximeter=/work/oxidecomputer/crucible/target/debug/deps/liboximeter-90ae047d6b643e4e.rlib --extern oximeter_producer=/work/oxidecomputer/crucible/target/debug/deps/liboximeter_producer-5dc4f732e258486e.rlib --extern rand=/work/oxidecomputer/crucible/target/debug/deps/librand-1f91a9ea4aed49ee.rlib --extern rand_chacha=/work/oxidecomputer/crucible/target/debug/deps/librand_chacha-add466c063ef8725.rlib --extern repair_client=/work/oxidecomputer/crucible/target/debug/deps/librepair_client-1452d56087b6ccb7.rlib --extern reqwest=/work/oxidecomputer/crucible/target/debug/deps/libreqwest-6407fc4e9374ca8e.rlib --extern ringbuffer=/work/oxidecomputer/crucible/target/debug/deps/libringbuffer-e91d75e4694e6351.rlib --extern rusqlite=/work/oxidecomputer/crucible/target/debug/deps/librusqlite-e4d2316a88b06837.rlib --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rlib --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rlib --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rlib --extern sha2=/work/oxidecomputer/crucible/target/debug/deps/libsha2-9b09b4b286e2cb62.rlib --extern slog=/work/oxidecomputer/crucible/target/debug/deps/libslog-84fd25666c3c26ee.rlib --extern slog_async=/work/oxidecomputer/crucible/target/debug/deps/libslog_async-03e743f036119074.rlib --extern slog_dtrace=/work/oxidecomputer/crucible/target/debug/deps/libslog_dtrace-8024beacfb95325b.rlib --extern slog_term=/work/oxidecomputer/crucible/target/debug/deps/libslog_term-31c687431f6dd53c.rlib --extern statistical=/work/oxidecomputer/crucible/target/debug/deps/libstatistical-f2809857d7fe1c91.rlib --extern tempfile=/work/oxidecomputer/crucible/target/debug/deps/libtempfile-05cb6a5f7e86cb66.rlib --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rlib --extern tokio_rustls=/work/oxidecomputer/crucible/target/debug/deps/libtokio_rustls-eafe4ab74a176b7d.rlib --extern tokio_util=/work/oxidecomputer/crucible/target/debug/deps/libtokio_util-279b3765a2b5aad1.rlib --extern toml=/work/oxidecomputer/crucible/target/debug/deps/libtoml-de0eb3fcc3b95b5c.rlib --extern tracing=/work/oxidecomputer/crucible/target/debug/deps/libtracing-1de351a7f2b0ab55.rlib --extern tracing_opentelemetry=/work/oxidecomputer/crucible/target/debug/deps/libtracing_opentelemetry-8f8cfc1900c3a663.rlib --extern tracing_subscriber=/work/oxidecomputer/crucible/target/debug/deps/libtracing_subscriber-e39dae5ba339bc78.rlib --extern usdt=/work/oxidecomputer/crucible/target/debug/deps/libusdt-86bb76e3b8fcea87.rlib --extern uuid=/work/oxidecomputer/crucible/target/debug/deps/libuuid-7cc8e87b4149b49e.rlib -L native=/usr/lib/amd64 -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out -L native=/usr/ssl-3/lib/amd64`
    Finished test [unoptimized + debuginfo] target(s) in 5m 19s
     Running `/work/oxidecomputer/crucible/target/debug/deps/cmon-2a9909624d24c98d --nocapture`

running 0 tests

test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s

     Running `/work/oxidecomputer/crucible/target/debug/deps/crucible-53d074fabbf363e8 --nocapture`

running 351 tests
test block_req::test::test_blockreqwaiter_send ... ok
test block_req::test::test_blockreq_and_blockreqwaiter_err ... ok
test block_req::test::test_blockreq_and_blockreqwaiter ... ok
{{{""msg"msg":"Upstairs starts":"Upstairs starts",,""v"v"::00,","name":name"":"crucible"crucible","level",:"level":3030{"msg":"Upstairs starts","v":0,"name":"crucible","level":30"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.417161849Z","hostname":","ip-10-150-1-74.us-west-2.compute.internal"time,"":pid"":42912023-09-22T23:08:02.417107971Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,",upstairs"":upstairs":11}
,"time":"2023-09-22T23:08:02.417217374Z"{,"hostname":"ip-10-150-1-74.us-west-2.compute.internal",""pid":msg"4291:","upstairs"Crucible Version: BuildInfo {:\n1    version: }\"
0.0.1\",\n    git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",{\n    git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n    git_branch: \"main\",\n    rustc_semver: \"1.70.0\",\n    rustc_channel: \"stable\",\n    rustc_host_triple: \"x86_64-unknown-illumos\",\n    rustc_commit_sha: \""msg":"90c541806f23a127002de5b4038be731ba1458ca\",\nCrucible Version: BuildInfo {    cargo_triple: \n\"    version: \"x86_64-unknown-illumos\"0.0.1\",,\n\n    git_sha: \"    debug: true,\n    opt_level: 0,\ned48f294784d46ea7d4bb99336918b74358eca46}"\",\n    git_commit_timestamp: \","v"2023-09-22T22:51:18.000000000Z:\",\n0    git_branch: \",main\"",\nname":"    rustc_semver: \"crucible"1.70.0\",,\n"level    rustc_channel: \""stable\":,\n30    rustc_host_triple: \"x86_64-unknown-illumos\",\n    rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n    cargo_triple: \"x86_64-unknown-illumos\",\n    debug: true,\n    opt_level: 0,\n}","v":0,"name":"crucible","level",:"30time":"2023-09-22T23:08:02.417922851Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
,"time":"2023-09-22T23:08:02.417944281Z",{"hostname":""msgip-10-150-1-74.us-west-2.compute.internal"",:""pid":4291Upstairs <-> Downstairs Message Version: 4",","upstairsv""::01,"}name
":"crucible","level":30{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.417988891Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
,"time":"{2023-09-22T23:08:02.418002831Z","hostname":""msg":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"Crucible stats registered with UUID: 024c3783-b9b4-4453-823f-769dad90d4f2"upstairs":,1"v}"
:0,"name":"{crucible","level":30"msg":"Crucible stats registered with UUID: ed581aed-4fb6-4c12-84d1-a857283bcda2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.418041683Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,",time":""upstairs":12023-09-22T23:08:02.418051531Z"}
,"hostname":"{ip-10-150-1-74.us-west-2.compute.internal","pid":4291",msg"":"upstairs":1}
Crucible 024c3783-b9b4-4453-823f-769dad90d4f2 has session id: 3ce64a0f-e985-43b7-a27a-96f3099e6cf4","v":0{,"name":"crucible",""level"msg:"30:"Crucible ed581aed-4fb6-4c12-84d1-a857283bcda2 has session id: 9bfc6996-1929-4f26-805c-da7e600d5990","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.418095651Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,",upstairs"":time"1:"}
2023-09-22T23:08:02.418105923Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
}
{"msg":"Crucible Version: BuildInfo {\n    version: \"0.0.1\",\n    git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n    git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n    git_branch: \"main\",\n    rustc_semver: \"1.70.0\",\n    rustc_channel: \"stable\",\n    rustc_host_triple: \"x86_64-unknown-illumos\",\n    rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n    cargo_triple: \"x86_64-unknown-illumos\",\n    debug: true,\n    opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.41820299Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.418248767Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"Crucible stats registered with UUID: df6648fa-e480-4dd9-b7c4-7fec45dbddf8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.418287638Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"Crucible df6648fa-e480-4dd9-b7c4-7fec45dbddf8 has session id: c2bd09c5-6ed6-4145-ba72-617a46286f0f","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.418333415Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
,"time":"2023-09-22T23:08:02.417100875Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"Crucible Version: BuildInfo {\n    version: \"0.0.1\",\n    git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n    git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n    git_branch: \"main\",\n    rustc_semver: \"1.70.0\",\n    rustc_channel: \"stable\",\n    rustc_host_triple: \"x86_64-unknown-illumos\",\n    rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n    cargo_triple: \"x86_64-unknown-illumos\",\n    debug: true,\n    opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.418472491Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30{"msg":"[0] connecting to 127.0.0.1:52643","v":,0",time":""name":"crucible2023-09-22T23:08:02.418514768Z"",","hostname"level"::"30ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"Crucible stats registered with UUID: a2b1abbb-2148-4d24-93b2-2a0a303e7718","v":0,"name":"crucible","level",:"30time":"2023-09-22T23:08:02.418536976Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,","time":"2023-09-22T23:08:02.418559377Z","hostname":"looper"ip-10-150-1-74.us-west-2.compute.internal":,""0pid"":4291,","upstairs":upstairs":11}}

{"msg":"[0] connecting to 127.0.0.1:60909","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.418914004Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"looper":"0","upstairs":1}
{"msg":"[1] connecting to 127.0.0.1:41466","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.418972893Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"looper":"1","upstairs":1}
{"msg":"[2] connecting to 127.0.0.1:55647","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.419020958Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"looper":"2","upstairs":1}
{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.419061307Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1,"task":"up_listen"}
{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.419104593Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"Flush timeout: 86400","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.419145458Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] connecting to 127.0.0.1:39941","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.419202392Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"looper":"0","upstairs":1}
{"msg":"[1] connecting to 127.0.0.1:51919","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.41925921Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"looper":"1","upstairs":1}
{"msg":"[2] connecting to 127.0.0.1:45755","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.419304485Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"looper":"2","upstairs":1}
{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.419346892Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1,"task":"up_listen"}
{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.419392108Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"Flush timeout: 86400","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.419432132Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[1] connecting to 127.0.0.1:34619","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.419598558Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"looper":"1","upstairs":1}
{"msg":"[2] connecting to 127.0.0.1:44064","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.419655847Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"looper":"2","upstairs":1}
{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.419701509Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1,"task":"up_listen"}
{"msg":"Wait for all three downstairs to come online","v":0{,"name":"crucible","level":30"msg":"Crucible a2b1abbb-2148-4d24-93b2-2a0a303e7718 has session id: 8079de44-89af-4e82-ab2d-095962b2363a","v":0,"name":"crucible","level":,"30time":"2023-09-22T23:08:02.419751463Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":","Flush timeout: 86400"time",:""v":02023-09-22T23:08:02.419767569Z","name,":""crucible"hostname":,""level":30ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
,"time{":"2023-09-22T23:08:02.419798287Z"",msg":""hostname":"[0] connecting to 127.0.0.1:61368","ip-10-150-1-74.us-west-2.compute.internal"v,"":pid":04291,"name",:""crucible"upstairs,"":level1":}30
,"time":"2023-09-22T23:08:02.419834637Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"looper":"0","upstairs":1}
{"msg":"[1] connecting to 127.0.0.1:50978","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.419880703Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"looper":"1","upstairs":1}
{"msg{":"[2] connecting to 127.0.0.1:57228"","vmsg""::"0,"name":"crucible"[0] ed581aed-4fb6-4c12-84d1-a857283bcda2 looper connected","level",:"v30":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.419925649Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal,"","time"pid:"":4291,2023-09-22T23:08:02.419930312Z""looper",:""2"hostname",:""upstairs":1ip-10-150-1-74.us-west-2.compute.internal}"
,"pid":4291{,"looper":""0"msg":,""upstairs"up_listen starts:"1,"v"}:
0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.41996917Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1,"task":"up_listen"}
{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.420005751Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":{1}
"msg":"{"[0] 024c3783-b9b4-4453-823f-769dad90d4f2 looper connected"msg":"{,"Flush timeout: 86400v"":,"0"v":,0msg"",:"name""name"::""crucible[0] df6648fa-e480-4dd9-b7c4-7fec45dbddf8 looper connected""crucible",,""level"level:,30""v"::300,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.420060398Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1,}"
,time"":"time"{:"2023-09-22T23:08:02.420063804Z"",2023-09-22T23:08:02.420068874Z"msg"",:"hostname""hostname"::"[0] a2b1abbb-2148-4d24-93b2-2a0a303e7718 looper connected"","v"ip-10-150-1-74.us-west-2.compute.internal"ip-10-150-1-74.us-west-2.compute.internal:,""pid0",,""pid:"4291name:"4291:,"",looper""crucible"looper"::,""0"0,"""level",upstairs""::30upstairs"1:1}
}
,"time":"2023-09-22T23:08:02.420134198Z","{hostname":""ip-10-150-1-74.us-west-2.compute.internal"msg",:""pid":4291[0] Proc runs for 127.0.0.1:39941 in state New",",looper"":"v0"":,"0upstairs,":"1name":}"
crucible","level":30,"time":"2023-09-22T23:08:02.420175591Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] Proc runs for 127.0.0.1:52643 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.420270535Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] Proc runs for 127.0.0.1:61368 in state New","v":0,"name":"crucible","level":30{"msg":","time":"[0] Proc runs for 127.0.0.1:60909 in state New"2023-09-22T23:08:02.420330114Z",",v"":hostname":"0,"name":ip-10-150-1-74.us-west-2.compute.internal"","crucible"pid":,4291"level,"":upstairs"30:1}
,"time":"2023-09-22T23:08:02.420359995Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{{""msgmsg""::""{[1] a2b1abbb-2148-4d24-93b2-2a0a303e7718 looper connected[1] ed581aed-4fb6-4c12-84d1-a857283bcda2 looper connected""",,""vvmsg""::"0:,"{"0name",:""name":crucible""[1] df6648fa-e480-4dd9-b7c4-7fec45dbddf8 looper connectedcrucible"","",msg"levellevel"":,:3030"":v"":0,"name":"[1] 024c3783-b9b4-4453-823f-769dad90d4f2 looper connected"crucible",","level"v"::300,"name":"crucible","level":30,,""timetime""::""2023-09-22T23:08:02.420541835Z2023-09-22T23:08:02.420541832Z"",,""hostnamehostname""::""ip-10-150-1-74.us-west-2.compute.internalip-10-150-1-74.us-west-2.compute.internal"",,""pid"pid:":42914291,",,""timelooperlooper""::""1,"1,"""",upstairs"":upstairs:"time1:"":}1
}2023-09-22T23:08:02.420552781Z""
,2023-09-22T23:08:02.420557696Z""hostname,"":"hostname":"ip-10-150-1-74.us-west-2.compute.internal",ip-10-150-1-74.us-west-2.compute.internal""pid",:"4291pid":4291,"looper,"":"looper1"":,""1"upstairs":,"1upstairs":}1
}
{"msg":"[1] Proc runs for 127.0.0.1:51919 in state New","v":0,"name":"crucible","level":30{,""time":"msg":"2023-09-22T23:08:02.420642014Z","[1] Proc runs for 127.0.0.1:41466 in state New"hostname":","v":0ip-10-150-1-74.us-west-2.compute.internal",,""name"pid:":"4291crucible",,""level":upstairs"30:1}
{"msg":"[2] df6648fa-e480-4dd9-b7c4-7fec45dbddf8 looper connected","v":0,"name":"crucible",",level"":time"30:"2023-09-22T23:08:02.420675248Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
,"time":"2023-09-22T23:08:02.420695691Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"looper":"2","upstairs":1}
{"msg":"[2] Proc runs for 127.0.0.1:45755 in state New","v":0,"name":"crucible","level":{30"msg":"[2] ed581aed-4fb6-4c12-84d1-a857283bcda2 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.420743127Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
,"time":"2023-09-22T23:08:02.420756491Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"looper":"2","upstairs":1}
{"msg":"[2] Proc runs for 127.0.0.1:55647 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.420806756Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}The guest has requested activation

{"msg":"[1] Proc runs for 127.0.0.1:34619 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.420980607Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[2] 024c3783-b9b4-4453-823f-769dad90d4f2 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.421031408Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"looper":"2","upstairs":1}
{"msg":"[2] Proc runs for 127.0.0.1:44064 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.421077083Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
The guest has requested activation
{"msg":"[1] Proc runs for 127.0.0.1:50978 in state New","v":0,"name":"crucible","level":30The guest has requested activation
,"time":"2023-09-22T23:08:02.421160928Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[2] a2b1abbb-2148-4d24-93b2-2a0a303e7718 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.421207343Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"looper":"2","upstairs":1}
{"msg":"[2] Proc runs for 127.0.0.1:57228 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.421253366Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"a2b1abbb-2148-4d24-93b2-2a0a303e7718 active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.421294916Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{{"msg"":"msg":"024c3783-b9b4-4453-823f-769dad90d4f2 active request set"df6648fa-e480-4dd9-b7c4-7fec45dbddf8 active request set","v",:"v0",:"name":0","crucible"name":","crucible"level":,"30level":30The guest has requested activation
,"time":,""time":"2023-09-22T23:08:02.421365898Z","2023-09-22T23:08:02.421367674Z"hostname":","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":ip-10-150-1-74.us-west-2.compute.internal"4291,"pid":,"4291upstairs":,1"}upstairs
":1}
{"msg":"ed581aed-4fb6-4c12-84d1-a857283bcda2 active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.421478924Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"negotiate packet HereIAm { version: 4, upstairs_id: df6648fa-e480-4dd9-b7c4-7fec45dbddf8, session_id: b793572f-19dc-423b-877e-0f0cfe9147d9, gen: 1, read_only: false, encrypted: false, alternate_versions: [] }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.421574296Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"negotiate packet HereIAm { version: 4, upstairs_id: ed581aed-4fb6-4c12-84d1-a857283bcda2, session_id: 99a7dfa5-d205-4d07-9135-9f9e70387e3a, gen: 1, read_only: false, encrypted: false, alternate_versions: [] }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.421646375Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"negotiate packet HereIAm { version: 4, upstairs_id: ed581aed-4fb6-4c12-84d1-a857283bcda2, session_id: 99a7dfa5-d205-4d07-9135-9f9e70387e3a, gen: 1, read_only: false, encrypted: false, alternate_versions: [] }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.421698437Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":2}
{"msg":"negotiate packet HereIAm { version: 4, upstairs_id: ed581aed-4fb6-4c12-84d1-a857283bcda2, session_id: 99a7dfa5-d205-4d07-9135-9f9e70387e3a, gen: 1, read_only: false, encrypted: false, alternate_versions: [] }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.421774494Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":3}
{"msg":"negotiate packet HereIAm { version: 4, upstairs_id: df6648fa-e480-4dd9-b7c4-7fec45dbddf8, session_id: b793572f-19dc-423b-877e-0f0cfe9147d9, gen: 1, read_only: false, encrypted: false, alternate_versions: [] }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.421862204Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":2}
{"msg":"negotiate packet HereIAm { version: 4, upstairs_id: df6648fa-e480-4dd9-b7c4-7fec45dbddf8, session_id: b793572f-19dc-423b-877e-0f0cfe9147d9, gen: 1, read_only: false, encrypted: false, alternate_versions: [] }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.421913689Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":3}
{"msg":"[0] df6648fa-e480-4dd9-b7c4-7fec45dbddf8 (b793572f-19dc-423b-877e-0f0cfe9147d9) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.421960012Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.422000253Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] client is_active_req TRUE, promote! session b793572f-19dc-423b-877e-0f0cfe9147d9","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.422042578Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[1] df6648fa-e480-4dd9-b7c4-7fec45dbddf8 (b793572f-19dc-423b-877e-0f0cfe9147d9) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.422086884Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.422134075Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[1] client is_active_req TRUE, promote! session b793572f-19dc-423b-877e-0f0cfe9147d9","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.422178104Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[2] df6648fa-e480-4dd9-b7c4-7fec45dbddf8 (b793572f-19dc-423b-877e-0f0cfe9147d9) WaitActive WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.422222121Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.422262782Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[2] client is_active_req TRUE, promote! session b793572f-19dc-423b-877e-0f0cfe9147d9","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.422304767Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"negotiate packet PromoteToActive { upstairs_id: df6648fa-e480-4dd9-b7c4-7fec45dbddf8, session_id: b793572f-19dc-423b-877e-0f0cfe9147d9, gen: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.422349358Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"negotiate packet PromoteToActive { upstairs_id: df6648fa-e480-4dd9-b7c4-7fec45dbddf8, session_id: b793572f-19dc-423b-877e-0f0cfe9147d9, gen: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.42239377Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":2}
{"msg":"negotiate packet PromoteToActive { upstairs_id: df6648fa-e480-4dd9-b7c4-7fec45dbddf8, session_id: b793572f-19dc-423b-877e-0f0cfe9147d9, gen: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.422438018Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":3}
{"msg":"[0] ed581aed-4fb6-4c12-84d1-a857283bcda2 (99a7dfa5-d205-4d07-9135-9f9e70387e3a) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.422502549Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.422545681Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] client is_active_req TRUE, promote! session 99a7dfa5-d205-4d07-9135-9f9e70387e3a","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.422586567Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[1] ed581aed-4fb6-4c12-84d1-a857283bcda2 (99a7dfa5-d205-4d07-9135-9f9e70387e3a) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.422630652Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.42267082Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[1] client is_active_req TRUE, promote! session 99a7dfa5-d205-4d07-9135-9f9e70387e3a","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.422712518Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[2] ed581aed-4fb6-4c12-84d1-a857283bcda2 (99a7dfa5-d205-4d07-9135-9f9e70387e3a) WaitActive WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.422756376Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.422796138Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[2] client is_active_req TRUE, promote! session 99a7dfa5-d205-4d07-9135-9f9e70387e3a","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.422845979Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"negotiate packet PromoteToActive { upstairs_id: ed581aed-4fb6-4c12-84d1-a857283bcda2, session_id: 99a7dfa5-d205-4d07-9135-9f9e70387e3a, gen: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.422892707Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"negotiate packet PromoteToActive { upstairs_id: ed581aed-4fb6-4c12-84d1-a857283bcda2, session_id: 99a7dfa5-d205-4d07-9135-9f9e70387e3a, gen: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.422937644Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":2}
{"msg":"negotiate packet PromoteToActive { upstairs_id: ed581aed-4fb6-4c12-84d1-a857283bcda2, session_id: 99a7dfa5-d205-4d07-9135-9f9e70387e3a, gen: 1 }","v":0,"name":"crucible","level":30{","timemsg":"":"2023-09-22T23:08:02.42298288Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs"negotiate packet HereIAm { version: 4, upstairs_id: 024c3783-b9b4-4453-823f-769dad90d4f2, session_id: db076280-2529-45a1-a093-a3cd3f0799dc, gen: 1, read_only: false, encrypted: false, alternate_versions: [] }":3},
"v":0,"{name":"crucible"","msg"level":":30negotiate packet RegionInfoPlease","v":0,"name":"crucible","level":30,"time":","2023-09-22T23:08:02.423029769Z"time":,""hostname":"2023-09-22T23:08:02.42302412Z","ip-10-150-1-74.us-west-2.compute.internalhostname"":,""pid":4291ip-10-150-1-74.us-west-2.compute.internal",","downstairspid""::42911}
,"downstairs":1{}
"msg":"negotiate packet RegionInfoPlease","v{":0,"name":""crucible"msg",:""level":30negotiate packet HereIAm { version: 4, upstairs_id: 024c3783-b9b4-4453-823f-769dad90d4f2, session_id: db076280-2529-45a1-a093-a3cd3f0799dc, gen: 1, read_only: false, encrypted: false, alternate_versions: [] }","v":0,"name":"crucible,""time",:""level":302023-09-22T23:08:02.423085422Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":2}
{,""time"msg:"":"2023-09-22T23:08:02.42310327Z"negotiate packet RegionInfoPlease",","hostname"v"::"0,"name":"ip-10-150-1-74.us-west-2.compute.internal"crucible",,""pid":level"4291:30,"downstairs":2}
{"msg":,""time":"2023-09-22T23:08:02.423135861Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291negotiate packet HereIAm { version: 4, upstairs_id: 024c3783-b9b4-4453-823f-769dad90d4f2, session_id: db076280-2529-45a1-a093-a3cd3f0799dc, gen: 1, read_only: false, encrypted: false, alternate_versions: [] },"","downstairs"v"::30},
"name":"crucible"{,"level":30"msg":"[0] downstairs client at 127.0.0.1:60909 has UUID 761a4173-d6d6-4e0b-98cb-d77e4463419a","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.423175522Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,,""time":downstairs"":3}2023-09-22T23:08:02.423186888Z
","hostname":"{ip-10-150-1-74.us-west-2.compute.internal","pid":"4291msg":","upstairs":1}
[0] 024c3783-b9b4-4453-823f-769dad90d4f2 (db076280-2529-45a1-a093-a3cd3f0799dc) New New New ds_transition to WaitActive","v":0{,"name":"crucible"","msg":level":"30[0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 10, shift: 9 }, extent_count: 10, uuid: 761a4173-d6d6-4e0b-98cb-d77e4463419a, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible",,""level"time"::30"2023-09-22T23:08:02.423231359Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
,"time":"{2023-09-22T23:08:02.423250072Z",""hostname":"msg":"ip-10-150-1-74.us-west-2.compute.internal"[0] Transition from New to WaitActive,""pid",:"4291v":,0","upstairs"name"::"1crucible"}
,"level":30,"time":"2023-09-22T23:08:02.423289472Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] client is_active_req TRUE, promote! session db076280-2529-45a1-a093-a3cd3f0799dc"{,"v":0,"name":""crucible"msg",:""level":30negotiate packet RegionInfoPlease","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.423336014Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal",,""pid":time4291":","upstairs":2023-09-22T23:08:02.423342762Z1"}
,"hostname":"{ip-10-150-1-74.us-west-2.compute.internal","pid":4291"msg,"":"downstairs":1}
[1] 024c3783-b9b4-4453-823f-769dad90d4f2 (db076280-2529-45a1-a093-a3cd3f0799dc) WaitActive New New ds_transition to WaitActive","v":0,{"name":"crucible"",msg":""level":30negotiate packet RegionInfoPlease","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.423397859Z","hostname":","time":"ip-10-150-1-74.us-west-2.compute.internal","2023-09-22T23:08:02.42340416Z"pid":,"4291hostname":","upstairs":1ip-10-150-1-74.us-west-2.compute.internal"},"
pid":4291,"downstairs{":2}
"msg":"{[1] Transition from New to WaitActive","v":"0msg",:""name":"crucible"negotiate packet RegionInfoPlease",","level":v30":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.423454785Z","hostname":","time":"ip-10-150-1-74.us-west-2.compute.internal","2023-09-22T23:08:02.423459493Z"pid":,4291"hostname":,""upstairs":1ip-10-150-1-74.us-west-2.compute.internal"},"
pid":4291,"{downstairs":3}
"msg":"{"msg":[1] client is_active_req TRUE, promote! session db076280-2529-45a1-a093-a3cd3f0799dc"","v":0,"name":"crucible"[0] downstairs client at 127.0.0.1:39941 has UUID c7df82f0-e00e-410b-ac45-3ec316aefac5",","level"v"::300,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.42351133Z",",hostname":""time":"ip-10-150-1-74.us-west-2.compute.internal",2023-09-22T23:08:02.423515401Z""pid,"":hostname":4291","upstairs":ip-10-150-1-74.us-west-2.compute.internal"1,"}pid"
:4291,"upstairs":{1}
"msg":"{"msg":"[2] 024c3783-b9b4-4453-823f-769dad90d4f2 (db076280-2529-45a1-a093-a3cd3f0799dc) WaitActive WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30[0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 10, shift: 9 }, extent_count: 10, uuid: c7df82f0-e00e-410b-ac45-3ec316aefac5, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.423569209Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":,4291"time",:""upstairs":12023-09-22T23:08:02.423577538Z"}
,"hostname":"{ip-10-150-1-74.us-west-2.compute.internal","pid":4291"msg",:""upstairs":1}[2] Transition from New to WaitActive
","v":0,"{name":"crucible"","msg"level"::"30df6648fa-e480-4dd9-b7c4-7fec45dbddf8 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.42362419Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":,"4291time":","upstairs":2023-09-22T23:08:02.42363307Z"1,}"
hostname":"{ip-10-150-1-74.us-west-2.compute.internal","pid":4291"msg",:""upstairs":1}
[2] client is_active_req TRUE, promote! session db076280-2529-45a1-a093-a3cd3f0799dc","v":{0,"name":""cruciblemsg"":,""level":30[1] downstairs client at 127.0.0.1:51919 has UUID 2a63ad8c-68c4-4a78-a731-eeb47bc60138","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.423677979Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal",","pid"time"::"4291,"2023-09-22T23:08:02.42368647Z"upstairs":,1"}hostname"
:"{ip-10-150-1-74.us-west-2.compute.internal","pid":"4291msg":","upstairs":1}
negotiate packet PromoteToActive { upstairs_id: 024c3783-b9b4-4453-823f-769dad90d4f2, session_id: db076280-2529-45a1-a093-a3cd3f0799dc, gen: 1 }","v":0{,"name":"crucible"",msg"":"level":30[1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 10, shift: 9 }, extent_count: 10, uuid: 2a63ad8c-68c4-4a78-a731-eeb47bc60138, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible",",level"":time30":"2023-09-22T23:08:02.42373233Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
,"time":"{2023-09-22T23:08:02.423749563Z",""hostname"msg"::""ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1negotiate packet PromoteToActive { upstairs_id: 024c3783-b9b4-4453-823f-769dad90d4f2, session_id: db076280-2529-45a1-a093-a3cd3f0799dc, gen: 1 }"}
,"v":0,"name":"{crucible","level":30"msg":"df6648fa-e480-4dd9-b7c4-7fec45dbddf8 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.423793905Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":,2"}time":
"2023-09-22T23:08:02.423804144Z",{"hostname":""msg":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
negotiate packet PromoteToActive { upstairs_id: 024c3783-b9b4-4453-823f-769dad90d4f2, session_id: db076280-2529-45a1-a093-a3cd3f0799dc, gen: 1 }","v":0,"{name":"crucible",""level"msg"::30"[2] downstairs client at 127.0.0.1:45755 has UUID f4127e0c-bc93-4b19-ac81-caee623ec347","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.423849045Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,","downstairs":time3":"}
2023-09-22T23:08:02.423859566Z","hostname":"{ip-10-150-1-74.us-west-2.compute.internal"","msg"pid"::"4291,"negotiate packet RegionInfoPlease"upstairs":,1"v"}:
0,"name":"crucible"{,"level":30"msg":"[2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 10, shift: 9 }, extent_count: 10, uuid: f4127e0c-bc93-4b19-ac81-caee623ec347, encrypted: false, database_read_version: 1, database_write_version: 1 }",,""time":v"":0,"2023-09-22T23:08:02.423902561Zname"":","crucible"hostname",:""level":30ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{,""time"msg"::""2023-09-22T23:08:02.423926464Z"negotiate packet RegionInfoPlease",","v"hostname"::0","name":"crucibleip-10-150-1-74.us-west-2.compute.internal"",",pid"":level4291":30,"upstairs":1}
{"msg":","df6648fa-e480-4dd9-b7c4-7fec45dbddf8 WaitActive WaitActive WaitActivetime"":","v":2023-09-22T23:08:02.423958066Z"0,,""name":"hostname":"crucible","level":ip-10-150-1-74.us-west-2.compute.internal"30,"pid":4291,"downstairs":2}
{"msg",:""time":"negotiate packet RegionInfoPlease"2023-09-22T23:08:02.423983605Z,""v",:"0,hostname"":"name":"crucible",ip-10-150-1-74.us-west-2.compute.internal"","level"pid"::304291,"upstairs":1}
{"msg":"negotiate packet ExtentVersionsPlease",,""time"v"::"0,"name"2023-09-22T23:08:02.424013241Z":","crucible"hostname":","level":30ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":3}
{,""time"msg"::""2023-09-22T23:08:02.424038688Z","hostname":"[0] downstairs client at 127.0.0.1:52643 has UUID af1eb908-dd0e-4704-a165-8346bf74d171","vip-10-150-1-74.us-west-2.compute.internal"":,0","pid"name"::4291"crucible",","downstairs"level"::130}
{"msg":"negotiate packet ExtentVersionsPlease","v":0,"name":","crucible"time",:""level":302023-09-22T23:08:02.424073582Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
,"time":"{2023-09-22T23:08:02.424091801Z",""hostname":msg"":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":2}
[0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 10, shift: 9 }, extent_count: 10, uuid: af1eb908-dd0e-4704-a165-8346bf74d171, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":{0,"name":""cruciblemsg"":","level":30negotiate packet ExtentVersionsPlease","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.424130306Z","hostname":","timeip-10-150-1-74.us-west-2.compute.internal"",:""pid":42912023-09-22T23:08:02.42413634Z",","upstairs":hostname":1"}
ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":3}
{"msg":"[0] df6648fa-e480-4dd9-b7c4-7fec45dbddf8 (b793572f-19dc-423b-877e-0f0cfe9147d9) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.424189843Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.424229856Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:02.424276564Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.424317166Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[1] df6648fa-e480-4dd9-b7c4-7fec45dbddf8 (b793572f-19dc-423b-877e-0f0cfe9147d9) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.424360894Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.424401449Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:02.424441491Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.424480471Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[2] df6648fa-e480-4dd9-b7c4-7fec45dbddf8 (b793572f-19dc-423b-877e-0f0cfe9147d9) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.424524263Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0The guest has finished waiting for activation
,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.424571289Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:02.424610754Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.424648083Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] 127.0.0.1:39941 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.424681545Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{{"msg":""msg"df6648fa-e480-4dd9-b7c4-7fec45dbddf8 WaitQuorum WaitQuorum WaitQuorum":,""v":0,"name":"crucible","level":30negotiate packet HereIAm { version: 4, upstairs_id: a2b1abbb-2148-4d24-93b2-2a0a303e7718, session_id: a7f1d842-f19a-4c81-abcd-c1f78df01722, gen: 1, read_only: false, encrypted: false, alternate_versions: [] }","v":0,"name":"crucible","level":,"30time":"2023-09-22T23:08:02.424717941Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0]R  flush_numbers: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]",",v"":time":0","name":"2023-09-22T23:08:02.424730984Zcrucible"","level",:"30hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
,"time":"2023-09-22T23:08:02.42476087Z","hostname{":"ip-10-150-1-74.us-west-2.compute.internal",""msg"pid:":"4291,"upstairs":1}
negotiate packet HereIAm { version: 4, upstairs_id: a2b1abbb-2148-4d24-93b2-2a0a303e7718, session_id: a7f1d842-f19a-4c81-abcd-c1f78df01722, gen: 1, read_only: false, encrypted: false, alternate_versions: [] }"{,"v":0","{msg"name":":"crucible",""[0]R  generation: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]levelmsg"""::30","v":0,"name":"024c3783-b9b4-4453-823f-769dad90d4f2 WaitActive WaitActive WaitActive"crucible","level":,30"v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.424810014Z",",hostname"":"time":"ip-10-150-1-74.us-west-2.compute.internal"2023-09-22T23:08:02.424816447Z,""pid",":hostname4291":","downstairs":ip-10-150-1-74.us-west-2.compute.internal"2,"pid":}4291
,,""timeupstairs""{::"1"}msg"
2023-09-22T23:08:02.424823109Z:"","{hostname":""ip-10-150-1-74.us-west-2.compute.internal"msg":","pid":negotiate packet HereIAm { version: 4, upstairs_id: a2b1abbb-2148-4d24-93b2-2a0a303e7718, session_id: a7f1d842-f19a-4c81-abcd-c1f78df01722, gen: 1, read_only: false, encrypted: false, alternate_versions: [] }"4291[0]R  dirty: [false, false, false, false, false, false, false, false, false, false]",",v":"0,,upstairs""":v"name1"::"}0
test impacted_blocks::test::empty_contains_nothing ... {{"ok"msg"msg
:"":"crucibleed581aed-4fb6-4c12-84d1-a857283bcda2 WaitActive WaitActive WaitActive[1] downstairs client at 127.0.0.1:34619 has UUID b5944748-a762-4e0a-bbf1-fcda234e697f""","v",:,""0level",":name"30:"v"crucible":0,","level"name"::"30crucible","level":30,"time":"2023-09-22T23:08:02.424938213Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,","time":downstairs"":,3"2023-09-22T23:08:02.42494519Z"time"}:"
,"2023-09-22T23:08:02.424947725Z"hostname":","hostname"{:"ip-10-150-1-74.us-west-2.compute.internal","pid":ip-10-150-1-74.us-west-2.compute.internal"4291","msgpid"",::"4291"upstairs",:"1upstairs":}1
[0] a2b1abbb-2148-4d24-93b2-2a0a303e7718 (a7f1d842-f19a-4c81-abcd-c1f78df01722) New New New ds_transition to WaitActive"},
"{v":0,"name":""crucible"msg{,"":"level":"30msg":","[1] downstairs client at 127.0.0.1:41466 has UUID e69c25b9-b0d6-406a-850b-6b5dba8b6e69"name",:""v"crucible[1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 10, shift: 9 }, extent_count: 10, uuid: b5944748-a762-4e0a-bbf1-fcda234e697f, encrypted: false, database_read_version: 1, database_write_version: 1 }"":,,""0vlevel""::,030,,""name"time"name"::"""crucible:"2023-09-22T23:08:02.425012404Z""crucible,",",""level"level"hostname::3030":,""time":"ip-10-150-1-74.us-west-2.compute.internal2023-09-22T23:08:02.425027911Z"",","pid":hostname"4291:","upstairs":ip-10-150-1-74.us-west-2.compute.internal,""1time,"":pid""}:
,2023-09-22T23:08:02.425046405Z4291"","timehostname,"{:"""upstairs"::ip-10-150-1-74.us-west-2.compute.internal"1,"}"
"2023-09-22T23:08:02.425046431Zpidmsg"""::{4291,"",""hostname"upstairs:msg"""::1[0] Transition from New to WaitActive}"
[1]R  flush_numbers: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]""ip-10-150-1-74.us-west-2.compute.internal"{,,"",""pidv"v"":::msg042910",:"",,"name"024c3783-b9b4-4453-823f-769dad90d4f2 WaitActive WaitActive WaitActive:"""upstairsnamecrucible,"""v,""level:"0:,30":"1name"::}""
cruciblecrucible,"","time"level:"":"{2023-09-22T23:08:02.425121117Z30",","level"hostname""::"msg30":"ip-10-150-1-74.us-west-2.compute.internal","pid":,"4291time":,""upstairs":12023-09-22T23:08:02.42513842Z"},
"hostname":"[1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 10, shift: 9 }, extent_count: 10, uuid: e69c25b9-b0d6-406a-850b-6b5dba8b6e69, encrypted: false, database_read_version: 1, database_write_version: 1 }"{ip-10-150-1-74.us-west-2.compute.internal,""",msg""pid:"":v4291":,0[1]R  generation: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]"",upstairs",",:time1""v}"
:"name{0:"",""2023-09-22T23:08:02.425145289Znamemsg""::"":""cruciblecrucible""[2] downstairs client at 127.0.0.1:44064 has UUID f94c50e4-ac3c-41cd-806f-21404d88edf4,"",level""v:"30:,,0","hostname"":name"":level"":ip-10-150-1-74.us-west-2.compute.internal,crucible""time,""30:"",level""2023-09-22T23:08:02.425190309Z:"30pid,"":hostname"4291:","upstairs":ip-10-150-1-74.us-west-2.compute.internal"1,"}pid"
,:"4291time":","upstairs":,1"}2023-09-22T23:08:02.425207483Z
"{time,{"":hostname"":msg""":""ip-10-150-1-74.us-west-2.compute.internalmsg2023-09-22T23:08:02.425203803Z""",,"[1]R  dirty: [false, false, false, false, false, false, false, false, false, false]:""pid",:"4291v":,0",""upstairsname":":"1[0] client is_active_req TRUE, promote! session a7f1d842-f19a-4c81-abcd-c1f78df01722"hostname"}crucible
":",{","ip-10-150-1-74.us-west-2.compute.internalvlevel"""msg:"30:,"""pid"::04291,"name":","crucible"upstairs",:"1level":,30"[2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 10, shift: 9 }, extent_count: 10, uuid: f94c50e4-ac3c-41cd-806f-21404d88edf4, encrypted: false, database_read_version: 1, database_write_version: 1 }time"":,""}v"2023-09-22T23:08:02.425258673Z":
,{"time":"2023-09-22T23:08:02.425270299Z","hostname":""msg"ip-10-150-1-74.us-west-2.compute.internal":","pid":4291,"ed581aed-4fb6-4c12-84d1-a857283bcda2 WaitActive WaitActive WaitActive"upstairs":1,"}0v"
,:"0name",:"{"crucible"name",""msg":level"::30""crucible","level":30[1] a2b1abbb-2148-4d24-93b2-2a0a303e7718 (a7f1d842-f19a-4c81-abcd-c1f78df01722) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.425321427Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":,"4291time":",",2023-09-22T23:08:02.425325215Zupstairs":"1",}"time
"hostname"::""{2023-09-22T23:08:02.425332437Z"ip-10-150-1-74.us-west-2.compute.internal","",msg""hostname"::""pid":4291ip-10-150-1-74.us-west-2.compute.internal"024c3783-b9b4-4453-823f-769dad90d4f2 WaitActive WaitActive WaitActive",,"","pid"v"upstairs":::014291,}"
name",:""upstairs"crucible{":,"1level":"}msg30":
"{[2] downstairs client at 127.0.0.1:55647 has UUID 6fbc425f-e617-4cfe-95cb-547a3dd7a1a3","v":0","msg"name"::""The guest has finished waiting for activation
crucible,""[1] Transition from New to WaitActive",time,"""v:"level""::2023-09-22T23:08:02.425392735Z"030,,""hostname"name"::""crucible","level"ip-10-150-1-74.us-west-2.compute.internal":,"30pid":4291,"upstairs":1}
,"time":"{2023-09-22T23:08:02.425425363Z","hostname":""msg":"ip-10-150-1-74.us-west-2.compute.internal"negotiate packet ExtentVersionsPlease,"",pid",v""":time"0:,:"4291name":",""crucible"upstairs2023-09-22T23:08:02.425434748Z","",:level"1:"}30
hostname":"ip-10-150-1-74.us-west-2.compute.internal"{,"pid":4291","msg":upstairs"":1,"}time"
:"2023-09-22T23:08:02.425472103Z"[2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 10, shift: 9 }, extent_count: 10, uuid: 6fbc425f-e617-4cfe-95cb-547a3dd7a1a3, encrypted: false, database_read_version: 1, database_write_version: 1 }",{","hostname"v"":"msg"::"ip-10-150-1-74.us-west-2.compute.internal"0,",pid"":name4291[1] client is_active_req TRUE, promote! session a7f1d842-f19a-4c81-abcd-c1f78df01722"":,,"""downstairs"v":crucible"1:,}0
","levelname""::{"30crucible",""levelmsg""::"30negotiate packet ExtentVersionsPlease","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.425525857Z","hostname":","time":"ip-10-150-1-74.us-west-2.compute.internal","pid2023-09-22T23:08:02.425530466Z"":,",4291hostname"":,"time":""ip-10-150-1-74.us-west-2.compute.internal"upstairs,"2023-09-22T23:08:02.42553781Z"",pid:""1:hostname}4291"
:,""upstairs":1ip-10-150-1-74.us-west-2.compute.internal"},
"{pid":4291,""{downstairs"msg"::2""}msg"
:"ed581aed-4fb6-4c12-84d1-a857283bcda2 WaitActive WaitActive WaitActive","v":{0,"name":""cruciblemsg":"",[2] a2b1abbb-2148-4d24-93b2-2a0a303e7718 (a7f1d842-f19a-4c81-abcd-c1f78df01722) WaitActive WaitActive New ds_transition to WaitActive""levelnegotiate packet ExtentVersionsPlease",",:"30v"":v"0:,"0name",:""name":"crucible"crucible",","levellevel""::3030,"time":"2023-09-22T23:08:02.42560798Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":,1"}time
,":""time":"{2023-09-22T23:08:02.425621189Z"2023-09-22T23:08:02.425620978Z","",msg"hostname""::""hostname":"ip-10-150-1-74.us-west-2.compute.internal"negotiate packet ExtentVersionsPlease"ip-10-150-1-74.us-west-2.compute.internal",,,""pid"pid""v:"4291::42910,,,"""name"upstairs"downstairs::"1:"}3
crucible"}
,{"level":{30"msg":""msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level[0] 024c3783-b9b4-4453-823f-769dad90d4f2 (db076280-2529-45a1-a093-a3cd3f0799dc) WaitActive WaitActive WaitActive ds_transition to WaitQuorum"":,30"v":0,"name":"crucible",","level":time":30"2023-09-22T23:08:02.425686193Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"time":,""downstairs":2023-09-22T23:08:02.425698288Z"1,",}"
time"hostname"::""{2023-09-22T23:08:02.425705354Z"ip-10-150-1-74.us-west-2.compute.internal",,"""msghostname"pid":":"4291:",ip-10-150-1-74.us-west-2.compute.internal""negotiate packet ExtentVersionsPlease",upstairs""pid,"":v"1::4291}0,
test impacted_blocks::test::empty_impacted_blocks_never_conflict ... ,"{upstairs""okname"":msg:
":"1"crucible"}
,[2] client is_active_req TRUE, promote! session a7f1d842-f19a-4c81-abcd-c1f78df01722""level"{,":v"30":msg"0:","name":"crucible"[0] Transition from WaitActive to WaitQuorum",",level"":v"30:0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.425788672Z","hostname":","ip-10-150-1-74.us-west-2.compute.internaltime"",,":"timepid""::"4291"2023-09-22T23:08:02.425796997Z","2023-09-22T23:08:02.425801644Z",,downstairs""":hostnamehostname""::""2ip-10-150-1-74.us-west-2.compute.internal","ip-10-150-1-74.us-west-2.compute.internal"pid,"":pid"4291:}4291,
","upstairs":upstairs"1:1{}}

{"msg":""msg"{:"negotiate packet ExtentVersionsPlease","v"[0] new RM replaced this: None"":,0"msg",v":":name0":"","crucible"name",:""level"crucible":,30"negotiate packet PromoteToActive { upstairs_id: a2b1abbb-2148-4d24-93b2-2a0a303e7718, session_id: a7f1d842-f19a-4c81-abcd-c1f78df01722, gen: 1 }level"":40,"v":0,"name":"crucible","level":30,",time"":"time":"2023-09-22T23:08:02.425884668Z"2023-09-22T23:08:02.425881654Z,"",hostname"":,hostname""":time""ip-10-150-1-74.us-west-2.compute.internal":,"ip-10-150-1-74.us-west-2.compute.internal"","2023-09-22T23:08:02.425890717Z"pid"pid",:"4291:hostname4291",:"","upstairs":downstairs"1:ip-10-150-1-74.us-west-2.compute.internal"3,}}
"
{pid":4291","{msg"downstairs:"":"1[0] Starts reconcile loop"}msg":",
"v":0,"[0] ed581aed-4fb6-4c12-84d1-a857283bcda2 (99a7dfa5-d205-4d07-9135-9f9e70387e3a) WaitActive WaitActive WaitActive ds_transition to WaitQuorum"name"{:,"""cruciblemsg"":v":,""0level",:"30name":"crucible","level"negotiate packet PromoteToActive { upstairs_id: a2b1abbb-2148-4d24-93b2-2a0a303e7718, session_id: a7f1d842-f19a-4c81-abcd-c1f78df01722, gen: 1 }":,30"v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.42596589Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":,1"},time
"":time"":{"2023-09-22T23:08:02.42597117Z",2023-09-22T23:08:02.425976017Z"""hostnamemsg"":,:"""hostname":"ip-10-150-1-74.us-west-2.compute.internal"ip-10-150-1-74.us-west-2.compute.internal",",pid"":[1] 024c3783-b9b4-4453-823f-769dad90d4f2 (db076280-2529-45a1-a093-a3cd3f0799dc) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum"4291pid,",:"4291"upstairs",":v1"downstairs"::}02,
}"
{name":"crucible",""{msg"level"::""30msg[0] Transition from WaitActive to WaitQuorum","v":"0:,""name":"crucible","level":30The guest has finished waiting for activation
negotiate packet PromoteToActive { upstairs_id: a2b1abbb-2148-4d24-93b2-2a0a303e7718, session_id: a7f1d842-f19a-4c81-abcd-c1f78df01722, gen: 1 }",",time"":v"":0,"2023-09-22T23:08:02.426041175Zname"":","crucible"hostname,"",level"::""30time":"ip-10-150-1-74.us-west-2.compute.internal","2023-09-22T23:08:02.426050422Z"pid":,4291"hostname",:""upstairs":1}ip-10-150-1-74.us-west-2.compute.internal"
,"pid":4291,"{,"time":upstairs""":msg12023-09-22T23:08:02.426071288Z"}"
,:""hostname":"{[1] Transition from WaitActive to WaitQuorum"ip-10-150-1-74.us-west-2.compute.internal",,""v"pid"::04291,""name"msg"::,"""crucible"downstairs"[0] new RM replaced this: None",":,3"levelv"":}0
:,30"name":"crucible{","level":40"msg":"negotiate packet RegionInfoPlease","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.426127536Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal",","time"pid"::"4291,2023-09-22T23:08:02.42613225Z""upstairs",:"1hostname":"}
ip-10-150-1-74.us-west-2.compute.internal",,""pid"time"{:":42912023-09-22T23:08:02.426140725Z","msg"upstairs":"":,"1[1] new RM replaced this: None"}hostname
,"":"v":0{,ip-10-150-1-74.us-west-2.compute.internal"",name"":""pid"msgcrucible""::,4291""level",:"40[0] Starts reconcile loopdownstairs"":,1"v":}0
,"name":"crucible","level":{30",msg"":"time":"negotiate packet RegionInfoPlease"2023-09-22T23:08:02.4261991Z",",v"":hostname"0:","name":"crucibleip-10-150-1-74.us-west-2.compute.internal"",",",level"":pid"30time"::4291","2023-09-22T23:08:02.426212025Z"upstairs":,1"}hostname"
:,""time":"ip-10-150-1-74.us-west-2.compute.internal{"2023-09-22T23:08:02.426231706Z","pid",:"4291"hostname":msg"":,""ip-10-150-1-74.us-west-2.compute.internal"upstairs[1] Starts reconcile loop"":,",pid"1":v"4291}:
,0","downstairs"name":{2:"}crucible"
,""msglevel""::30"{"msg":"negotiate packet RegionInfoPlease"[1] ed581aed-4fb6-4c12-84d1-a857283bcda2 (99a7dfa5-d205-4d07-9135-9f9e70387e3a) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum","v",:"0v",:"0name":","crucible"name":,,""leveltime"":":"crucible"30,2023-09-22T23:08:02.426288745Z"",level"":hostname"30:"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
,"time":"{2023-09-22T23:08:02.426311232Z"","msg":hostname","":"time":"ip-10-150-1-74.us-west-2.compute.internal"2023-09-22T23:08:02.426316107Z,""[2] 024c3783-b9b4-4453-823f-769dad90d4f2 (db076280-2529-45a1-a093-a3cd3f0799dc) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum,pid"":"4291hostname",:"",v""ip-10-150-1-74.us-west-2.compute.internal:downstairs"0,"",pid":name"3:"":}crucible"
4291,"level":,30"{upstairs":1}
"msg":"{[0] downstairs client at 127.0.0.1:61368 has UUID 56a24d20-5fb0-4556-90e5-68512760a2f4","v",:"0"msg","time"name"::"":"crucible2023-09-22T23:08:02.426366087Z"",,""[1] Transition from WaitActive to WaitQuorum"hostname"level",:":"v"30:0ip-10-150-1-74.us-west-2.compute.internal",","namepid""::"4291crucible",,""upstairs"level"::130}
,"time":"{2023-09-22T23:08:02.426399712Z",""hostname":msg"":"ip-10-150-1-74.us-west-2.compute.internal"[2] Transition from WaitActive to WaitQuorum",",pid"":v4291":0,,,"""upstairs":time1name"}"::
""crucible","2023-09-22T23:08:02.426412432Z{level"":30,""msg":hostname"":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
[0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 10, shift: 9 }, extent_count: 10, uuid: 56a24d20-5fb0-4556-90e5-68512760a2f4, encrypted: false, database_read_version: 1, database_write_version: 1 }",","time"v":{:"0,"name":""crucible2023-09-22T23:08:02.426447329Z""msg,,""level":"hostname"":[1] new RM replaced this: None":"30,"v":0ip-10-150-1-74.us-west-2.compute.internal",,""pid"name:"4291:"crucible,"","upstairs"level:"1:,40"}time"
:"2023-09-22T23:08:02.426480715Z"{,"hostname":""msg":"ip-10-150-1-74.us-west-2.compute.internal"[2] new RM replaced this: None",,""pid":v"4291:,0",,time"""name:upstairs""":2023-09-22T23:08:02.426494912Z"1:","}crucible
hostname"":,""{level":40ip-10-150-1-74.us-west-2.compute.internal",""pid"msg"::"4291,"upstairs":a2b1abbb-2148-4d24-93b2-2a0a303e7718 WaitActive WaitActive WaitActive"1,"}v
",:"0time",:"{"name":"crucible"2023-09-22T23:08:02.426531338Z",,"""msglevel"hostname":""::"[1] Starts reconcile loop"30,"v"ip-10-150-1-74.us-west-2.compute.internal":,"0pid",:"4291name":","crucible"upstairs",":1level"}:
30,{"time"":"msg":"2023-09-22T23:08:02.426560775Z"[2] Starts reconcile loop",","hostname"v:"":0,"name":ip-10-150-1-74.us-west-2.compute.internal"","crucible"pid,"",:level4291"",:"30time"upstairs"::"1}
2023-09-22T23:08:02.42657612Z",",hostname"":{"time":"ip-10-150-1-74.us-west-2.compute.internal"",msg2023-09-22T23:08:02.426597074Z"",""pid"hostname"::":"4291ip-10-150-1-74.us-west-2.compute.internal",,""upstairspid[1] downstairs client at 127.0.0.1:50978 has UUID 4a0427cd-ca30-4862-a8cb-bfc11d93d1ff""":,:"1v":}04291,
,""name":upstairs"":crucible1{"}The guest has finished waiting for activation

","msg"level"{:":"30msg":"[0] 127.0.0.1:52643 task reports connection:true","[2] ed581aed-4fb6-4c12-84d1-a857283bcda2 (99a7dfa5-d205-4d07-9135-9f9e70387e3a) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum"v":0,,""v"name"::"0crucible",,""name"level":":,30"crucible"time":","level":2023-09-22T23:08:02.426656078Z"30,"hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":,4291"time",:""upstairs":12023-09-22T23:08:02.426672258Z"},
"hostname":","{time"ip-10-150-1-74.us-west-2.compute.internal":,"""pid"2023-09-22T23:08:02.426676677Z":msg,4291"",:""hostname":upstairs":"1}
ip-10-150-1-74.us-west-2.compute.internal","pid"{:4291[1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 10, shift: 9 }, extent_count: 10, uuid: 4a0427cd-ca30-4862-a8cb-bfc11d93d1ff, encrypted: false, database_read_version: 1, database_write_version: 1 }""msg":,"","v":upstairs"0:,1024c3783-b9b4-4453-823f-769dad90d4f2 WaitQuorum WaitQuorum WaitQuorum"",}name""
v"::"0,crucible""{,name"":"level":crucible""30,msg"":level"":30[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":","2023-09-22T23:08:02.426748226Z"time":,""hostname":"2023-09-22T23:08:02.426744678Z","hostname"ip-10-150-1-74.us-west-2.compute.internal":,""pid":4291,ip-10-150-1-74.us-west-2.compute.internal",,""pid"upstairs"":time":42911:}",
"2023-09-22T23:08:02.426755916Z"upstairs"{:,""1msg"hostname"}::"
"ip-10-150-1-74.us-west-2.compute.internal","pid":4291[0]R  flush_numbers: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]"{,,""v":upstairs"0:,"1"msg"name"::""}crucible"
,a2b1abbb-2148-4d24-93b2-2a0a303e7718 WaitActive WaitActive WaitActive""level":,"30v"{:0,"name":""crucible"msg",:""level":30[2] new RM replaced this: None",",v"":time"0:","name":"2023-09-22T23:08:02.426819392Z"crucible",","hostname"level"::"40ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs,"":1time"}:
"{2023-09-22T23:08:02.426827874Z"",msg"":"hostname":","time":[0]R  generation: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]"ip-10-150-1-74.us-west-2.compute.internal,""",v"2023-09-22T23:08:02.426839692Z"pid",""::hostname"0:,""4291name":"ip-10-150-1-74.us-west-2.compute.internalcrucible"",,,"""levelpid""upstairs":::4291301,"}upstairs"
:1}
,{"time":"{"2023-09-22T23:08:02.426884196Z"msg",:"""hostname"msg"::""[2] downstairs client at 127.0.0.1:57228 has UUID 2ac204bf-fd46-40b7-a314-196d508e471d[2] Starts reconcile loop""ip-10-150-1-74.us-west-2.compute.internal,,""vv""":,0:",pid"0:",4291name",:""name"upstairs"::""1crucible}"
crucible","{,level":""30level"msg"::"30[0]R  dirty: [false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.426936938Z",",time"":,time"""hostname":2023-09-22T23:08:02.426939613Z":"","2023-09-22T23:08:02.426948382Z"hostname"ip-10-150-1-74.us-west-2.compute.internal,"",":pid"hostname:"4291":",ip-10-150-1-74.us-west-2.compute.internal""ip-10-150-1-74.us-west-2.compute.internal","upstairs",pid"::"42911pid":,}"4291
,upstairs"":upstairs"1:{1}}

"{msg":"{"[0] 127.0.0.1:60909 task reports connection:true"msg",msg""":":v"":[1]R  flush_numbers: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]"0,,""vname""::"0crucible",","name"level":[2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 10, shift: 9 }, extent_count: 10, uuid: 2ac204bf-fd46-40b7-a314-196d508e471d, encrypted: false, database_read_version: 1, database_write_version: 1 }"":crucible"30,","v"level"::030,"name":"crucible","level":30,"time":","time":2023-09-22T23:08:02.427028838Z"","hostname":2023-09-22T23:08:02.427032937Z"",,""timeip-10-150-1-74.us-west-2.compute.internal"":"hostname",:""pid"2023-09-22T23:08:02.427037501Z":ip-10-150-1-74.us-west-2.compute.internal",,4291""hostname"pid:","":upstairs4291":ip-10-150-1-74.us-west-2.compute.internal1,"",}upstairs""
:pid"1:}4291
,"{upstairs":{1}
""msg"msg:":""{[1]R  generation: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]",""ed581aed-4fb6-4c12-84d1-a857283bcda2 WaitQuorum WaitQuorum WaitQuorum"v",msg""::0v",:""0name",:""crucible"a2b1abbb-2148-4d24-93b2-2a0a303e7718 WaitActive WaitActive WaitActive",name""level:,"":"30v"crucible":,"0level":,"30name":"crucible","level":30,"time":"2023-09-22T23:08:02.427123338Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
,"time{",:""msg"":time":2023-09-22T23:08:02.427128267Z""","hostname":2023-09-22T23:08:02.427131734Z"","[1]R  dirty: [false, false, false, false, false, false, false, false, false, false]hostname"ip-10-150-1-74.us-west-2.compute.internal"",:,"""v"pid"ip-10-150-1-74.us-west-2.compute.internal::"0,4291,""pid",":name4291":upstairs,"":"upstairs"crucible1:"1,}"}
level"
:{30"{msg":""[0]R  flush_numbers: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]"msg":","v":0negotiate packet ExtentVersionsPlease,"",",time""name"v"::":"0crucible"2023-09-22T23:08:02.427194738Z",,",level""":name"hostname"30::""crucible","level":ip-10-150-1-74.us-west-2.compute.internal"30,"pid":4291,"upstairs":1}
,"time":"{2023-09-22T23:08:02.42722207Z""msg",:",""time":"hostname":"[2]R  flush_numbers: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]"2023-09-22T23:08:02.427227026Z","ip-10-150-1-74.us-west-2.compute.internal"v",,"":pidhostname0",""::name""4291:"ip-10-150-1-74.us-west-2.compute.internal"crucible,",,"""pidupstairs"level:"":1:30}4291
,"downstairs":1}
,{"time":"{"2023-09-22T23:08:02.427274535Z"msg":,"""msg":hostname""[0]R  generation: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]":",negotiate packet ExtentVersionsPlease""v"ip-10-150-1-74.us-west-2.compute.internal":,,"0"pid"v"::4291,"0name,,""":upstairs""name"::crucible""1,crucible"}",level"
:"30level"{:30"msg":"[2]R  generation: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":","time"2023-09-22T23:08:02.427326017Z":,"","time"2023-09-22T23:08:02.427328198Z:hostname""":",2023-09-22T23:08:02.42733701Z""ip-10-150-1-74.us-west-2.compute.internal"hostname",":,"hostname"":"pid"ip-10-150-1-74.us-west-2.compute.internal":,"ip-10-150-1-74.us-west-2.compute.internalpid"":4291,4291",pid","":upstairs4291downstairs",:"2"upstairs"}::
11}
}{
{"msg":""msg":"{negotiate packet ExtentVersionsPlease","v":0"[2]R  dirty: [false, false, false, false, false, false, false, false, false, false]",",msg"":v""name:"0:,""[0]R  dirty: [false, false, false, false, false, false, false, false, false, false]name"crucible:""",crucible",,"""vlevellevel":"30"::300,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.427421016Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,","upstairs":time"1:}",
2023-09-22T23:08:02.427422952Z""{,time"":"hostname"msg""::""2023-09-22T23:08:02.427427433Z"Max found gen is 1ip-10-150-1-74.us-west-2.compute.internal,""",hostname",""v"::pid0"",:"4291name"ip-10-150-1-74.us-west-2.compute.internal",:"",crucible"downstairs"",pid""::level4291"3:,}30
"upstairs":1}
,{"time":"{2023-09-22T23:08:02.427481988Z"","msg":hostname"":""msg":"ip-10-150-1-74.us-west-2.compute.internal","[1]R  flush_numbers: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]"pid":,"4291v",:"0[0] a2b1abbb-2148-4d24-93b2-2a0a303e7718 (a7f1d842-f19a-4c81-abcd-c1f78df01722) WaitActive WaitActive WaitActive ds_transition to WaitQuorum,"upstairsname"""::,"1v}"
"crucible":{,0"",msg"level"":name"30::""crucible","Generation requested: 1 >= found:1"level",:"30v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.427529747Z","hostname":","time":"ip-10-150-1-74.us-west-2.compute.internal","pid",:2023-09-22T23:08:02.427540289Z"4291",",hostname"time"upstairs"":::""1}ip-10-150-1-74.us-west-2.compute.internal"2023-09-22T23:08:02.427536227Z","
pid",:"4291hostname"{,:""upstairs"":msg"ip-10-150-1-74.us-west-2.compute.internal"1:"},
[1]R  generation: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]""pid,{"":v"4291":msg",:"0",Next flush: 1upstairs""":name",1:""}v":crucible
"0,","level"name"::{"30crucible","level":"30msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,","time":"time":"2023-09-22T23:08:02.427615145Z"2023-09-22T23:08:02.427612502Z,""hostname":","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid",ip-10-150-1-74.us-west-2.compute.internal":"4291time",",:""upstairs"pid"::12023-09-22T23:08:02.427624354Z4291}"
,","hostname":"upstairs"{:ip-10-150-1-74.us-west-2.compute.internal"1,"}"pidmsg":""
:All extents match"4291,"v":,"0,upstairs""{name"::1"}"
crucible"msg",:""level{":30[1]R  dirty: [false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible"","msg"level"::"30[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[2]R  flush_numbers: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,,""timetime""::""2023-09-22T23:08:02.427690719Z2023-09-22T23:08:02.427756577Z"",,""hostnamehostname""::""ip-10-150-1-74.us-west-2.compute.internalip-10-150-1-74.us-west-2.compute.internal"",,""pidpid""::42914291,",",upstairs""upstairs:":11time}}

{"msg":"No downstairs repair required"{,"v":0",msg"":"name":"crucible","[2]R  generation: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]",level,":"30v"":0time",:""name":"crucible"2023-09-22T23:08:02.427700749Z,""level":,30"hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid,":"4291time":","upstairs2023-09-22T23:08:02.427817291Z"",,"":hostnametime""::""1}2023-09-22T23:08:02.427825784Z"ip-10-150-1-74.us-west-2.compute.internal",
","hostname"pid:"":4291,"ip-10-150-1-74.us-west-2.compute.internalupstairs"":,1{"}
pid"":msg"4291{:",""msg":upstairs"":1[2]R  flush_numbers: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]"}No initial repair work was required
",",{"vv"":"0msg,"":name""::"0crucible",,"[2]R  dirty: [false, false, false, false, false, false, false, false, false, false]level"",":v30"":name0":,""crucible"name":,""level":crucible"30,","level"time:":30"2023-09-22T23:08:02.427878304Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
,"time":"{2023-09-22T23:08:02.427888504Z""msg,"":,hostname"":""Set Downstairs and Upstairs activetime"":",ip-10-150-1-74.us-west-2.compute.internal""v,"":pid0"2023-09-22T23:08:02.427887456Z":,"4291,,name"""upstairs:"hostname":"1:"crucible}"
,"ip-10-150-1-74.us-west-2.compute.internal"level{","":pid30"msg"::"4291Max found gen is 1",,""vupstairs""::10,},
""timename""::""crucible","2023-09-22T23:08:02.427922787Zlevel"":,30"{hostname":""ip-10-150-1-74.us-west-2.compute.internal"msg",":pid"":4291,,""timeupstairs"":":[2]R  generation: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]1"}2023-09-22T23:08:02.427941028Z
",,"{hostname":""msg":"ip-10-150-1-74.us-west-2.compute.internal",""pidv""::024c3783-b9b4-4453-823f-769dad90d4f2 is now active with session: db076280-2529-45a1-a093-a3cd3f0799dc4291"0,,"",upstairs"v:""1:}name"test control::test::test_crucible_control_openapi ... :
0"crucible{,"""name"msg:ok"":crucible"",
"level"Generation requested: 1 >= found:1,"":level,"":v30"30:0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.427999413Z","hostname,"":"time":"ip-10-150-1-74.us-west-2.compute.internal2023-09-22T23:08:02.428004146Z"",",,pid"":hostname"4291":",time"":ip-10-150-1-74.us-west-2.compute.internalupstairs"",:"1"}pid
":2023-09-22T23:08:02.428000542Z"{4291,",""hostname":upstairsmsg""::"1"}
024c3783-b9b4-4453-823f-769dad90d4f2 Set Active after no repair"ip-10-150-1-74.us-west-2.compute.internal,{"v":"0msg,"":"name":"Next flush: 1"crucible"",,,""v"level:"0:,30""pid"name"::"4291crucible",","level":upstairs30":1}
,,"time":""{time":"2023-09-22T23:08:02.428054335Z","2023-09-22T23:08:02.428047478Zhostname"":,"""msg":"hostnameip-10-150-1-74.us-west-2.compute.internal"":",""pid"::ip-10-150-1-74.us-west-2.compute.internal""4291[2]R  dirty: [false, false, false, false, false, false, false, false, false, false]",,"",upstairs2023-09-22T23:08:02.427707985Z"pid""::1"}4291
,,v"{"hostname"upstairs""::msg1"}:"
All extents match""{":,"0ip-10-150-1-74.us-west-2.compute.internal,""name":msg""":vcrucible"",:"Notify all downstairs, region set compare is done.0",pid"":",,4291""namev"",level":::0""crucible,"upstairs","":levelname130"":":}crucible30"
,"level":30{,"time":","2023-09-22T23:08:02.428130599Z","",time""msghostname:"":time"2023-09-22T23:08:02.428137042Z""":":ip-10-150-1-74.us-west-2.compute.internal,""","hostname"pid[0] Starts reconcile loop""::",2023-09-22T23:08:02.428128412Z""ip-10-150-1-74.us-west-2.compute.internalv4291",",,"""pidupstairs"":::14291hostname,}0"
":"upstairs{","ip-10-150-1-74.us-west-2.compute.internalmsg""::1""},
No downstairs repair required""name,{"pid""vmsg""::0",:"Set check for repair":"name,"":"4291"v"crucible:crucible0","","name,level"""::level"30,:""30upstairs":crucible"1,"}level"
,:"30time":"2023-09-22T23:08:02.428214842Z"{,"hostname":""msg"ip-10-150-1-74.us-west-2.compute.internal":","pid,""Max found gen is 1time:"4291:","",,"2023-09-22T23:08:02.428226056Z"upstairs"",:"1time"v:}hostname
""":{:"02023-09-22T23:08:02.428217517Z""ip-10-150-1-74.us-west-2.compute.internalmsg"",:"",,pidNo initial repair work was required""":,4291"hostname"v:",:"0"upstairs,"""namename"ip-10-150-1-74.us-west-2.compute.internal"",::"1"}pid"
crucible::""{crucible"4291",msg""level:,,"""upstairs":[1] 127.0.0.1:34619 task reports connection:true30"",":level1v""::0},30"
name,"":"time":crucible"","level"2023-09-22T23:08:02.428297015Z:"30{,"hostname":""msg":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1,",}"
timetime"[1] a2b1abbb-2148-4d24-93b2-2a0a303e7718 (a7f1d842-f19a-4c81-abcd-c1f78df01722) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum:""{":",2023-09-22T23:08:02.428305722Z"2023-09-22T23:08:02.428314105Z"v"",:"0,""hostname"msghostname""::"":","Set Downstairs and Upstairs active"ip-10-150-1-74.us-west-2.compute.internal",,name"""vpidip-10-150-1-74.us-west-2.compute.internal"""::42910:,,"","name"upstairs:"":crucible"",1cruciblepid"":"},
"level"level{":":msg30"4291:30","upstairs":1024c3783-b9b4-4453-823f-769dad90d4f2 Active Active Active"},"
,v"":time0":,""name":"2023-09-22T23:08:02.428371884Zcrucible""{,,""levelhostname""::"30","msg"ip-10-150-1-74.us-west-2.compute.internal:time"":,"""pid":Generation requested: 1 >= found:12023-09-22T23:08:02.42837382Z"",4291",time,"":upstairs",""2023-09-22T23:08:02.428388638Z"":,1v"}hostname"
"::hostname{"":""msg":"ip-10-150-1-74.us-west-2.compute.internal"ip-10-150-1-74.us-west-2.compute.internal"0,,,"df6648fa-e480-4dd9-b7c4-7fec45dbddf8 is now active with session: b793572f-19dc-423b-877e-0f0cfe9147d9"pidname"""pid",:"4291v":,:"":0",4291upstairs""name:,crucible"1"}"
:,"{upstairs"""levelmsgcrucible"":,""level:""1::Set check for repair30"},30"
v":0,"name":,""timecrucible"",:""{level":302023-09-22T23:08:02.428454271Z",""hostname"msg"::""ip-10-150-1-74.us-west-2.compute.internal"[1] Transition from WaitActive to WaitQuorum,,",pid""time:"4291:",""",upstairstime"""2023-09-22T23:08:02.428467397Z:v"::"1"02023-09-22T23:08:02.428457095Z",}",,hostname
"""name"{::hostname""":""msgip-10-150-1-74.us-west-2.compute.internal"":,""ip-10-150-1-74.us-west-2.compute.internalpid"":crucibledf6648fa-e480-4dd9-b7c4-7fec45dbddf8 Set Active after no repair4291",,,""v"","upstairs:"0:,pid"":name1"}:
"4291"crucible{"level,",""msglevel""::"30:"30upstairs"[2] 127.0.0.1:44064 task reports connection:true":,"1v":}0
,,""timename""::""crucible","2023-09-22T23:08:02.42853364Z"level",:"30{hostname",:""time":""msg"ip-10-150-1-74.us-west-2.compute.internal"2023-09-22T23:08:02.428534983Z",:,",""pidtime"Next flush: 1":"":",4291hostname",""2023-09-22T23:08:02.428548947Z"upstairs"v":,1"}:hostname:"":
"0ip-10-150-1-74.us-west-2.compute.internal",,{ip-10-150-1-74.us-west-2.compute.internal"""msg,"":"pid""pidname"Notify all downstairs, region set compare is done.:"4291,","::"4291v"":upstairs,crucible"0",:"1"},name
"":{"upstairs":"cruciblemsg""1level",:""level"}::
30024c3783-b9b4-4453-823f-769dad90d4f2 Active Active Active"30,"v":0{,"name":"crucible",""msg"level"::,30""time":"[1] new RM replaced this: None"2023-09-22T23:08:02.428616483Z",",v":"0,hostname"":"name",,ip-10-150-1-74.us-west-2.compute.internal:""","time"crucible"timepid"""::"4291,:,2023-09-22T23:08:02.428625951Z""level"""2023-09-22T23:08:02.428617985Zupstairs,"":hostname1"}:
":"40{ip-10-150-1-74.us-west-2.compute.internal",,""msg"pid:"":"4291Set check for repair",hostname","":upstairsv""::10",}"
name"ip-10-150-1-74.us-west-2.compute.internal"{:,"",cruciblemsg""time""",:""pid"Set check for repairlevel"",::30"":v":42912023-09-22T23:08:02.428659097Z"0,,",""name"upstairs:,"hostname""":crucible:""time,"":1ip-10-150-1-74.us-west-2.compute.internal""level,"2023-09-22T23:08:02.42868559Z:"30"}pid,
"":,hostname"":time""4291:{",ip-10-150-1-74.us-west-2.compute.internal""2023-09-22T23:08:02.428707246Z,""",pid"msg"hostname:upstairs"4291"::,"""1upstairsip-10-150-1-74.us-west-2.compute.internal"":,1"}pid:""}:
4291
,{All extents match"""msgupstairs""::1{}"
[1] 127.0.0.1:51919 task reports connection:true""{,,""msg"v:"""msg"v"[0] received reconcile message:"0::,,"""vname""::"[1] Starts reconcile loopcrucible0","",name""0:level,,"""crucible:"30"name"v"::"0,,""crucible"level":,"name",30"level"time:"":"crucible","2023-09-22T23:08:02.428778624Z"level",:",30hostname"":time"":"ip-10-150-1-74.us-west-2.compute.internal2023-09-22T23:08:02.428789601Z"",","pid":hostname4291":","upstairs":1ip-10-150-1-74.us-west-2.compute.internal"},
",pid":{4291":",time"msg"":upstairs:30""":1}
df6648fa-e480-4dd9-b7c4-7fec45dbddf8 Active Active Active"2023-09-22T23:08:02.428799162Z",{"v,"":msg0",:""name"":"[0] All repairs completed, exitcrucible"",hostname","":vlevel""::030,"",name""ip-10-150-1-74.us-west-2.compute.internal":time"":,crucible"""pid",:,""2023-09-22T23:08:02.428823255Zleveltime"4291":":30,"","upstairs"2023-09-22T23:08:02.428839709Zhostname":"":,"ip-10-150-1-74.us-west-2.compute.internal"1hostname,":"",time"ip-10-150-1-74.us-west-2.compute.internal:"",""2023-09-22T23:08:02.428855962Z"pid",":pid"hostname4291":,:}4291
"","upstairs"upstairsip-10-150-1-74.us-west-2.compute.internal:"1,"}"
pid"{::4291{1,""upstairs""msg":}:"
1msg":}"
Set check for repair"[2] a2b1abbb-2148-4d24-93b2-2a0a303e7718 (a7f1d842-f19a-4c81-abcd-c1f78df01722) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum",{",{"vmsg""::0",""[0] Starts cmd_loopname"""msg":,""v:cruciblev""":,0",""level"name"::30:No downstairs repair required""0crucible,"",v,"""name",:"leveltime0""::30:,"""name"crucible2023-09-22T23:08:02.428927922Z"",:",""hostname,"":time""crucible":level"ip-10-150-1-74.us-west-2.compute.internal"":,30"2023-09-22T23:08:02.428941469Zpid"",,:"4291"hostname,level""":upstairs:"":301}ip-10-150-1-74.us-west-2.compute.internal
","pid":{4291,""upstairsmsg""::",1"}
[2] 127.0.0.1:45755 task reports connection:true"time":{,"""msgv""::"02023-09-22T23:08:02.428959465Z",[1] received reconcile message"",,name,""""v:"time":":crucible0",,"hostname"namelevel""::""crucible30":"","level"2023-09-22T23:08:02.428967806Z":ip-10-150-1-74.us-west-2.compute.internal","30,hostname"":pid,"":time"4291":",",ip-10-150-1-74.us-west-2.compute.internal"2023-09-22T23:08:02.428999471Ztimeupstairs""":,""":,2023-09-22T23:08:02.429006077Z""hostname"1:,"pid"}"ip-10-150-1-74.us-west-2.compute.internal
:hostname""4291:,""pid"{,ip-10-150-1-74.us-west-2.compute.internal:"4291",,""upstairspid""::42911"},
"msg"upstairs{":":msg1"}:
"upstairs"":{df6648fa-e480-4dd9-b7c4-7fec45dbddf8 Active Active Active1"",msg"}[2] Transition from WaitActive to WaitQuorum"
"v:"":,0"[1] All repairs completed, exit,{""name",:""v"v:crucible0",,""level"""::name30"0:msg"":,crucible"",""namelevelNo initial repair work was required"":,","":time30"crucible":v""":,0"2023-09-22T23:08:02.429080675Z"level",,"","timehostname""::"":name"30:"ip-10-150-1-74.us-west-2.compute.internal2023-09-22T23:08:02.429094514Z"",",crucible"pid"",hostname:"4291:"",level"":ip-10-150-1-74.us-west-2.compute.internalupstairs"30,"":pid1":}4291
,"upstairs":1{,}""
msg"time":{:"""msg":"Set check for repair"2023-09-22T23:08:02.429110226Z[1] Starts cmd_loop,"""v,"":v0",:"0,name,"""name:":"",cruciblecrucible"",,"""levellevel"hostname"time:"30::30"":"2023-09-22T23:08:02.429120559Z"ip-10-150-1-74.us-west-2.compute.internal",,""pid"hostname"::4291,"",time"":time"ip-10-150-1-74.us-west-2.compute.internal",2023-09-22T23:08:02.429154222Z""":,""upstairs,hostname2023-09-22T23:08:02.42915556Z"":",""pidhostnameip-10-150-1-74.us-west-2.compute.internal"":,""""pid:"ip-10-150-1-74.us-west-2.compute.internal::42914291"1,,,}""pid""upstairsupstairs""::42911:},
1
"}{
"upstairs{msg"""::1msg}""
:[2] new RM replaced this: None"{{"[0] received reconcile message""msg",:""v,"[2] received reconcile message"":v"0",,""namev""::"0:cruciblemsg"",,:""name"":"Set Downstairs and Upstairs active0cruciblelevel"","",:"30,levelname"""::v""30:crucible"0,",,"level"name"::"40",time"crucible"time""::"","level":2023-09-22T23:08:02.429245985Z2023-09-22T23:08:02.429252315Z""30,,""hostname"hostname:"":"ip-10-150-1-74.us-west-2.compute.internal"ip-10-150-1-74.us-west-2.compute.internal",","pid"pid:":42914291,","upstairs"upstairs:"1:}1
},
{"time"{":""msg"msg:"":",2023-09-22T23:08:02.429261283Z""[0] All repairs completed, exit[2] All repairs completed, exit"time"",:,"",v""v""::00,,""hostnamenamename""::"2023-09-22T23:08:02.429267906Z"crucible""crucible",,"",levellevel""::3030"":"hostname":"ip-10-150-1-74.us-west-2.compute.internal"ip-10-150-1-74.us-west-2.compute.internal",",pid"":pid"4291:4291,",,""timetime""::""upstairs",:2023-09-22T23:08:02.429308641Z2023-09-22T23:08:02.429308819Z""",,upstairs"":hostname"11hostname""}::""
ip-10-150-1-74.us-west-2.compute.internalip-10-150-1-74.us-west-2.compute.internal"",,""pid"pid:"}{
":4291msg4291,","{:upstairs"""upstairs:"1"msg}:
1"}{
":msg"{:"ed581aed-4fb6-4c12-84d1-a857283bcda2 is now active with session: 99a7dfa5-d205-4d07-9135-9f9e70387e3a"""[2] Starts reconcile loop",[2] Starts cmd_loopmsg""":",v"",v:"[0] Starts cmd_loop":"00v",,"",vname":"0"::0",crucible"",name,""name"level:"":"30:"crucible"name,"":crucible"level"",:,30crucible"""level,time""::""30level",2023-09-22T23:08:02.429401345Z"":time,"":30hostname"":"2023-09-22T23:08:02.429410517Z","ip-10-150-1-74.us-west-2.compute.internal"hostname,"":"pid":4291,"ip-10-150-1-74.us-west-2.compute.internal"upstairs",:"1pid"}:
4291,"upstairs,"":time1"}:
"{,2023-09-22T23:08:02.429418202Z"""time"msg:"",:""2023-09-22T23:08:02.4294226Z"hostname":[1] received reconcile message","","hostnameip-10-150-1-74.us-west-2.compute.internal""v":,:""pid"0:,4291ip-10-150-1-74.us-west-2.compute.internal"",name"":,"upstairs""crucible"pid,"":level":4291:130,}"
upstairs":1}
{,"time":""msg":{"2023-09-22T23:08:02.429474936Z","hostname":""[0] 127.0.0.1:61368 task reports connection:true"msg":",ip-10-150-1-74.us-west-2.compute.internal"",v"ed581aed-4fb6-4c12-84d1-a857283bcda2 Set Active after no repair:"0"pid",:,4291""vname"","::"0upstairs":crucible1","}name":",
crucible"","level"level{:"30:"30msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.429525699Z",","hostname":"time":,""ip-10-150-1-74.us-west-2.compute.internaltime"2023-09-22T23:08:02.429527512Z:"",""2023-09-22T23:08:02.429535164Zpid,""":,hostname4291"":",hostname"":ip-10-150-1-74.us-west-2.compute.internal"upstairs"",:"1pid":ip-10-150-1-74.us-west-2.compute.internal}4291"
,,""upstairs"pid"{::42911,}""
upstairs"msg":":1}
a2b1abbb-2148-4d24-93b2-2a0a303e7718 WaitQuorum WaitQuorum WaitQuorum"{,"v":{0,""name"msg:"":"crucible"",msg":"Notify all downstairs, region set compare is done."level","":[1] Starts cmd_loop"v"30:,"0v",:"0name":,""crucible"name",:""level"crucible":,"30level":30,"time":"2023-09-22T23:08:02.429610036Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1},
,""time"time":":{"2023-09-22T23:08:02.429621862Z","2023-09-22T23:08:02.429622967Z"hostname"":"msg",:ip-10-150-1-74.us-west-2.compute.internal"""hostname",:""[0]R  flush_numbers: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]"pid":,ip-10-150-1-74.us-west-2.compute.internal"4291"v",,:""0pid":,upstairs"4291":,1name""}:
"upstairs":crucible"1,"}level{":
30"msg":"Set check for repair"{,"v":0,""name"msg"::""crucible","[2] received reconcile message"level":,"30v":0,"name":"crucible",","time"level"::"302023-09-22T23:08:02.429686674Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,,""upstairs"time"::1"}
,2023-09-22T23:08:02.429698833Z""time",{:""hostname":""2023-09-22T23:08:02.429705848Zmsg"":ip-10-150-1-74.us-west-2.compute.internal,"""hostname",[0]R  generation: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]":pid""",:"4291ip-10-150-1-74.us-west-2.compute.internal"v",:,""pid0"upstairs,:""name"4291::"1,"crucible"upstairs"},
":level"1:30}{
"msg":"[1] 127.0.0.1:41466 task reports connection:true"{,"v":0,""msg"name":":"crucible",","[2] All repairs completed, exitlevel"time"":",:"30v":2023-09-22T23:08:02.429765269Z"0,","name":"hostname":crucible"","level":30ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":","time":"2023-09-22T23:08:02.429794083Z"[0]R  dirty: [false, false, false, false, false, false, false, false, false, false]",","hostnamev":"0:,""name":"crucibleip-10-150-1-74.us-west-2.compute.internal"",","level":pid"30:4291,"upstairs":1},
"time":"{2023-09-22T23:08:02.429786605Z",""hostname",msg":"":"time"[2] Starts cmd_loop:"ip-10-150-1-74.us-west-2.compute.internal"",,""v"2023-09-22T23:08:02.429824677Z":pid",0:,4291""hostname"name":,":""upstairs"ip-10-150-1-74.us-west-2.compute.internal":crucible"1,,""pid"}:
4291level":30,"upstairs":1}{
"msg":"{ed581aed-4fb6-4c12-84d1-a857283bcda2 Active Active Active",""msg"v"::,"0","time"name[1]R  flush_numbers: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]""::"","crucible2023-09-22T23:08:02.42987923Z"v",:""0,hostname","":levelname"""::ip-10-150-1-74.us-west-2.compute.internal"30,""pidcrucible"":,4291"level,"":upstairs"30:1}
,"time":"2023-09-22T23:08:02.429915503Z","hostname":","ip-10-150-1-74.us-west-2.compute.internal"time",:""pid":42912023-09-22T23:08:02.429924309Z",","upstairs"hostname"::"1}
ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"{upstairs":1}
"msg":"Set check for repair"{,"v":0,""name"msg"::""crucible","level":[1]R  generation: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]"30,"v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.42997416Z","hostname,"":"time":"ip-10-150-1-74.us-west-2.compute.internal"2023-09-22T23:08:02.429980185Z,""pid",:"4291hostname":","upstairs":1ip-10-150-1-74.us-west-2.compute.internal"},
"pid":4291,"upstairs"{:1}
"msg":"{[2] 127.0.0.1:55647 task reports connection:true","v"":msg"0:","name":"crucible","level":[1]R  dirty: [false, false, false, false, false, false, false, false, false, false]"30,"v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.430030764Z","hostname":,""time":"ip-10-150-1-74.us-west-2.compute.internal","2023-09-22T23:08:02.430036248Zpid"":,4291"hostname",:""upstairs":1}ip-10-150-1-74.us-west-2.compute.internal"
,"pid":4291,"{upstairs":1}
"msg":"{ed581aed-4fb6-4c12-84d1-a857283bcda2 Active Active Active",""msg"v"::"0,"name":"[2]R  flush_numbers: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]"crucible",",v"":level0",:"30name":"crucible","level":30,"time":"2023-09-22T23:08:02.430093784Z,"",time"":"hostname":"2023-09-22T23:08:02.430096672Z",ip-10-150-1-74.us-west-2.compute.internal"","hostname"pid"::"4291,"ip-10-150-1-74.us-west-2.compute.internal"upstairs",:"1pid":}4291
,"upstairs":1}
{{"msg":""Set check for repairmsg"":","v":0,"[2]R  generation: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]"name":","crucible"v":,"0level",:"30name":"crucible","level":30,"time":","2023-09-22T23:08:02.430155178Z"time":","hostname":"2023-09-22T23:08:02.430158275Z","hostnameip-10-150-1-74.us-west-2.compute.internal"",:""pid":4291ip-10-150-1-74.us-west-2.compute.internal,""upstairs",:"1pid":}4291
,"upstairs":1}
{{"msg":""[0] received reconcile messagemsg":"","v":0,"name":"[2]R  dirty: [false, false, false, false, false, false, false, false, false, false]"crucible",","v"level"::030,"name":"crucible","level":30,"time":","2023-09-22T23:08:02.4302157Z"time":,""hostname":"2023-09-22T23:08:02.43021928Z",ip-10-150-1-74.us-west-2.compute.internal"","hostname"pid"::"4291,"upstairs"ip-10-150-1-74.us-west-2.compute.internal":,"1pid":}4291
,"upstairs":1}
{{"msg":""msg"[0] All repairs completed, exit":","v":Max found gen is 1"0,","name"v"::"0crucible",","name"level"::"30crucible","level":30,",time":""time":"2023-09-22T23:08:02.430277798Z"2023-09-22T23:08:02.430279866Z",","hostname":"hostname":"ip-10-150-1-74.us-west-2.compute.internal","ip-10-150-1-74.us-west-2.compute.internal"pid",:"4291pid":4291,",upstairs"":upstairs"1:1}
}
{{"msg":""[0] Starts cmd_loop"msg":","v":0Generation requested: 1 >= found:1",","namev""::0","crucible"name",:""level":crucible"30,"level":30,,""time":time"":"2023-09-22T23:08:02.430341221Z"2023-09-22T23:08:02.430342917Z",",hostname"":"hostname":"ip-10-150-1-74.us-west-2.compute.internal","ip-10-150-1-74.us-west-2.compute.internal"pid",:"4291pid":,4291"upstairs",:"1upstairs":}1
}
{"msg{":"[1] received reconcile message"",msg"":v"":0Next flush: 1",","namev""::"0crucible",","name"level:"":crucible"30,"level":30,"time":"2023-09-22T23:08:02.430406918Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
,"time":"{2023-09-22T23:08:02.430405349Z",""hostname"msg":":"All extents match"ip-10-150-1-74.us-west-2.compute.internal",,""v"pid:"0:,4291"name":,""upstairs"crucible":,"1level":}30
,"time":"{2023-09-22T23:08:02.430457556Z","hostname"":"msg":"ip-10-150-1-74.us-west-2.compute.internal"[1] All repairs completed, exit",",pid":"4291v":0,","upstairs"name"::"1crucible"}
,"level":30{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.430493671Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,","upstairs":time"1:"}
2023-09-22T23:08:02.43050522Z","hostname":"{ip-10-150-1-74.us-west-2.compute.internal","pid":4291"msg":,""upstairs":[1] Starts cmd_loop"1,"}v"
:0,"name":"{crucible","level":30"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.430548376Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"time,"":upstairs"":1}2023-09-22T23:08:02.430557579Z"
,"hostname":"{ip-10-150-1-74.us-west-2.compute.internal","pid":4291","msg":upstairs"":1}[2] received reconcile message"
,"v":0,"{name":"crucible",""level"msg"::"30Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.43060304Z","hostname":","time":ip-10-150-1-74.us-west-2.compute.internal"","pid":2023-09-22T23:08:02.430609946Z"4291,","hostname":"upstairs":1}ip-10-150-1-74.us-west-2.compute.internal"
,"pid":4291,"{upstairs":1}
"msg":"{[2] All repairs completed, exit","v":"0msg",:""name":"crucible","level":30a2b1abbb-2148-4d24-93b2-2a0a303e7718 is now active with session: a7f1d842-f19a-4c81-abcd-c1f78df01722","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.430657605Z","hostname":","time":"ip-10-150-1-74.us-west-2.compute.internal","pid":2023-09-22T23:08:02.430664643Z"4291,",hostname"":"upstairs":1}
ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"{upstairs":1}
"msg":"[2] Starts cmd_loop"{,"v":0,""name"msg:"":"crucible","level":30a2b1abbb-2148-4d24-93b2-2a0a303e7718 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.43070992Z","hostname":","time"ip-10-150-1-74.us-west-2.compute.internal":","pid":42912023-09-22T23:08:02.430716927Z",","upstairs"hostname"::"1}
ip-10-150-1-74.us-west-2.compute.internal","pid":4291,{"upstairs":1}"
msg":"[0] downstairs disconnected"{,"v":0,""msg"name"::""crucible","level":Notify all downstairs, region set compare is done."40,"v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.430765717Z","hostname":","time"ip-10-150-1-74.us-west-2.compute.internal":,""pid":42912023-09-22T23:08:02.430771486Z",","upstairs"hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"Set check for repair","v":0:,"1name":"}
crucible","level":30{"msg":"[0] ed581aed-4fb6-4c12-84d1-a857283bcda2 Gone missing, transition from Active to Offline","v":0,"name":"crucible",",level"":time30":"2023-09-22T23:08:02.430820897Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{,"time"":"msg":"2023-09-22T23:08:02.430838357Z","[1] 127.0.0.1:50978 task reports connection:truehostname"":","v":0ip-10-150-1-74.us-west-2.compute.internal,"","name":pid"":crucible4291",","level"upstairs:"30:1}
{"msg":","time":"[0] ed581aed-4fb6-4c12-84d1-a857283bcda2 connection to 127.0.0.1:60909 closed"2023-09-22T23:08:02.430874485Z",",v"":hostname"0:,""name":"crucible"ip-10-150-1-74.us-west-2.compute.internal",","levelpid""::429130,"upstairs":1}
{"msg":"a2b1abbb-2148-4d24-93b2-2a0a303e7718 Active Active Active",,""v":time":0","name":"2023-09-22T23:08:02.430902788Z"crucible",,""level"hostname"::"30ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"looper":"0","upstairs":1}
,"time":"{2023-09-22T23:08:02.430929747Z",""hostname"msg"::""[0] pm_task rx.recv() is None"ip-10-150-1-74.us-west-2.compute.internal",",v"":pid":04291,"name",":"upstairs":crucible1"},
"level":40{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.430968677Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1,"}time"
:"2023-09-22T23:08:02.430980165Z","{hostname":""msg"ip-10-150-1-74.us-west-2.compute.internal":,""pid":4291[0] 127.0.0.1:60909 task reports connection:false",","upstairs"v"::10},
"name":"crucible","{level":30"msg":"[2] 127.0.0.1:57228 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.431031739Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1},"
time":"2023-09-22T23:08:02.431042511Z{","hostname"":msg"":"ip-10-150-1-74.us-west-2.compute.internal",ed581aed-4fb6-4c12-84d1-a857283bcda2 Offline Active Active"",pid"":v4291":0,","upstairs"name"::1"}crucible"
,"level":30{"msg":"a2b1abbb-2148-4d24-93b2-2a0a303e7718 Active Active Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.431087551Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
,"time":"2023-09-22T23:08:02.431100528Z"{,"hostname":""msg":"ip-10-150-1-74.us-west-2.compute.internal","pid":[0] 127.0.0.1:60909 task reports offline"4291,"v":,0","upstairsname""::"1crucible"},"
level":30{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.431139092Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1},
"time":"2023-09-22T23:08:02.431151503Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.431197247Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.431238627Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.431277332Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.43131669Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.431357679Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.431397001Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.431436829Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.431478124Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.431516027Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
test impacted_blocks::test::extent_from_offset_can_recreate_iblocks ... ok
thread 'impacted_blocks::test::extent_from_offset_panics_for_offsets_outside_region' panicked at 'assertion failed: offset.value < ddef.extent_count() as u64 * extent_size', upstairs/src/impacted_blocks.rs:280:5
note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace
test impacted_blocks::test::iblocks_conflicts_is_commutative ... ok
test impacted_blocks::test::extent_from_offset_panics_for_offsets_outside_region ... ok
test impacted_blocks::test::iblocks_extents_returns_correct_extents ... ok
test impacted_blocks::test::extent_from_offset_panics_when_num_blocks_outside_region ... ok
test impacted_blocks::test::iblocks_from_offset_is_empty_for_zero_blocks ... ok
test impacted_blocks::test::iblocks_from_offset_with_zero_extent_size_panics ... ok
test impacted_blocks::test::intersection_is_associative ... ok
test impacted_blocks::test::intersection_is_commutative ... ok
test impacted_blocks::test::iblocks_new_panics_for_flipped_polarity ... ok
test impacted_blocks::test::intersection_produces_less_than_or_equal_block_count ... ok
test impacted_blocks::test::intersection_with_empty_is_empty ... ok
test impacted_blocks::test::nothing_contains_empty ... ok
test impacted_blocks::test::test_extent_from_offset ... ok
test impacted_blocks::test::test_extent_from_offset_single_block_only ... ok
test impacted_blocks::test::test_extent_to_impacted_blocks ... ok
test impacted_blocks::test::test_impacted_blocks_from_offset ... ok
test impacted_blocks::test::test_large_extent_to_impacted_blocks ... ok
test impacted_blocks::test::test_new_range_panics_when_last_block_before_first - should panic ... ok
test impacted_blocks::test::test_new_range_panics_when_last_extent_before_first - should panic ... ok
test impacted_blocks::test::overlapping_impacted_blocks_should_conflict ... ok
test impacted_blocks::test::union_is_associative ... ok
test impacted_blocks::test::union_is_commutative ... ok
test impacted_blocks::test::iblocks_blocks_iterates_over_all_blocks ... ok
{"msg":"Crucible stats registered with UUID: 8be15b9a-fc42-4ab7-8a35-70c3310bdb05","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.516438939Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 8be15b9a-fc42-4ab7-8a35-70c3310bdb05 has session id: babb90fa-6130-4b9a-9c9b-8915d0effdb2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.516527374Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"8be15b9a-fc42-4ab7-8a35-70c3310bdb05 is now active with session: 9779cae0-6ece-4b8c-8c9e-8d519b6abff7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.516571021Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 8be15b9a-fc42-4ab7-8a35-70c3310bdb05 (9779cae0-6ece-4b8c-8c9e-8d519b6abff7) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.516610036Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.516648346Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 8be15b9a-fc42-4ab7-8a35-70c3310bdb05 (9779cae0-6ece-4b8c-8c9e-8d519b6abff7) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.516689051Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.516724839Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 8be15b9a-fc42-4ab7-8a35-70c3310bdb05 (9779cae0-6ece-4b8c-8c9e-8d519b6abff7) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.516761217Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.5167827Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 8be15b9a-fc42-4ab7-8a35-70c3310bdb05 (9779cae0-6ece-4b8c-8c9e-8d519b6abff7) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.51680354Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.516822067Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 8be15b9a-fc42-4ab7-8a35-70c3310bdb05 (9779cae0-6ece-4b8c-8c9e-8d519b6abff7) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.516842515Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.516860558Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 8be15b9a-fc42-4ab7-8a35-70c3310bdb05 (9779cae0-6ece-4b8c-8c9e-8d519b6abff7) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.516880642Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.516905998Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 8be15b9a-fc42-4ab7-8a35-70c3310bdb05 (9779cae0-6ece-4b8c-8c9e-8d519b6abff7) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.516927422Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.516945436Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 8be15b9a-fc42-4ab7-8a35-70c3310bdb05 (9779cae0-6ece-4b8c-8c9e-8d519b6abff7) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.51696585Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.516984105Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 8be15b9a-fc42-4ab7-8a35-70c3310bdb05 (9779cae0-6ece-4b8c-8c9e-8d519b6abff7) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.517004122Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.517021977Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 8be15b9a-fc42-4ab7-8a35-70c3310bdb05 (9779cae0-6ece-4b8c-8c9e-8d519b6abff7) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.517042102Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.517059926Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 8be15b9a-fc42-4ab7-8a35-70c3310bdb05 (9779cae0-6ece-4b8c-8c9e-8d519b6abff7) Active Faulted Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.51708017Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.5170981Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 8be15b9a-fc42-4ab7-8a35-70c3310bdb05 (9779cae0-6ece-4b8c-8c9e-8d519b6abff7) Active LiveRepairReady Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.517118509Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.517136596Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 8be15b9a-fc42-4ab7-8a35-70c3310bdb05 (9779cae0-6ece-4b8c-8c9e-8d519b6abff7) Active LiveRepair Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.517156725Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.517174605Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 8be15b9a-fc42-4ab7-8a35-70c3310bdb05 (9779cae0-6ece-4b8c-8c9e-8d519b6abff7) Faulted LiveRepair Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.517194989Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.517213047Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.517235846Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Upstairs already in repair, trying again later","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:02.517254679Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test live_repair::repair_test::test_check_for_repair_already_repair ... ok
{"msg":"Crucible stats registered with UUID: 3b20cd5e-ffe3-43dd-abf9-e945e76502a7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.518116661Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 3b20cd5e-ffe3-43dd-abf9-e945e76502a7 has session id: 9ae51c7e-5b1e-4ae4-a036-dc19e88449e9","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.518170645Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"3b20cd5e-ffe3-43dd-abf9-e945e76502a7 is now active with session: b7a9eedf-81a9-4851-b077-0fe235e6b4d0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.518213929Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 3b20cd5e-ffe3-43dd-abf9-e945e76502a7 (b7a9eedf-81a9-4851-b077-0fe235e6b4d0) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.51825436Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.518289691Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 3b20cd5e-ffe3-43dd-abf9-e945e76502a7 (b7a9eedf-81a9-4851-b077-0fe235e6b4d0) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.518329651Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.518363681Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 3b20cd5e-ffe3-43dd-abf9-e945e76502a7 (b7a9eedf-81a9-4851-b077-0fe235e6b4d0) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.518404291Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.518440504Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 3b20cd5e-ffe3-43dd-abf9-e945e76502a7 (b7a9eedf-81a9-4851-b077-0fe235e6b4d0) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.518470724Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.518489449Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 3b20cd5e-ffe3-43dd-abf9-e945e76502a7 (b7a9eedf-81a9-4851-b077-0fe235e6b4d0) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.518509884Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.518528046Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 3b20cd5e-ffe3-43dd-abf9-e945e76502a7 (b7a9eedf-81a9-4851-b077-0fe235e6b4d0) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.518548067Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.518565826Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 3b20cd5e-ffe3-43dd-abf9-e945e76502a7 (b7a9eedf-81a9-4851-b077-0fe235e6b4d0) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.518591852Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.518610928Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 3b20cd5e-ffe3-43dd-abf9-e945e76502a7 (b7a9eedf-81a9-4851-b077-0fe235e6b4d0) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.518631408Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.518649487Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 3b20cd5e-ffe3-43dd-abf9-e945e76502a7 (b7a9eedf-81a9-4851-b077-0fe235e6b4d0) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.51866952Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.518687603Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 3b20cd5e-ffe3-43dd-abf9-e945e76502a7 (b7a9eedf-81a9-4851-b077-0fe235e6b4d0) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.518707514Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.518725417Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 3b20cd5e-ffe3-43dd-abf9-e945e76502a7 (b7a9eedf-81a9-4851-b077-0fe235e6b4d0) Active Faulted Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.518745692Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.518763776Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.518781328Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 3b20cd5e-ffe3-43dd-abf9-e945e76502a7 (b7a9eedf-81a9-4851-b077-0fe235e6b4d0) Active LiveRepairReady Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.518801766Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.518820101Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test live_repair::repair_test::test_check_for_repair_do_repair ... ok
{"msg":"Crucible stats registered with UUID: 25bf7e8b-971c-4e8e-9e32-9d598106e189","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.519633275Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 25bf7e8b-971c-4e8e-9e32-9d598106e189 has session id: e8f7fcdc-3b78-4df0-aa00-a5795a260b2c","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.519687149Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"25bf7e8b-971c-4e8e-9e32-9d598106e189 is now active with session: 88fb9508-2fa9-4920-81d3-845a3885e04c","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.519730369Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 25bf7e8b-971c-4e8e-9e32-9d598106e189 (88fb9508-2fa9-4920-81d3-845a3885e04c) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.519770252Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.519813247Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 25bf7e8b-971c-4e8e-9e32-9d598106e189 (88fb9508-2fa9-4920-81d3-845a3885e04c) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.519853172Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.51989357Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 25bf7e8b-971c-4e8e-9e32-9d598106e189 (88fb9508-2fa9-4920-81d3-845a3885e04c) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.519934405Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.519961191Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 25bf7e8b-971c-4e8e-9e32-9d598106e189 (88fb9508-2fa9-4920-81d3-845a3885e04c) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.519982538Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.52001594Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 25bf7e8b-971c-4e8e-9e32-9d598106e189 (88fb9508-2fa9-4920-81d3-845a3885e04c) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.520043554Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","test impacted_blocks::test::subregions_are_contained ... pid":ok4291}

{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.520083541Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 25bf7e8b-971c-4e8e-9e32-9d598106e189 (88fb9508-2fa9-4920-81d3-845a3885e04c) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.520123671Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.520156585Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 25bf7e8b-971c-4e8e-9e32-9d598106e189 (88fb9508-2fa9-4920-81d3-845a3885e04c) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.520179467Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.520197568Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 25bf7e8b-971c-4e8e-9e32-9d598106e189 (88fb9508-2fa9-4920-81d3-845a3885e04c) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.520218009Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.520236061Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 25bf7e8b-971c-4e8e-9e32-9d598106e189 (88fb9508-2fa9-4920-81d3-845a3885e04c) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.520256129Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.520273964Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 25bf7e8b-971c-4e8e-9e32-9d598106e189 (88fb9508-2fa9-4920-81d3-845a3885e04c) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.520294106Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.520317686Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 25bf7e8b-971c-4e8e-9e32-9d598106e189 (88fb9508-2fa9-4920-81d3-845a3885e04c) Active Faulted Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.52033933Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.520365421Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 25bf7e8b-971c-4e8e-9e32-9d598106e189 (88fb9508-2fa9-4920-81d3-845a3885e04c) Active LiveRepairReady Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.520388778Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.520406652Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 25bf7e8b-971c-4e8e-9e32-9d598106e189 (88fb9508-2fa9-4920-81d3-845a3885e04c) Active LiveRepairReady Faulted ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.520426952Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.520444972Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.520462567Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 25bf7e8b-971c-4e8e-9e32-9d598106e189 (88fb9508-2fa9-4920-81d3-845a3885e04c) Active LiveRepairReady LiveRepairReady ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.520484232Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.520502736Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 25bf7e8b-971c-4e8e-9e32-9d598106e189 (88fb9508-2fa9-4920-81d3-845a3885e04c) Active LiveRepair LiveRepairReady ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.520523139Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.520541199Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible stats registered with UUID: 46e3c056-00ff-4d1a-9e2a-764650276e46","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.52076919Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test live_repair::repair_test::test_check_for_repair_do_two_repair ... {ok
"msg":"Crucible 46e3c056-00ff-4d1a-9e2a-764650276e46 has session id: b40b4606-8bb3-454d-9ded-0c9e3424390e","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.520834857Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.520874263Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"46e3c056-00ff-4d1a-9e2a-764650276e46 is now active with session: 89be093f-a3e6-4c47-9b32-a64289baafd3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.520915216Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 46e3c056-00ff-4d1a-9e2a-764650276e46 (89be093f-a3e6-4c47-9b32-a64289baafd3) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.5209564Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.521003266Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 46e3c056-00ff-4d1a-9e2a-764650276e46 (89be093f-a3e6-4c47-9b32-a64289baafd3) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.521045459Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.521081737Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 46e3c056-00ff-4d1a-9e2a-764650276e46 (89be093f-a3e6-4c47-9b32-a64289baafd3) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.52112095Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.52115477Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 46e3c056-00ff-4d1a-9e2a-764650276e46 (89be093f-a3e6-4c47-9b32-a64289baafd3) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.521194336Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.521230438Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 46e3c056-00ff-4d1a-9e2a-764650276e46 (89be093f-a3e6-4c47-9b32-a64289baafd3) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.521265297Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.521301411Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 46e3c056-00ff-4d1a-9e2a-764650276e46 (89be093f-a3e6-4c47-9b32-a64289baafd3) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.52134202Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30{"msg":","time":"Crucible stats registered with UUID: ca3d570b-0142-4846-8b94-8a2df932685d"2023-09-22T23:08:02.521379103Z",,""v"hostname"::"0,"name"ip-10-150-1-74.us-west-2.compute.internal":,""crucible"pid":,4291"level"}:
30{"msg":"[2] 46e3c056-00ff-4d1a-9e2a-764650276e46 (89be093f-a3e6-4c47-9b32-a64289baafd3) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.521411432Z","hostname":","time":"ip-10-150-1-74.us-west-2.compute.internal"2023-09-22T23:08:02.521426851Z",",pid"":hostname":4291"}
ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{{"msg":""msg":"[2] Transition from New to WaitActive","v"Crucible ca3d570b-0142-4846-8b94-8a2df932685d has session id: ed63297a-d753-4422-aa8e-3627007c4617":0,,""v"name"::"0crucible",","name":"level":crucible"30,"level":30,"time":",2023-09-22T23:08:02.521482552Z"","time":hostname"":"2023-09-22T23:08:02.521484396Z"ip-10-150-1-74.us-west-2.compute.internal",,""hostname"pid"::"4291}
ip-10-150-1-74.us-west-2.compute.internal","pid":4291{}
"msg":"{"msg":"[2] 46e3c056-00ff-4d1a-9e2a-764650276e46 (89be093f-a3e6-4c47-9b32-a64289baafd3) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible",ca3d570b-0142-4846-8b94-8a2df932685d is now active with session: 4ec111d6-91eb-4bad-893c-0cab18122c49"",level""v:"30:0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.521538546Z","hostname",:""time":"ip-10-150-1-74.us-west-2.compute.internal","2023-09-22T23:08:02.521543293Z"pid":,"4291hostname":"}
ip-10-150-1-74.us-west-2.compute.internal","pid":4291{}
"msg":"{[2] Transition from WaitActive to WaitQuorum","v"":msg0",":name"":"crucible","level":30[0] ca3d570b-0142-4846-8b94-8a2df932685d (4ec111d6-91eb-4bad-893c-0cab18122c49) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.521592855Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"time}"
:"2023-09-22T23:08:02.521601426Z"{,"hostname":""msg":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
[2] 46e3c056-00ff-4d1a-9e2a-764650276e46 (89be093f-a3e6-4c47-9b32-a64289baafd3) Active Active WaitQuorum ds_transition to Active","v":{0,"name":"crucible"",msg"":level":"30[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.521642954Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":","[2] Transition from WaitQuorum to Active"time":,""v":0,"name":"2023-09-22T23:08:02.521650625Zcrucible"","level,":"30hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
,"time":"2023-09-22T23:08:02.521672475Z","{hostname":"ip-10-150-1-74.us-west-2.compute.internal",""pid":msg"4291:"}
{"msg":"Checking if live repair is needed",[0] ca3d570b-0142-4846-8b94-8a2df932685d (4ec111d6-91eb-4bad-893c-0cab18122c49) WaitActive New New ds_transition to WaitQuorum""v",":v0",":name0":,""crucible",name"":level":"30crucible","level":30,"time":"2023-09-22T23:08:02.521705874Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"No Live Repair required at this time",","v":time"0:,""name":"crucible","level":2023-09-22T23:08:02.521708559Z"30,"hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"}time":"
2023-09-22T23:08:02.521728857Z","hostname":"{ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.521755732Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] ca3d570b-0142-4846-8b94-8a2df932685d (4ec111d6-91eb-4bad-893c-0cab18122c49) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.521797509Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.521835536Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] ca3d570b-0142-4846-8b94-8a2df932685d (4ec111d6-91eb-4bad-893c-0cab18122c49) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.521876905Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.521914642Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] ca3d570b-0142-4846-8b94-8a2df932685d (4ec111d6-91eb-4bad-893c-0cab18122c49) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.521955642Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.521993983Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] ca3d570b-0142-4846-8b94-8a2df932685d (4ec111d6-91eb-4bad-893c-0cab18122c49) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.522035738Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.522073317Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test live_repair::repair_test::test_check_for_repair_normal ... {ok"
msg":"[2] ca3d570b-0142-4846-8b94-8a2df932685d (4ec111d6-91eb-4bad-893c-0cab18122c49) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.522122215Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.522179216Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] ca3d570b-0142-4846-8b94-8a2df932685d (4ec111d6-91eb-4bad-893c-0cab18122c49) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.522222627Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.522260701Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] ca3d570b-0142-4846-8b94-8a2df932685d (4ec111d6-91eb-4bad-893c-0cab18122c49) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.522302351Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.522340949Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] ca3d570b-0142-4846-8b94-8a2df932685d (4ec111d6-91eb-4bad-893c-0cab18122c49) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.522382503Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.52242025Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] ca3d570b-0142-4846-8b94-8a2df932685d (4ec111d6-91eb-4bad-893c-0cab18122c49) Active Faulted Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.522462136Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.522500549Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.52253788Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Upstairs repair task running, trying again later","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:02.52257574Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible stats registered with UUID: 799e9fee-e865-4dfc-bb55-9cc2f161f350","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.522716225Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 799e9fee-e865-4dfc-bb55-9cc2f161f350 has session id: 71e074b3-11f6-4d45-abac-dffe02ff0dc6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.522774102Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 799e9fee-e865-4dfc-bb55-9cc2f161f350 (85cf2e10-e0ba-47f8-95d8-3a05a2ce5d33) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"test live_repair::repair_test::test_check_for_repair_task_running ... 2023-09-22T23:08:02.522840832Zok",
"hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.522897259Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 799e9fee-e865-4dfc-bb55-9cc2f161f350 (85cf2e10-e0ba-47f8-95d8-3a05a2ce5d33) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.522942857Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.522982544Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 799e9fee-e865-4dfc-bb55-9cc2f161f350 (85cf2e10-e0ba-47f8-95d8-3a05a2ce5d33) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.523032632Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.523074187Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 799e9fee-e865-4dfc-bb55-9cc2f161f350 (85cf2e10-e0ba-47f8-95d8-3a05a2ce5d33) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.523117258Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.523157321Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 799e9fee-e865-4dfc-bb55-9cc2f161f350 (85cf2e10-e0ba-47f8-95d8-3a05a2ce5d33) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.523200682Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.523241307Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 799e9fee-e865-4dfc-bb55-9cc2f161f350 (85cf2e10-e0ba-47f8-95d8-3a05a2ce5d33) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.523285122Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30{,"time"":"msg":"2023-09-22T23:08:02.523323598Z","hostname":"Crucible stats registered with UUID: 07959179-ab76-4a4b-8cb7-dfeed5f1ae35"ip-10-150-1-74.us-west-2.compute.internal",",pid":"4291v":}0
,"name":"crucible"{,"level":30"msg":"[2] 799e9fee-e865-4dfc-bb55-9cc2f161f350 (85cf2e10-e0ba-47f8-95d8-3a05a2ce5d33) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.523357752Z",,""hostname"time":":"2023-09-22T23:08:02.523369499Z"ip-10-150-1-74.us-west-2.compute.internal",,""hostname"pid"::"4291ip-10-150-1-74.us-west-2.compute.internal}"
,"pid":4291}
{{"msg":""msg":"[2] Transition from New to WaitActive"Crucible 07959179-ab76-4a4b-8cb7-dfeed5f1ae35 has session id: 0a0958a3-354c-47a2-a637-e70cf90da86b","v,"":v"0:,"0name":,""crucible"name":","crucible"level":,"30level":30,"time":"2023-09-22T23:08:02.523430224Z",","time":hostname"":"2023-09-22T23:08:02.523431417Z"ip-10-150-1-74.us-west-2.compute.internal,"","hostnamepid""::"4291}
ip-10-150-1-74.us-west-2.compute.internal","pid":4291{}
"msg":"{"[2] 799e9fee-e865-4dfc-bb55-9cc2f161f350 (85cf2e10-e0ba-47f8-95d8-3a05a2ce5d33) Active Active WaitActive ds_transition to WaitQuorum"msg":","v":0,"name":"crucible","level":[0] 07959179-ab76-4a4b-8cb7-dfeed5f1ae35 (e64c863e-d0f0-4d25-a523-4286ae993f56) New New New ds_transition to WaitActive"30,"v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.52348674Z",,""hostname":"time":"ip-10-150-1-74.us-west-2.compute.internal"2023-09-22T23:08:02.523492412Z,""pid":,4291"hostname"}:
"ip-10-150-1-74.us-west-2.compute.internal","{pid":4291}
"msg":"{[2] Transition from WaitActive to WaitQuorum","v":"0msg,"":"name":"crucible"[0] Transition from New to WaitActive",",level""v"::300,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.523544863Z",,""time":"hostname":"2023-09-22T23:08:02.523548986Z","ip-10-150-1-74.us-west-2.compute.internal"hostname":,""pid":4291ip-10-150-1-74.us-west-2.compute.internal",}"
pid":4291}
{"{msg":""msg":"[2] 799e9fee-e865-4dfc-bb55-9cc2f161f350 (85cf2e10-e0ba-47f8-95d8-3a05a2ce5d33) Active Active WaitQuorum ds_transition to Active","v":0,"[0] 07959179-ab76-4a4b-8cb7-dfeed5f1ae35 (e64c863e-d0f0-4d25-a523-4286ae993f56) WaitActive New New ds_transition to WaitQuorum"name":","crucible"v",:"0level",:"30name":"crucible","level":30,"time":"2023-09-22T23:08:02.523605867Z",","hostname":time"":"ip-10-150-1-74.us-west-2.compute.internal2023-09-22T23:08:02.523609414Z"",",pid"":hostname"4291:"}
ip-10-150-1-74.us-west-2.compute.internal","pid":4291}{
"test impacted_blocks::test::union_with_empty_is_identity ... msg"{:ok
""msg":"[2] Transition from WaitQuorum to Active","v"[0] Transition from WaitActive to WaitQuorum":,0",v"":name":0","crucible"name":,""cruciblelevel"":,30"level":30,",time":""time":"2023-09-22T23:08:02.523682694Z"2023-09-22T23:08:02.523684257Z,"",hostname":""hostname":"ip-10-150-1-74.us-west-2.compute.internal","ip-10-150-1-74.us-west-2.compute.internal"pid,""pid"::42914291}}

{"msg":{""msg":"[0] 07959179-ab76-4a4b-8cb7-dfeed5f1ae35 (e64c863e-d0f0-4d25-a523-4286ae993f56) WaitQuorum New New ds_transition to Active","v":0,"799e9fee-e865-4dfc-bb55-9cc2f161f350 is now active with session: 85cf2e10-e0ba-47f8-95d8-3a05a2ce5d33name"":","crucible"v":,"0level",:"30name":"crucible","level":30,"time":"2023-09-22T23:08:02.523746955Z","hostname,"":"time":"ip-10-150-1-74.us-west-2.compute.internal"2023-09-22T23:08:02.523750497Z",",pid"":hostname"4291:"}
ip-10-150-1-74.us-west-2.compute.internal","pid":4291}{
{"msg":"[1] 799e9fee-e865-4dfc-bb55-9cc2f161f350 (85cf2e10-e0ba-47f8-95d8-3a05a2ce5d33) Active Active Active ds_transition to Faulted","v":0","name":"msgcrucible"",":level"":30[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.523867192Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
,"time":"2023-09-22T23:08:02.523876375Z"{,"hostname":""msg":"ip-10-150-1-74.us-west-2.compute.internal","pid":[1] Transition from Active to Faulted"4291,"v":}0
,"name":"crucible","level":30{"msg":"[1] 07959179-ab76-4a4b-8cb7-dfeed5f1ae35 (e64c863e-d0f0-4d25-a523-4286ae993f56) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":,30"time":"2023-09-22T23:08:02.523919903Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{,""time"msg"::""2023-09-22T23:08:02.523935465Z","hostname":"[1] 799e9fee-e865-4dfc-bb55-9cc2f161f350 (85cf2e10-e0ba-47f8-95d8-3a05a2ce5d33) Active Faulted Active ds_transition to LiveRepairReady"ip-10-150-1-74.us-west-2.compute.internal",",v"":pid"0:,"4291name":"}crucible"
,"level":30{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.523973926Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
,"time":"2023-09-22T23:08:02.523986307Z"{,"hostname":""msg":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291[1] Transition from Faulted to LiveRepairReady"},
"v":0,"name":"{crucible","level":30"msg":"[1] 07959179-ab76-4a4b-8cb7-dfeed5f1ae35 (e64c863e-d0f0-4d25-a523-4286ae993f56) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.5240223Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
,"time":"2023-09-22T23:08:02.52403464Z{","hostname":""msg":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
[1] 799e9fee-e865-4dfc-bb55-9cc2f161f350 (85cf2e10-e0ba-47f8-95d8-3a05a2ce5d33) Active LiveRepairReady Active ds_transition to LiveRepair","v":0,"{name":"crucible",""level"msg"::"30[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.524074124Z","hostname":","time"ip-10-150-1-74.us-west-2.compute.internal":,""pid":42912023-09-22T23:08:02.524081427Z"},
"hostname":"{ip-10-150-1-74.us-west-2.compute.internal","pid":4291"msg":}"
[1] Transition from LiveRepairReady to LiveRepair","v":0,"{name":"crucible",""level":msg"30:"[1] 07959179-ab76-4a4b-8cb7-dfeed5f1ae35 (e64c863e-d0f0-4d25-a523-4286ae993f56) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.524124008Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
,"time":"2023-09-22T23:08:02.524135423Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.524185831Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 07959179-ab76-4a4b-8cb7-dfeed5f1ae35 (e64c863e-d0f0-4d25-a523-4286ae993f56) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"{time":"2023-09-22T23:08:02.524230385Z","hostname":""msg"ip-10-150-1-74.us-west-2.compute.internal":,""pid":4291}
Crucible stats registered with UUID: 1da14c8a-8793-41f8-a35d-f1956ee49350","v":{0,"name":""cruciblemsg"":,""level":[2] Transition from New to WaitActive"30,"v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.524274762Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
,"{time":""msg":2023-09-22T23:08:02.524270377Z"","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid"[2] 07959179-ab76-4a4b-8cb7-dfeed5f1ae35 (e64c863e-d0f0-4d25-a523-4286ae993f56) Active Active WaitActive ds_transition to WaitQuorum":,"4291v":0,"}name"
:"crucible","level":30{"msg":"Crucible 1da14c8a-8793-41f8-a35d-f1956ee49350 has session id: 29e54210-8611-44f9-9caa-a5f3863d338c","v":0,"name":"crucible",","level":time"30:"2023-09-22T23:08:02.524325888Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{,"time":""msg":2023-09-22T23:08:02.52434194Z"","hostname":"[2] Transition from WaitActive to WaitQuorum","v":ip-10-150-1-74.us-west-2.compute.internal"0,",pid"":name"4291:"crucible"},
"level":30{"msg":"[0] 1da14c8a-8793-41f8-a35d-f1956ee49350 (1a0bf2e2-a8c9-482a-a779-90cd6a9523de) New New New ds_transition to WaitActive","v":0,"nametest live_repair::repair_test::test_live_repair_deps_after_no_overlap ... ":,ok""
crucibletime"":","level":2023-09-22T23:08:02.524377228Z"30,"hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{,"time":""msg":"2023-09-22T23:08:02.524409929Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","[2] 07959179-ab76-4a4b-8cb7-dfeed5f1ae35 (e64c863e-d0f0-4d25-a523-4286ae993f56) Active Active WaitQuorum ds_transition to Activepid"":4291,"v":}0
,"name":"crucible","{level":30"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.524443363Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
,"time":"{2023-09-22T23:08:02.524453436Z","hostname":""msg":"ip-10-150-1-74.us-west-2.compute.internal"[2] Transition from WaitQuorum to Active",",pid"":v"4291:0,}"
name":"crucible","level":{30"msg":"[0] 1da14c8a-8793-41f8-a35d-f1956ee49350 (1a0bf2e2-a8c9-482a-a779-90cd6a9523de) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.524491447Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
,"time":"{2023-09-22T23:08:02.524505994Z",""hostnamemsg":"":"ip-10-150-1-74.us-west-2.compute.internal","pid":429107959179-ab76-4a4b-8cb7-dfeed5f1ae35 is now active with session: e64c863e-d0f0-4d25-a523-4286ae993f56"}
,"v":0,"name":"{crucible","level":30"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.524543465Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid",:"4291time":"}
2023-09-22T23:08:02.524552289Z","hostname":"{ip-10-150-1-74.us-west-2.compute.internal","pid":"4291msg":"}
[1] 07959179-ab76-4a4b-8cb7-dfeed5f1ae35 (e64c863e-d0f0-4d25-a523-4286ae993f56) Active Active Active ds_transition to Faulted","v":0{,"name":"crucible",""level"msg"::"30[0] 1da14c8a-8793-41f8-a35d-f1956ee49350 (1a0bf2e2-a8c9-482a-a779-90cd6a9523de) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.524594479Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
,"time":"2023-09-22T23:08:02.524605152Z"{,"hostname":""msg":"ip-10-150-1-74.us-west-2.compute.internal","pid":[1] Transition from Active to Faulted"4291,"v":}0
,"name":"crucible","{level":30"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.524642977Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
,"time":"2023-09-22T23:08:02.524652267Z"{,"hostname":""msgip-10-150-1-74.us-west-2.compute.internal"",:""pid":4291}
[1] 07959179-ab76-4a4b-8cb7-dfeed5f1ae35 (e64c863e-d0f0-4d25-a523-4286ae993f56) Active Faulted Active ds_transition to LiveRepairReady","v":0{,"name":"crucible"",msg":""level":30[1] 1da14c8a-8793-41f8-a35d-f1956ee49350 (1a0bf2e2-a8c9-482a-a779-90cd6a9523de) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.524701444Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,}"
time":"2023-09-22T23:08:02.524711114Z"{,"hostname":""msg":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291[1] Transition from Faulted to LiveRepairReady",}"
v":0,"name":"{crucible","level":"30msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.524750377Z","hostname":","timeip-10-150-1-74.us-west-2.compute.internal"",:""pid":2023-09-22T23:08:02.524758101Z"4291,"hostname":"}
ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{{""msg"msg":":"[1] 1da14c8a-8793-41f8-a35d-f1956ee49350 (1a0bf2e2-a8c9-482a-a779-90cd6a9523de) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":[1] 07959179-ab76-4a4b-8cb7-dfeed5f1ae35 (e64c863e-d0f0-4d25-a523-4286ae993f56) Active LiveRepairReady Active ds_transition to LiveRepair"30,"v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.524797039Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitActive to WaitQuorum,"","timev""::0","name":"2023-09-22T23:08:02.524803168Z"crucible",,""level"hostname"::30"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
,"{time":"2023-09-22T23:08:02.524830127Z"",msg"":hostname"":{"[1] Transition from LiveRepairReady to LiveRepairip-10-150-1-74.us-west-2.compute.internal""",",msgpid""::""4291v":}0
Crucible stats registered with UUID: 8bd67f2e-1fde-4913-85f8-577d1b57f202","name":{,"""vmsg""::crucible"0",","level"name"::"30crucible","[1] 1da14c8a-8793-41f8-a35d-f1956ee49350 (1a0bf2e2-a8c9-482a-a779-90cd6a9523de) Active WaitQuorum New ds_transition to Activelevel"":,"v30":0,"name":"crucible","level":30,"time":",2023-09-22T23:08:02.524870791Z""time":,"","time"2023-09-22T23:08:02.524877159Z:""hostname,""2023-09-22T23:08:02.524873977Z:hostname"":,"""hostname":ip-10-150-1-74.us-west-2.compute.internal"","ip-10-150-1-74.us-west-2.compute.internal"ip-10-150-1-74.us-west-2.compute.internalpid,"",:"4291"pid}pid
""::{42914291}"}msg
":
{""[1] Transition from WaitQuorum to Activemsg"":","v":0,"name":"crucible","Crucible 8bd67f2e-1fde-4913-85f8-577d1b57f202 has session id: 9c94dd0d-4edf-440f-b6fa-7bb2f8c5b2e6level"":,30"v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.524929557Z",","hostname"time:"":"2023-09-22T23:08:02.52493391Z"ip-10-150-1-74.us-west-2.compute.internal",,""pidhostname""::"4291}
ip-10-150-1-74.us-west-2.compute.internal","pid":{4291}
"msg":"{"msg":"[2] 1da14c8a-8793-41f8-a35d-f1956ee49350 (1a0bf2e2-a8c9-482a-a779-90cd6a9523de) Active Active New ds_transition to WaitActive","v":0,"[0] 8bd67f2e-1fde-4913-85f8-577d1b57f202 (7024e503-4e36-4db3-924a-6f1bc0972c72) New New New ds_transition to WaitActive"name":,""v"crucible:"0,,""levelname""::"30crucible","level":30,"time":,""time":"2023-09-22T23:08:02.524969316Z","2023-09-22T23:08:02.524971256Z"hostname",:""hostname":"ip-10-150-1-74.us-west-2.compute.internal","ip-10-150-1-74.us-west-2.compute.internalpid"":,"4291pid":}4291
}
{"{msg":""msg":"[2] Transition from New to WaitActive","v":[0] Transition from New to WaitActive"0,,""vname""::0","crucible"name",:""level"crucible:"30,"level":30,",time"":"time":"2023-09-22T23:08:02.525001901Z"2023-09-22T23:08:02.525003136Z,"","hostname":hostname"":"ip-10-150-1-74.us-west-2.compute.internal",ip-10-150-1-74.us-west-2.compute.internal""pid,"":pid4291":4291}
}
{"{msg":""msg":"[2] 1da14c8a-8793-41f8-a35d-f1956ee49350 (1a0bf2e2-a8c9-482a-a779-90cd6a9523de) Active Active WaitActive ds_transition to WaitQuorum",[0] 8bd67f2e-1fde-4913-85f8-577d1b57f202 (7024e503-4e36-4db3-924a-6f1bc0972c72) WaitActive New New ds_transition to WaitQuorum""v",:"0v",:"0,name"":"name":crucible"",crucible"",level"":level30":30,,""timetime""::""2023-09-22T23:08:02.525039524Z2023-09-22T23:08:02.52503909Z"",,""hostnamehostname""::""ip-10-150-1-74.us-west-2.compute.internalip-10-150-1-74.us-west-2.compute.internal"",,""pidpid""::42914291}}

{{"msg"":"msg":"[2] Transition from WaitActive to WaitQuorum"[0] Transition from WaitActive to WaitQuorum,""v,"":v"0:,0","name":name"":"crucible"crucible,"","level":level"30:30,,""timetime""::""2023-09-22T23:08:02.525080441Z2023-09-22T23:08:02.525080778Z"",,""hostnamehostname""::""ip-10-150-1-74.us-west-2.compute.internalip-10-150-1-74.us-west-2.compute.internal"",,""pidpid""::42914291}}

{"{msg":""msg":"[0] 8bd67f2e-1fde-4913-85f8-577d1b57f202 (7024e503-4e36-4db3-924a-6f1bc0972c72) WaitQuorum New New ds_transition to Active","v"[2] 1da14c8a-8793-41f8-a35d-f1956ee49350 (1a0bf2e2-a8c9-482a-a779-90cd6a9523de) Active Active WaitQuorum ds_transition to Active:"0,,""v"name:"0:","crucible"name",:""level"crucible:"30,"level":30,",time"test live_repair::repair_test::test_live_repair_deps_flush_repair_flush ... time""::""2023-09-22T23:08:02.525120608Z2023-09-22T23:08:02.52511934Z""ok,,""
hostnamehostname""::""ip-10-150-1-74.us-west-2.compute.internalip-10-150-1-74.us-west-2.compute.internal"",,""pidpid""::42914291}}

{"{msg":""msg":"[0] Transition from WaitQuorum to Active","[2] Transition from WaitQuorum to Activev"":,0",v"":name0":,""cruciblename"":,""levelcrucible"":,30"level":30,","time"time:"":"2023-09-22T23:08:02.52516768Z"2023-09-22T23:08:02.525168756Z",","hostname":hostname"":"ip-10-150-1-74.us-west-2.compute.internal"ip-10-150-1-74.us-west-2.compute.internal,"","pid"pid:":42914291}}

{"{msg":""msg":"[1] 8bd67f2e-1fde-4913-85f8-577d1b57f202 (7024e503-4e36-4db3-924a-6f1bc0972c72) Active New New ds_transition to WaitActive"1da14c8a-8793-41f8-a35d-f1956ee49350 is now active with session: 1a0bf2e2-a8c9-482a-a779-90cd6a9523de",",v"":v0":,"0,name"":"name":"crucible"crucible,"","level":level30":30,,""timetime""::""2023-09-22T23:08:02.525204481Z2023-09-22T23:08:02.525204004Z"",,""hostnamehostname""::""ip-10-150-1-74.us-west-2.compute.internal"ip-10-150-1-74.us-west-2.compute.internal,"","pid"pid:":42914291}}

{"{msg":""msg":"[1] Transition from New to WaitActive","v":0,"[1] 1da14c8a-8793-41f8-a35d-f1956ee49350 (1a0bf2e2-a8c9-482a-a779-90cd6a9523de) Active Active Active ds_transition to Faulted"name":,""v"crucible:"0,,""levelname""::"30crucible","level":30,"time":","time":"2023-09-22T23:08:02.525238878Z","2023-09-22T23:08:02.525240943Z"hostname",:""hostname":"ip-10-150-1-74.us-west-2.compute.internal","ip-10-150-1-74.us-west-2.compute.internalpid"":,"4291pid":}4291
}
{"{msg":""msg":"[1] Transition from Active to Faulted","v":0,"name[1] 8bd67f2e-1fde-4913-85f8-577d1b57f202 (7024e503-4e36-4db3-924a-6f1bc0972c72) Active WaitActive New ds_transition to WaitQuorum"":","cruciblev"":,0","level":name"30:"crucible","level":30,"time":","time"2023-09-22T23:08:02.525272924Z:"","hostname":2023-09-22T23:08:02.525275347Z"","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291ip-10-150-1-74.us-west-2.compute.internal","}
pid":4291}{
"msg":"{"msg":"[1] Transition from WaitActive to WaitQuorum","v":[1] 1da14c8a-8793-41f8-a35d-f1956ee49350 (1a0bf2e2-a8c9-482a-a779-90cd6a9523de) Active Faulted Active ds_transition to LiveRepairReady0",","name"v:"":0crucible,"","name":level"":crucible30","level":30,"time":","time":"2023-09-22T23:08:02.525307795Z","2023-09-22T23:08:02.525309366Zhostname"":,""hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid"ip-10-150-1-74.us-west-2.compute.internal:"4291,"pid}"
:4291}
{"{msg":""msg":"[1] Transition from Faulted to LiveRepairReady","v":0,"[1] 8bd67f2e-1fde-4913-85f8-577d1b57f202 (7024e503-4e36-4db3-924a-6f1bc0972c72) Active WaitQuorum New ds_transition to Active"name":,""v"crucible:"0,,""levelname""::"30crucible","level":30,"time":","time":"2023-09-22T23:08:02.525342218Z","2023-09-22T23:08:02.525344264Zhostname"":","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":ip-10-150-1-74.us-west-2.compute.internal"4291,"pid}"
:4291}
{{"msg":""msg":"[1] Transition from WaitQuorum to Active","v":0,"name":[1] 1da14c8a-8793-41f8-a35d-f1956ee49350 (1a0bf2e2-a8c9-482a-a779-90cd6a9523de) Active LiveRepairReady Active ds_transition to LiveRepair"",crucible""v",:"0level,"":30name":"crucible","level":30,"time":","2023-09-22T23:08:02.525381386Z"time":,""hostname":"2023-09-22T23:08:02.525384125Z","hostname"ip-10-150-1-74.us-west-2.compute.internal:"","pid":4291ip-10-150-1-74.us-west-2.compute.internal}"
,"pid":4291{}
"msg":"{"msg":"[1] Transition from LiveRepairReady to LiveRepair","[2] 8bd67f2e-1fde-4913-85f8-577d1b57f202 (7024e503-4e36-4db3-924a-6f1bc0972c72) Active Active New ds_transition to WaitActive"v":,"0v,"":0name",:""name"crucible:"","crucible"level",:"30level":30,,""timetime""::""2023-09-22T23:08:02.525417405Z2023-09-22T23:08:02.525418188Z"",,""hostnamehostname""::""ip-10-150-1-74.us-west-2.compute.internalip-10-150-1-74.us-west-2.compute.internal"",,""pidpid""::42914291}}

{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.525449437Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 8bd67f2e-1fde-4913-85f8-577d1b57f202 (7024e503-4e36-4db3-924a-6f1bc0972c72) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.525486638Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.525517929Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 8bd67f2e-1fde-4913-85f8-577d1b57f202 (7024e503-4e36-4db3-924a-6f1bc0972c72) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.525539704Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.525557648Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"8bd67f2e-1fde-4913-85f8-577d1b57f202 is now active with session: 7024e503-4e36-4db3-924a-6f1bc0972c72","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.525578431Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{{"msg"":"msg":"Crucible stats registered with UUID: 53aa6152-9858-456d-bc14-bd6a95d97fdd","[1] 8bd67f2e-1fde-4913-85f8-577d1b57f202 (7024e503-4e36-4db3-924a-6f1bc0972c72) Active Active Active ds_transition to Faulted"v",:"v0":,0","name":"name":"crucible"crucible",","level"level:":3030,"time":","time":"2023-09-22T23:08:02.525615192Z","2023-09-22T23:08:02.525615311Zhostname"":","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":ip-10-150-1-74.us-west-2.compute.internal4291"test live_repair::repair_test::test_live_repair_deps_mix ... }
,ok{"
"pid"msg:":4291"}
[1] Transition from Active to Faulted","v":0{,"name":""cruciblemsg"":,""level":30Crucible 53aa6152-9858-456d-bc14-bd6a95d97fdd has session id: 22d33f84-a075-4645-936b-e06708f0564d","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.525665045Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid",":time4291":"}
2023-09-22T23:08:02.525673207Z","hostname{":""msg":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
[1] 8bd67f2e-1fde-4913-85f8-577d1b57f202 (7024e503-4e36-4db3-924a-6f1bc0972c72) Active Faulted Active ds_transition to LiveRepairReady"{,"v":0","msg":name"":"crucible","level":30[0] 53aa6152-9858-456d-bc14-bd6a95d97fdd (2276e112-0970-4da5-9cd5-228a41ff83b8) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.525700946Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal",,""pidtime""::"4291}
2023-09-22T23:08:02.525708332Z","hostname{":""msg":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291[1] Transition from Faulted to LiveRepairReady"},
"v":0,"{name":"crucible"","msg":level"":30[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.525740048Z","hostname":","timeip-10-150-1-74.us-west-2.compute.internal"":","pid":42912023-09-22T23:08:02.525746265Z"},"
hostname":"{ip-10-150-1-74.us-west-2.compute.internal"","msg"pid:"":4291}
{"[1] 8bd67f2e-1fde-4913-85f8-577d1b57f202 (7024e503-4e36-4db3-924a-6f1bc0972c72) Active LiveRepairReady Active ds_transition to LiveRepairmsg"":","v":0,"name":"crucible","level":30[0] 53aa6152-9858-456d-bc14-bd6a95d97fdd (2276e112-0970-4da5-9cd5-228a41ff83b8) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.525776546Z","hostname":","time":ip-10-150-1-74.us-west-2.compute.internal"","pid":2023-09-22T23:08:02.525782203Z4291","}
hostname":"{ip-10-150-1-74.us-west-2.compute.internal""msg,"":"pid":4291}
[1] Transition from LiveRepairReady to LiveRepair","v{":0,""namemsg""::""crucible","level":[0] Transition from WaitActive to WaitQuorum30","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.525809583Z","hostname":","time":"ip-10-150-1-74.us-west-2.compute.internal","2023-09-22T23:08:02.525814528Zpid"":,"4291hostname"}:
"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 53aa6152-9858-456d-bc14-bd6a95d97fdd (2276e112-0970-4da5-9cd5-228a41ff83b8) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.525848995Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.525883431Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 53aa6152-9858-456d-bc14-bd6a95d97fdd (2276e112-0970-4da5-9cd5-228a41ff83b8) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.525922571Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.525960326Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 53aa6152-9858-456d-bc14-bd6a95d97fdd (2276e112-0970-4da5-9cd5-228a41ff83b8) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.526002354Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.526038728Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{test live_repair::repair_test::test_live_repair_deps_no_overlap ... "okmsg":"
[1] 53aa6152-9858-456d-bc14-bd6a95d97fdd (2276e112-0970-4da5-9cd5-228a41ff83b8) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.526078009Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.526099427Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 53aa6152-9858-456d-bc14-bd6a95d97fdd (2276e112-0970-4da5-9cd5-228a41ff83b8) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.526119586Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.526151075Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 53aa6152-9858-456d-bc14-bd6a95d97fdd (2276e112-0970-4da5-9cd5-228a41ff83b8) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"{time":"2023-09-22T23:08:02.526191636Z","hostname":""msgip-10-150-1-74.us-west-2.compute.internal"",":pid"":4291}
Crucible stats registered with UUID: 25f8ca52-9766-4754-b9c9-b871f5033947","v":{0,"name":""crucible"msg":,""level":30[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":","2023-09-22T23:08:02.526242788Z"time":,""hostname":"2023-09-22T23:08:02.526237007Z"ip-10-150-1-74.us-west-2.compute.internal",,""hostname"pid":":4291}ip-10-150-1-74.us-west-2.compute.internal"
,"pid":4291{}
"msg":"{[2] 53aa6152-9858-456d-bc14-bd6a95d97fdd (2276e112-0970-4da5-9cd5-228a41ff83b8) Active Active WaitQuorum ds_transition to Active"","msg"v"::"0,"name":"crucible","level":30Crucible 25f8ca52-9766-4754-b9c9-b871f5033947 has session id: 8a5194ca-fccb-4e81-8cc4-211ed80171e1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.526300106Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"time":"}
2023-09-22T23:08:02.526306562Z","hostname":"{"ip-10-150-1-74.us-west-2.compute.internal"msg",":"pid":4291[2] Transition from WaitQuorum to Active"}
,"v":0,"name":"{crucible","level":30"msg":"[0] 25f8ca52-9766-4754-b9c9-b871f5033947 (8d12af84-3563-40e3-992b-37f1da12c218) New New New ds_transition to WaitActive","v":0,"name":"crucible",",level"":time"30:"2023-09-22T23:08:02.526347949Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg",:""time":"2023-09-22T23:08:02.526363452Z","53aa6152-9858-456d-bc14-bd6a95d97fdd is now active with session: 2276e112-0970-4da5-9cd5-228a41ff83b8hostname"":,""v":0,"ip-10-150-1-74.us-west-2.compute.internalname"":,""crucible"pid",":level":429130}
{"msg":","[0] Transition from New to WaitActive"time":","v":02023-09-22T23:08:02.526393598Z",",name"":"hostname":crucible"","level":30ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":","time":"[1] 53aa6152-9858-456d-bc14-bd6a95d97fdd (2276e112-0970-4da5-9cd5-228a41ff83b8) Active Active Active ds_transition to Faulted"2023-09-22T23:08:02.52641593Z",",v"":hostname":"0,"name":"ip-10-150-1-74.us-west-2.compute.internal"crucible",",pid"":level"4291:30}
{"msg":","[0] 25f8ca52-9766-4754-b9c9-b871f5033947 (8d12af84-3563-40e3-992b-37f1da12c218) WaitActive New New ds_transition to WaitQuorumtime"":","v":2023-09-22T23:08:02.526445725Z"0,,""hostnamename""::""crucible","level":ip-10-150-1-74.us-west-2.compute.internal"30,"pid":4291}
{"msg":"[1] Transition from Active to Faulted","v":,0","timename":"":"crucible","2023-09-22T23:08:02.526473807Z"level":,30"hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{,""time"msg"::""2023-09-22T23:08:02.526497011Z"[0] Transition from WaitActive to WaitQuorum",","hostnamev""::"0,"name":"ip-10-150-1-74.us-west-2.compute.internal"crucible",","pid"level:"4291:30}
{"msg":","time":"[1] 53aa6152-9858-456d-bc14-bd6a95d97fdd (2276e112-0970-4da5-9cd5-228a41ff83b8) Active Faulted Active ds_transition to LiveRepairReady","2023-09-22T23:08:02.52652824Zv"{":,0","hostname"name":""msg":"Crucible stats registered with UUID: 4959cad2-ae2a-45ba-8a58-01c25a8aabf8","v":0ip-10-150-1-74.us-west-2.compute.internal",","name":"pidcrucible"":,"4291level":}30
{"msg":":"crucible","level":[0] 25f8ca52-9766-4754-b9c9-b871f5033947 (8d12af84-3563-40e3-992b-37f1da12c218) WaitQuorum New New ds_transition to Active"30,"v":0,"name":",crucible""time,""level"::"302023-09-22T23:08:02.526568664Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"time":"}
,"time":"2023-09-22T23:08:02.526596462Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal"{,"pid":4291}
"{msg":""msg":"[0] Transition from WaitQuorum to Active2023-09-22T23:08:02.526587201Z""Crucible 4959cad2-ae2a-45ba-8a58-01c25a8aabf8 has session id: fc05e5b8-f2e7-4066-b75b-c280c4b7728c","v",,""vhostname":"ip-10-150-1-74.us-west-2.compute.internal"",:"0pid":,4291"name":"}crucible
",":level":030{,"name":"crucible"",msg"":"level":30[1] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.526657703Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"time":"}
{,"time":"2023-09-22T23:08:02.526671872Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
"{msg":""msg":"[0] 4959cad2-ae2a-45ba-8a58-01c25a8aabf8 (5205f651-d65a-4d53-a1c9-0ce1d9b0247f) New New New ds_transition to WaitActive","v":0,"name":"[1] 53aa6152-9858-456d-bc14-bd6a95d97fdd (2276e112-0970-4da5-9cd5-228a41ff83b8) Active LiveRepairReady Active ds_transition to LiveRepaircrucible"","level2023-09-22T23:08:02.526665027Z",":,"hostname":"30ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
"v":{0,"name":""crucible"msg":,"",level":"30time":"[1] 25f8ca52-9766-4754-b9c9-b871f5033947 (8d12af84-3563-40e3-992b-37f1da12c218) Active New New ds_transition to WaitActive2023-09-22T23:08:02.526753428Z"",","v"hostname:":0","name":"crucible"ip-10-150-1-74.us-west-2.compute.internal",,""pid"level":30,"time":"2023-09-22T23:08:02.526774667Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
,"time":":2023-09-22T23:08:02.526788239Z4291"{,"}hostname"
{"msg":"[1] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30","time"msg"::""2023-09-22T23:08:02.526844151Z","[0] Transition from New to WaitActive"hostname":","v"ip-10-150-1-74.us-west-2.compute.internal":,"pid0:"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":",crucible""name,"":"level":crucible"30,"level":30":4291}
,"time":"2023-09-22T23:08:02.526903228Z,"",time"":"hostname":"2023-09-22T23:08:02.526905424Z"ip-10-150-1-74.us-west-2.compute.internal",,""pid"hostname"::"4291}
ip-10-150-1-74.us-west-2.compute.internal","pid":4291{}
{"msg":""[0] 4959cad2-ae2a-45ba-8a58-01c25a8aabf8 (5205f651-d65a-4d53-a1c9-0ce1d9b0247f) WaitActive New New ds_transition to WaitQuorum","v":msg"0:,"name":"crucible","level":30"[1] 25f8ca52-9766-4754-b9c9-b871f5033947 (8d12af84-3563-40e3-992b-37f1da12c218) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.527032198Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.527081284Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,}
{""msg":"time":"2023-09-22T23:08:02.527046487Z","hostname":"[0] 4959cad2-ae2a-45ba-8a58-01c25a8aabf8 (5205f651-d65a-4d53-a1c9-0ce1d9b0247f) WaitQuorum New New ds_transition to Active"ip-10-150-1-74.us-west-2.compute.internal",",pid":"4291v":}0
,"name":"crucible","level{":30"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.527135156Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal",,"pid":4291}
{test live_repair::repair_test::test_live_repair_deps_reads ... "ok
msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30"time":"2023-09-22T23:08:02.527142148Z","hostname,"":"time":"2023-09-22T23:08:02.527220072Z"ip-10-150-1-74.us-west-2.compute.internal","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 4959cad2-ae2a-45ba-8a58-01c25a8aabf8 (5205f651-d65a-4d53-a1c9-0ce1d9b0247f) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.52733348Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{,""msg":pid"":4291[1] Transition from New to WaitActive","v"}:
0,"name":"crucible","level":30{"msg":"[1] 25f8ca52-9766-4754-b9c9-b871f5033947 (8d12af84-3563-40e3-992b-37f1da12c218) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible",","level":time":30"2023-09-22T23:08:02.527390111Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":","time":"2023-09-22T23:08:02.527409003Z","[1] 4959cad2-ae2a-45ba-8a58-01c25a8aabf8 (5205f651-d65a-4d53-a1c9-0ce1d9b0247f) Active WaitActive New ds_transition to WaitQuorum"hostname",":v":0,"name":"crucible","level":30"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
,"time":"2023-09-22T23:08:02.527445248Z","{hostname":"ip-10-150-1-74.us-west-2.compute.internal"","msg"pid:"":4291}[1] Transition from WaitQuorum to Active"
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.527528294Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":,""time":"{2023-09-22T23:08:02.527541321Z"[1] 4959cad2-ae2a-45ba-8a58-01c25a8aabf8 (5205f651-d65a-4d53-a1c9-0ce1d9b0247f) Active WaitQuorum New ds_transition to Active",",hostname""":v"msg:":"0,"Crucible stats registered with UUID: 42308f37-4093-47ec-8852-915c974792c6name"":"crucible",","v":level":030,"name":"crucible","level":30"ip-10-150-1-74.us-west-2.compute.internal","pid":,4291"time":}"
2023-09-22T23:08:02.527599882Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal"{,"pid",:"4291"time"msg}":"
{:""msg":2023-09-22T23:08:02.52760841Z"",[1] Transition from WaitQuorum to Active","v":0,"name":""crucible"hostname":,[2] 25f8ca52-9766-4754-b9c9-b871f5033947 (8d12af84-3563-40e3-992b-37f1da12c218) Active Active New ds_transition to WaitActive""level":30,"v":0,"name":"crucible",""level":30ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
,"time":"2023-09-22T23:08:02.527671168Z","{hostname":"ip-10-150-1-74.us-west-2.compute.internal"",msg""pid":4291:"}
Crucible 42308f37-4093-47ec-8852-915c974792c6 has session id: f9f486be-e34d-455f-a56e-92cc97480032","v":0{,"name":"msg":""crucible","level":30[2] 4959cad2-ae2a-45ba-8a58-01c25a8aabf8 (5205f651-d65a-4d53-a1c9-0ce1d9b0247f) Active Active New ds_transition to WaitActive","v":0,"name,":""crucible"time",:""level":302023-09-22T23:08:02.52767909Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":,"4291time":"}
2023-09-22T23:08:02.527720666Z",","time":hostname"":{"2023-09-22T23:08:02.527730075Z","ip-10-150-1-74.us-west-2.compute.internal"hostname,"pid":4291}
":"ip-10-150-1-74.us-west-2.compute.internal",{"pid":4291}
{"msg":"[0] 42308f37-4093-47ec-8852-915c974792c6 (f7a38787-241b-4e71-9b0d-a228c252b4a7) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.527799154Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
"{msg":"[2] Transition from New to WaitActive""msg,"":v""msg":":[2] Transition from New to WaitActive"0,"name":,""v":crucible"0,,""level"name"::"30crucible",""level":30[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.527858713Z",,""time"hostname":","time":"ip-10-150-1-74.us-west-2.compute.internal","2023-09-22T23:08:02.527870493Zpid"":4291,"hostname":"}
:"2023-09-22T23:08:02.527862368Z","hostname":"{ip-10-150-1-74.us-west-2.compute.internal","pid":"4291msg":"}
[2] 4959cad2-ae2a-45ba-8a58-01c25a8aabf8 (5205f651-d65a-4d53-a1c9-0ce1d9b0247f) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30{ip-10-150-1-74.us-west-2.compute.internal",""msg":"pid":4291}
[2] 25f8ca52-9766-4754-b9c9-b871f5033947 (8d12af84-3563-40e3-992b-37f1da12c218) Active Active WaitActive ds_transition to WaitQuorum","v":0,,""time{""msg":":"2023-09-22T23:08:02.527922675Z","hostname":"[0] 42308f37-4093-47ec-8852-915c974792c6 (f7a38787-241b-4e71-9b0d-a228c252b4a7) WaitActive New New ds_transition to WaitQuorum","v"ip-10-150-1-74.us-west-2.compute.internal:"0,","pid"name":name4291":"crucible","level":30}
{:""msg":"crucible","[2] Transition from WaitActive to WaitQuorum"level":,"30v":0,",name"":"time":"crucible","2023-09-22T23:08:02.527962868Zlevel"":30,"hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
,"time":"2023-09-22T23:08:02.527979279Z"{,",hostname":""time":"2023-09-22T23:08:02.527988094Z","hostname":msg"":"ip-10-150-1-74.us-west-2.compute.internal",[2] Transition from WaitActive to WaitQuorum""pid",:"4291v":0},""name":
{ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30"msg":"[2] 4959cad2-ae2a-45ba-8a58-01c25a8aabf8 (5205f651-d65a-4d53-a1c9-0ce1d9b0247f) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible",,""time""level"crucible","level":30:30:"2023-09-22T23:08:02.52806053Z","hostname":","ip-10-150-1-74.us-west-2.compute.internal",time"","time:pid"""::"42912023-09-22T23:08:02.528088077Z"2023-09-22T23:08:02.528088997Z"},
","hostname"hostname":":"{ip-10-150-1-74.us-west-2.compute.internal"ip-10-150-1-74.us-west-2.compute.internal",",pid""":msg"pid:"":4291[0] 42308f37-4093-47ec-8852-915c974792c6 (f7a38787-241b-4e71-9b0d-a228c252b4a7) WaitQuorum New New ds_transition to Active"},
4291}
{"v":"0msg"{,":name":""crucible""[2] Transition from WaitQuorum to Active","msg"level"::,"30v":0,"name":"crucible","level":30","[2] 25f8ca52-9766-4754-b9c9-b871f5033947 (8d12af84-3563-40e3-992b-37f1da12c218) Active Active WaitQuorum ds_transition to Active"time":",2023-09-22T23:08:02.528177334Z""v":,0",hostname"":"ip-10-150-1-74.us-west-2.compute.internal","timepid""::"42912023-09-22T23:08:02.528185724Z}"
,"hostname":"{ip-10-150-1-74.us-west-2.compute.internal""msg":","pid,"":4291}
{"msg":"4959cad2-ae2a-45ba-8a58-01c25a8aabf8 is now active with session: 5205f651-d65a-4d53-a1c9-0ce1d9b0247f","v":0,"name":"crucible","level":30name":"crucible","level":30,"time":"2023-09-22T23:08:02.528251224Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
,"time":"2023-09-22T23:08:02.528266483Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"{crucible","level":30"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible"[1] 4959cad2-ae2a-45ba-8a58-01c25a8aabf8 (5205f651-d65a-4d53-a1c9-0ce1d9b0247f) Active Active Active ds_transition to Faulted","level":,30"v":0,"name":"crucible","level":,"30time":"2023-09-22T23:08:02.528317695Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal",","pid":time4291":"}2023-09-22T23:08:02.52832779Z"
,"hostname":",ip-10-150-1-74.us-west-2.compute.internal"","{pid":time"msg":"25f8ca52-9766-4754-b9c9-b871f5033947 is now active with session: 8d12af84-3563-40e3-992b-37f1da12c218","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.528366909Z",""hostname":":"ip-10-150-1-74.us-west-2.compute.internal"2023-09-22T23:08:02.528333417Z,""pid":4291,"hostname"}:
"ip-10-150-1-74.us-west-2.compute.internal","{pid":4291}
4291}
{"msg":"[1] 42308f37-4093-47ec-8852-915c974792c6 (f7a38787-241b-4e71-9b0d-a228c252b4a7) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time"{:"2023-09-22T23:08:02.528425833Z","hostname"":"msg":"ip-10-150-1-74.us-west-2.compute.internal","[1] Transition from Active to Faultedpid"":,4291"v"}:
0,"name"{:"crucible""msg":","level"[1] Transition from New to WaitActive:","30"msg":"[1] 25f8ca52-9766-4754-b9c9-b871f5033947 (8d12af84-3563-40e3-992b-37f1da12c218) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.528480675Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
,"time":"2023-09-22T23:08:02.528491919Z",{"hostname":""ip-10-150-1-74.us-west-2.compute.internal"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.528531525Z",","hostname":pid":"4291}ip-10-150-1-74.us-west-2.compute.internal"
v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.528568334Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{{"msg":""msg":","pid":[1] 42308f37-4093-47ec-8852-915c974792c6 (f7a38787-241b-4e71-9b0d-a228c252b4a7) Active WaitActive New ds_transition to WaitQuorum"4291,"v":0},
"[1] 4959cad2-ae2a-45ba-8a58-01c25a8aabf8 (5205f651-d65a-4d53-a1c9-0ce1d9b0247f) Active Faulted Active ds_transition to LiveRepairReadyname":""crucible","level",{"v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.528619027Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}"
:30,"time":"2023-09-22T23:08:02.528664006Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{{""msgmsg""::"msg":"[1] 25f8ca52-9766-4754-b9c9-b871f5033947 (8d12af84-3563-40e3-992b-37f1da12c218) Active Faulted Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30[1] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.528712856Z"","hostname":"[1] Transition from WaitActive to WaitQuorum","ip-10-150-1-74.us-west-2.compute.internal"v",:"0pid",:"4291name":"crucible","}level":
,"{time":"2023-09-22T23:08:02.528730605Z"","hostname":"ip-10-150-1-74.us-west-2.compute.internal","msgpid""::"4291[1] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.528782371Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 25f8ca52-9766-4754-b9c9-b871f5033947 (8d12af84-3563-40e3-992b-37f1da12c218) Active LiveRepairReady Active ds_transition to LiveRepair","v":0,"}name
{30,"time":"2023-09-22T23:08:02.52884036Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 42308f37-4093-47ec-8852-915c974792c6 (f7a38787-241b-4e71-9b0d-a228c252b4a7) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30"msg":","time":"[1] 4959cad2-ae2a-45ba-8a58-01c25a8aabf8 (5205f651-d65a-4d53-a1c9-0ce1d9b0247f) Active LiveRepairReady Active ds_transition to LiveRepair"2023-09-22T23:08:02.528869713Z",","hostname"v"::":"crucible","level":30,"0time":","name"2023-09-22T23:08:02.528891074Z":"crucible",","hostname"level"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,":time"30:"2023-09-22T23:08:02.528934391Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
,"time":"2023-09-22T23:08:02.528947522Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{{""msg":msg"":[1] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30","time":"2023-09-22T23:08:02.528998909Z"[2] 42308f37-4093-47ec-8852-915c974792c6 (f7a38787-241b-4e71-9b0d-a228c252b4a7) Active Active New ds_transition to WaitActive",",hostname"":v":"0,"name":"crucible","ip-10-150-1-74.us-west-2.compute.internal"level",":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible",""level":pid"30:4291}
:30,"time":"2023-09-22T23:08:02.529059687Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
,"time":"2023-09-22T23:08:02.52907541Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.529108678Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 42308f37-4093-47ec-8852-915c974792c6 (f7a38787-241b-4e71-9b0d-a228c252b4a7) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.529130448Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.529163101Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 42308f37-4093-47ec-8852-915c974792c6 (f7a38787-241b-4e71-9b0d-a228c252b4a7) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.529203195Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.529242363Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"42308f37-4093-47ec-8852-915c974792c6 is now active with session: f7a38787-241b-4e71-9b0d-a228c252b4a7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.529281237Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 42308f37-4093-47ec-8852-915c974792c6 (f7a38787-241b-4e71-9b0d-a228c252b4a7) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.529321511Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test live_repair::repair_test::test_live_repair_deps_repair_flush_repair ... {ok"
msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.529366203Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 42308f37-4093-47ec-8852-915c974792c6 (f7a38787-241b-4e71-9b0d-a228c252b4a7) Active Faulted Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.529389646Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.529407995Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test live_repair::repair_test::test_live_repair_deps_repair_flush ... {ok"
msg":"[1] 42308f37-4093-47ec-8852-915c974792c6 (f7a38787-241b-4e71-9b0d-a228c252b4a7) Active LiveRepairReady Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.52943764Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.52945666Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Write to Extent 1:2:9 under repair","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:02.529477939Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test live_repair::repair_test::test_live_repair_deps_repair_kitchen_sink ... ok
{"msg":"Crucible stats registered with UUID: f60f7c4d-4385-45f0-b753-8c0c6a7e5045","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.52985635Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible f60f7c4d-4385-45f0-b753-8c0c6a7e5045 has session id: 053465c4-04e6-4b4c-89f2-ff90abc10dd1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.529907663Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] f60f7c4d-4385-45f0-b753-8c0c6a7e5045 (428178a5-4e75-4da2-abf3-724c4e0398b8) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.529953207Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.52999543Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] f60f7c4d-4385-45f0-b753-8c0c6a7e5045 (428178a5-4e75-4da2-abf3-724c4e0398b8) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.530037914Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.530077803Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] f60f7c4d-4385-45f0-b753-8c0c6a7e5045 (428178a5-4e75-4da2-abf3-724c4e0398b8) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.530119926Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.530158568Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] f60f7c4d-4385-45f0-b753-8c0c6a7e5045 (428178a5-4e75-4da2-abf3-724c4e0398b8) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.530200859Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30{"msg":"Crucible stats registered with UUID: 996915b6-f4ae-4178-8b00-04c24dd99dc3",","time"v"::"0,"2023-09-22T23:08:02.53024077Z"name":","crucible"hostname":","level":ip-10-150-1-74.us-west-2.compute.internal"30,"pid":4291}
{"msg":"[1] f60f7c4d-4385-45f0-b753-8c0c6a7e5045 (428178a5-4e75-4da2-abf3-724c4e0398b8) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible",",level"":time"30:"2023-09-22T23:08:02.530266697Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
,"time":"2023-09-22T23:08:02.530291418Z"{,"hostname":""ip-10-150-1-74.us-west-2.compute.internal"msg,":""pid":4291}
Crucible 996915b6-f4ae-4178-8b00-04c24dd99dc3 has session id: fa2b84fd-a1e1-4774-b83e-eb1039583139","v":0,"{name":"crucible","level""msg"::"30[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.530338047Z","hostname":","ip-10-150-1-74.us-west-2.compute.internaltime":"","pid":42912023-09-22T23:08:02.530345876Z",}"
hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid{":4291}
"msg":"{"msg":"[0] 996915b6-f4ae-4178-8b00-04c24dd99dc3 (17b68f05-acac-4680-865d-7ea2ec0cd8e8) New New New ds_transition to WaitActive","v":0,"name":"crucible","[1] f60f7c4d-4385-45f0-b753-8c0c6a7e5045 (428178a5-4e75-4da2-abf3-724c4e0398b8) Active WaitQuorum New ds_transition to Active"level",:"30v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.530396693Z",","time":hostname"":"2023-09-22T23:08:02.530401496Z",ip-10-150-1-74.us-west-2.compute.internal"","hostnamepid""::"4291}
ip-10-150-1-74.us-west-2.compute.internal","pid":4291}{
"msg":"[0] Transition from New to WaitActive{","v":0","msg"name"::""crucible","level"[1] Transition from WaitQuorum to Active":,30"v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.530451779Z","hostname":","ip-10-150-1-74.us-west-2.compute.internal"time,"":pid"":42912023-09-22T23:08:02.530457315Z"}
,"hostname":"ip-10-150-1-74.us-west-2.compute.internal{","pid":4291"msg"}:
"{[0] 996915b6-f4ae-4178-8b00-04c24dd99dc3 (17b68f05-acac-4680-865d-7ea2ec0cd8e8) WaitActive New New ds_transition to WaitQuorum"",msg"":{"v":0,"name":"crucible"","level"msg:[2] f60f7c4d-4385-45f0-b753-8c0c6a7e5045 (428178a5-4e75-4da2-abf3-724c4e0398b8) Active Active New ds_transition to WaitActive30"":","v":0,"name":"Crucible stats registered with UUID: 99477887-f8f6-4caf-a2e3-12b105487a29crucible"",",level"":v"30:0,"name":"crucible","level":,"30time":"2023-09-22T23:08:02.530510623Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
,"time":"2023-09-22T23:08:02.530520639Z"{,"hostname":""msg":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291[0] Transition from WaitActive to WaitQuorum","},
"time"v":":{02023-09-22T23:08:02.530527117Z","name,"":hostname"":""crucible"msg",:ip-10-150-1-74.us-west-2.compute.internal"","level"":pid"[2] Transition from New to WaitActive:"304291,"v":}0
,"name":"crucible","level":30{","msg":time"":"2023-09-22T23:08:02.530573717Z","hostname":"Crucible 99477887-f8f6-4caf-a2e3-12b105487a29 has session id: 60461d52-283b-4d0d-a975-c0f76952364d","v":ip-10-150-1-74.us-west-2.compute.internal"0,","pidname""::,"4291"crucible"time}",:
""level":302023-09-22T23:08:02.530585733Z"{,"hostname":""msg"ip-10-150-1-74.us-west-2.compute.internal":","pid":4291}
[0] 996915b6-f4ae-4178-8b00-04c24dd99dc3 (17b68f05-acac-4680-865d-7ea2ec0cd8e8) WaitQuorum New New ds_transition to Active","v":0,"{name":","crucible"time","":"msg"level"2023-09-22T23:08:02.53061336Z"::30,""hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291[2] f60f7c4d-4385-45f0-b753-8c0c6a7e5045 (428178a5-4e75-4da2-abf3-724c4e0398b8) Active Active WaitActive ds_transition to WaitQuorum","}v"
:0,,""name":{time"":"crucible",""2023-09-22T23:08:02.530638853Zmsg":level"":"30,"hostname":"[0] 99477887-f8f6-4caf-a2e3-12b105487a29 (fb767d23-0921-4b65-9076-40bc0c47b70a) New New New ds_transition to WaitActive"ip-10-150-1-74.us-west-2.compute.internal",","v"pid:"0:,"4291name":"crucible"},
",level":"30time":"{2023-09-22T23:08:02.530664794Z",""hostname"msg":":"ip-10-150-1-74.us-west-2.compute.internal"[0] Transition from WaitQuorum to Active,""pid,"","v":time":4291":}2023-09-22T23:08:02.530682882Z"
,"0hostname":","name":{ip-10-150-1-74.us-west-2.compute.internal"","pid":"4291msg"crucible}:
"","{level":[2] Transition from WaitActive to WaitQuorum"30"msg":","v"[0] Transition from New to WaitActive":,"v":00,",name":""crucible"name,"":"level":30crucible","level":30,"time":","2023-09-22T23:08:02.530722168Z"time":","2023-09-22T23:08:02.530732449Zhostname":"","hostname":"ip-10-150-1-74.us-west-2.compute.internal"ip-10-150-1-74.us-west-2.compute.internal",,""pid":pid"4291:}4291
,}"{
time""msg":":"{2023-09-22T23:08:02.530734966Z",[0] 99477887-f8f6-4caf-a2e3-12b105487a29 (fb767d23-0921-4b65-9076-40bc0c47b70a) WaitActive New New ds_transition to WaitQuorum""","hostname"msg":v"::0",""name":"crucible","level":ip-10-150-1-74.us-west-2.compute.internal"30,"pid":[1] 996915b6-f4ae-4178-8b00-04c24dd99dc3 (17b68f05-acac-4680-865d-7ea2ec0cd8e8) Active New New ds_transition to WaitActive"4291,"v":}
{,"time":"2023-09-22T23:08:02.530780294Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30"msg":","time":"2023-09-22T23:08:02.530874301Z","[2] f60f7c4d-4385-45f0-b753-8c0c6a7e5045 (428178a5-4e75-4da2-abf3-724c4e0398b8) Active Active WaitQuorum ds_transition to Activehostname":"","ip-10-150-1-74.us-west-2.compute.internal"v,""pid"::42910}
,{"msg":"[0] 99477887-f8f6-4caf-a2e3-12b105487a29 (fb767d23-0921-4b65-9076-40bc0c47b70a) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.530904524Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible"","name"level"::"30crucible","level":30,"time":"2023-09-22T23:08:02.530928785Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":",[1] 99477887-f8f6-4caf-a2e3-12b105487a29 (fb767d23-0921-4b65-9076-40bc0c47b70a) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.530955951Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30"time":"2023-09-22T23:08:02.530932093Z",","hostname":"time":"ip-10-150-1-74.us-west-2.compute.internal2023-09-22T23:08:02.530974766Z"",","hostnamepid""::"4291ip-10-150-1-74.us-west-2.compute.internal","}pid":
4291{}
"msg":"{0[2] Transition from WaitQuorum to Active""msg,"":"v",:"0,"name"name"::"[1] 99477887-f8f6-4caf-a2e3-12b105487a29 (fb767d23-0921-4b65-9076-40bc0c47b70a) Active WaitActive New ds_transition to WaitQuorum"","cruciblev":crucible"0",,","name"level":"":level"crucible":3030,"level":30,"time":"2023-09-22T23:08:02.5310361Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":","[1] Transition from WaitActive to WaitQuorum",time"":"v":0,",name2023-09-22T23:08:02.531035568Z""":",crucible"time""hostname"::",""level":2023-09-22T23:08:02.531035566Z"30ip-10-150-1-74.us-west-2.compute.internal",,""pid":hostname"4291:"}
ip-10-150-1-74.us-west-2.compute.internal",",time":""{pid"2023-09-22T23:08:02.531068051Z:"4291","msg"hostname":":"}
ip-10-150-1-74.us-west-2.compute.internal"f60f7c4d-4385-45f0-b753-8c0c6a7e5045 is now active with session: 428178a5-4e75-4da2-abf3-724c4e0398b8","pid",:"4291v":}0
{,"{name":""msg"crucible:""","msg"level"::"[1] 99477887-f8f6-4caf-a2e3-12b105487a29 (fb767d23-0921-4b65-9076-40bc0c47b70a) Active WaitQuorum New ds_transition to Active"30,"[1] Transition from New to WaitActive"v":0,,""name":"v"crucible":,"0level":30,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.531118192Z",",hostname":""time"ip-10-150-1-74.us-west-2.compute.internal":,""pid":4291}
2023-09-22T23:08:02.531110966Z"{,""msg":",hostname"":[1] Transition from WaitQuorum to Active"time,""v""::"0,"name":"ip-10-150-1-74.us-west-2.compute.internal"2023-09-22T23:08:02.531122598Z","crucible",pid","":level"4291hostname"::}"30
ip-10-150-1-74.us-west-2.compute.internal","pid":,4291"time":"{}
2023-09-22T23:08:02.531157819Z"","msghostname":""{:"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}"
msg":"{[1] f60f7c4d-4385-45f0-b753-8c0c6a7e5045 (428178a5-4e75-4da2-abf3-724c4e0398b8) Active Active Active ds_transition to Faulted"",msg":""v":0,"[1] 996915b6-f4ae-4178-8b00-04c24dd99dc3 (17b68f05-acac-4680-865d-7ea2ec0cd8e8) Active WaitActive New ds_transition to WaitQuorum"name[2] 99477887-f8f6-4caf-a2e3-12b105487a29 (fb767d23-0921-4b65-9076-40bc0c47b70a) Active Active New ds_transition to WaitActive,""",v":""v"crucible"::00,",,name""":namelevel""":crucible:30"","crucible"level":30,"level":30,"time":"2023-09-22T23:08:02.531206783Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
,,""{time"time""::""msg":"2023-09-22T23:08:02.531204735Z"2023-09-22T23:08:02.531208731Z"[2] Transition from New to WaitActive",,,"""v"hostname"hostname"::0:",""name":"crucible"ip-10-150-1-74.us-west-2.compute.internal,"ip-10-150-1-74.us-west-2.compute.internal"",,level":""pid"pid"30::42914291}}
,"
{time":"2023-09-22T23:08:02.53125268Z","hostname":""{msgip-10-150-1-74.us-west-2.compute.internal"","":pid":"4291msg"}:
"[1] Transition from WaitActive to WaitQuorum"{,"[1] Transition from Active to Faultedv""":msg0":,",""name"v"::[2] 99477887-f8f6-4caf-a2e3-12b105487a29 (fb767d23-0921-4b65-9076-40bc0c47b70a) Active Active WaitActive ds_transition to WaitQuorum""0,crucible"",,"v"name":"0level:,""name""crucible":",:"crucible"level30":,"30level":30,"time":"2023-09-22T23:08:02.531309043Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitActive to WaitQuorum",",,v"":"time"0time",:""name":":"crucible"2023-09-22T23:08:02.531307493Z","2023-09-22T23:08:02.531308913Zlevel"":,,"30"hostname":"hostname":"ip-10-150-1-74.us-west-2.compute.internal",ip-10-150-1-74.us-west-2.compute.internal"",pid",":time"":"pid"42912023-09-22T23:08:02.531341832Z":,}"
hostname":"4291ip-10-150-1-74.us-west-2.compute.internal","}pid{
"":4291msg":}"
{{"msg":""[1] f60f7c4d-4385-45f0-b753-8c0c6a7e5045 (428178a5-4e75-4da2-abf3-724c4e0398b8) Active Faulted Active ds_transition to LiveRepairReady"msg":","[2] 99477887-f8f6-4caf-a2e3-12b105487a29 (fb767d23-0921-4b65-9076-40bc0c47b70a) Active Active WaitQuorum ds_transition to Active"v",":v":00,,""[1] 996915b6-f4ae-4178-8b00-04c24dd99dc3 (17b68f05-acac-4680-865d-7ea2ec0cd8e8) Active WaitQuorum New ds_transition to Activename"":name,"":crucible""",crucible""v",:"level0level"",:"30name"::"30crucible","level":30,"time":"2023-09-22T23:08:02.53139852Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":","time"2023-09-22T23:08:02.531400922Z":","hostname",:2023-09-22T23:08:02.531403555Z""",time":"ip-10-150-1-74.us-west-2.compute.internal""2023-09-22T23:08:02.531417101Z",",hostname"pid"::"4291"hostname":"}ip-10-150-1-74.us-west-2.compute.internal"
,ip-10-150-1-74.us-west-2.compute.internal"","pid"pid"{::42914291}
}"{
"msg"msg"::""{[1] Transition from Faulted to LiveRepairReady99477887-f8f6-4caf-a2e3-12b105487a29 is now active with session: fb767d23-0921-4b65-9076-40bc0c47b70a""",",msg"":v"v""::00,"[1] Transition from WaitQuorum to Active"name",,:""crucible"","vlevel":name30""::"0crucible",","name"level":,""time":":crucible"302023-09-22T23:08:02.531477252Z,"","level"hostname:":"30ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 99477887-f8f6-4caf-a2e3-12b105487a29 (fb767d23-0921-4b65-9076-40bc0c47b70a) Active Active Active ds_transition to Faulted","v":0,"name":","crucible",time"":level":"30,"time":2023-09-22T23:08:02.531487963Z"","hostname"2023-09-22T23:08:02.531492595Z":",,""ip-10-150-1-74.us-west-2.compute.internal"hostname",time":":"pid""2023-09-22T23:08:02.531507762Z":,"4291ip-10-150-1-74.us-west-2.compute.internal"hostname":},"
ip-10-150-1-74.us-west-2.compute.internal"","pidpid"":{4291:}
"4291msg":{"}"
msg":"[1] f60f7c4d-4385-45f0-b753-8c0c6a7e5045 (428178a5-4e75-4da2-abf3-724c4e0398b8) Active LiveRepairReady Active ds_transition to LiveRepair"[1] Transition from Active to Faulted",",{v":"0v"","msg":name"0::",crucible"",name"":""level":30crucible","level":30[2] 996915b6-f4ae-4178-8b00-04c24dd99dc3 (17b68f05-acac-4680-865d-7ea2ec0cd8e8) Active Active New ds_transition to WaitActive","v":0,",time":""name":2023-09-22T23:08:02.531562073Z"","crucible"hostname":","level"ip-10-150-1-74.us-west-2.compute.internal":,"30pid":4291}
,"{time":""msg":"2023-09-22T23:08:02.531565119Z","hostname":"[1] 99477887-f8f6-4caf-a2e3-12b105487a29 (fb767d23-0921-4b65-9076-40bc0c47b70a) Active Faulted Active ds_transition to LiveRepairReady","v":0ip-10-150-1-74.us-west-2.compute.internal",",,""nametime":""crucible"pid":",":level":4291302023-09-22T23:08:02.531578079Z"}
,"hostname":","time":"{2023-09-22T23:08:02.531603089Z"ip-10-150-1-74.us-west-2.compute.internal,"",hostname"":pid""":ip-10-150-1-74.us-west-2.compute.internal4291msg""}:"
,"[1] Transition from LiveRepairReady to LiveRepair"pid":4291,"}v{"
:"0{,msg""msgname":"""::""crucible"[1] Transition from Faulted to LiveRepairReady",,"[2] Transition from New to WaitActive"level"":v"30:,0",v"":name":"0crucible",,""level":name"30:"crucible","level":30,"time":",2023-09-22T23:08:02.531657302Z"",time"":hostname":""ip-10-150-1-74.us-west-2.compute.internal","2023-09-22T23:08:02.531649138Z"pid":4291,}"
hostname":{,"""msg":"time":ip-10-150-1-74.us-west-2.compute.internal"","pid":2023-09-22T23:08:02.531661049Z4291[1] 99477887-f8f6-4caf-a2e3-12b105487a29 (fb767d23-0921-4b65-9076-40bc0c47b70a) Active LiveRepairReady Active ds_transition to LiveRepair""},"
v":,0","namehostname"":":"crucible","level":30ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
,"time":"2023-09-22T23:08:02.531698494Z","hostname":"{ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
"msg{":""msg":"[1] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30[2] 996915b6-f4ae-4178-8b00-04c24dd99dc3 (17b68f05-acac-4680-865d-7ea2ec0cd8e8) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible",","level"time":":302023-09-22T23:08:02.531720553Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
,"time":"2023-09-22T23:08:02.531729593Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.531770937Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 996915b6-f4ae-4178-8b00-04c24dd99dc3 (17b68f05-acac-4680-865d-7ea2ec0cd8e8) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.53181494Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.531854756Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"996915b6-f4ae-4178-8b00-04c24dd99dc3 is now active with session: 17b68f05-acac-4680-865d-7ea2ec0cd8e8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.531897024Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 996915b6-f4ae-4178-8b00-04c24dd99dc3 (17b68f05-acac-4680-865d-7ea2ec0cd8e8) Active Active Active ds_transition to Faulted","v"test live_repair::repair_test::test_live_repair_deps_repair_other ... :ok0
,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.53195126Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30test live_repair::repair_test::test_live_repair_deps_repair_rafter ... ok
,"time":"2023-09-22T23:08:02.531991722Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 996915b6-f4ae-4178-8b00-04c24dd99dc3 (17b68f05-acac-4680-865d-7ea2ec0cd8e8) Active Faulted Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.532034763Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.532074005Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 996915b6-f4ae-4178-8b00-04c24dd99dc3 (17b68f05-acac-4680-865d-7ea2ec0cd8e8) Active LiveRepairReady Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.532116458Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.53216212Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test live_repair::repair_test::test_live_repair_deps_repair_overlappers ... ok
{"msg":"Crucible stats registered with UUID: 1c3e124b-d8c5-4c0d-9558-10aa827e84d2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.532403093Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 1c3e124b-d8c5-4c0d-9558-10aa827e84d2 has session id: 27b5fb3d-5800-4e86-916d-15f076b1c956","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.532455979Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 1c3e124b-d8c5-4c0d-9558-10aa827e84d2 (66cadaf0-27ad-428b-882a-29ad9e6a6b14) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.532502572Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.532544678Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 1c3e124b-d8c5-4c0d-9558-10aa827e84d2 (66cadaf0-27ad-428b-882a-29ad9e6a6b14) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.532587279Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.532626765Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 1c3e124b-d8c5-4c0d-9558-10aa827e84d2 (66cadaf0-27ad-428b-882a-29ad9e6a6b14) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.532669535Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal",{"pid":4291}
"msg":"{"Crucible stats registered with UUID: 42df254b-7101-46d2-b33a-70cc63b38d0emsg"":","v":[0] Transition from WaitQuorum to Active"0,",v"":name":0","crucible"name":,""level"crucible":,"30level":30,"time":"2023-09-22T23:08:02.532719121Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal,"","time"pid":":42912023-09-22T23:08:02.532718343Z"}
,"hostname":"{ip-10-150-1-74.us-west-2.compute.internal","pid":4291"msg":"}
[1] 1c3e124b-d8c5-4c0d-9558-10aa827e84d2 (66cadaf0-27ad-428b-882a-29ad9e6a6b14) Active New New ds_transition to WaitActive","v":0,"name":"{crucible","level":30"msg":"Crucible 42df254b-7101-46d2-b33a-70cc63b38d0e has session id: 528bdb1e-dc63-4b25-9f0d-084a49b41225","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.53276976Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":,""time"2023-09-22T23:08:02.532811252Z":","hostname":2023-09-22T23:08:02.532781597Z"{"ip-10-150-1-74.us-west-2.compute.internal",","hostname"pid"":msg":"Crucible stats registered with UUID: 7c6b2b7f-d4b5-4ec2-a67b-4f02968436c8"4291,"v":}
0,"name":"crucible"{,"level":30"msg":"[1] 1c3e124b-d8c5-4c0d-9558-10aa827e84d2 (66cadaf0-27ad-428b-882a-29ad9e6a6b14) Active WaitActive New ds_transition to WaitQuorum","v":0,"name:":""crucible","levelip-10-150-1-74.us-west-2.compute.internal"":30,"pid":4291}
,"time":"2023-09-22T23:08:02.532861447Z"{,","hostname":time"""ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
:{"2023-09-22T23:08:02.532872826Z"msg","msg:":""Crucible 7c6b2b7f-d4b5-4ec2-a67b-4f02968436c8 has session id: f9ed11e1-1134-4e8a-a7b1-a56e65070b2e","v":0,"name":"crucible","[0] 42df254b-7101-46d2-b33a-70cc63b38d0e (4719a10d-d68a-415f-8de5-4a3581ecf476) New New New ds_transition to WaitActive"level":30,"v"":0hostname",":"name":"crucible","ip-10-150-1-74.us-west-2.compute.internal"level":,"30pid":4291}
,"time":"2023-09-22T23:08:02.53292596Z","{hostname":"ip-10-150-1-74.us-west-2.compute.internal"","msg"pid"::"4291}
{[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30"msg":"[0] 7c6b2b7f-d4b5-4ec2-a67b-4f02968436c8 (db630a11-b750-4e3d-a391-6686a58e1487) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":,"30time":",2023-09-22T23:08:02.53302444Z"","time"hostname"::""2023-09-22T23:08:02.532935962Z"ip-10-150-1-74.us-west-2.compute.internal","pid":,"4291hostname":"}
ip-10-150-1-74.us-west-2.compute.internal,"","time"pid"{:":2023-09-22T23:08:02.53304488Z4291"","msg"hostname}:""[1] 1c3e124b-d8c5-4c0d-9558-10aa827e84d2 (66cadaf0-27ad-428b-882a-29ad9e6a6b14) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30:"ip-10-150-1-74.us-west-2.compute.internal",",pid":4291"time"}:
{
{"msg":""msg":"[0] Transition from New to WaitActive"[0] Transition from New to WaitActive",","vv"":0:,"name":0","crucible"name,""level:"":crucible"30,""level":302023-09-22T23:08:02.533085908Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":,"time"4291:"}2023-09-22T23:08:02.533167818Z"
,"hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}{,
"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30{"msg":","time":"2023-09-22T23:08:02.533204331Z","hostname":"[0] 7c6b2b7f-d4b5-4ec2-a67b-4f02968436c8 (db630a11-b750-4e3d-a391-6686a58e1487) WaitActive New New ds_transition to WaitQuorum","v":0ip-10-150-1-74.us-west-2.compute.internal,"",name"":"pidcrucible"":,"4291level":30}
{","timemsg":":"[2] 1c3e124b-d8c5-4c0d-9558-10aa827e84d2 (66cadaf0-27ad-428b-882a-29ad9e6a6b14) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30"2023-09-22T23:08:02.533236028Z","hostname":",ip-10-150-1-74.us-west-2.compute.internal"","time"pid:"":42912023-09-22T23:08:02.533260381Z}"
{""msg":"time":[0] Transition from WaitActive to WaitQuorum"","v":0,"name":"2023-09-22T23:08:02.533171419Z"crucible","level":30,"hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
,"time":"2023-09-22T23:08:02.533335293Z"{,"hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":"4291,msg"}":
{hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
"msg":"{[0] 7c6b2b7f-d4b5-4ec2-a67b-4f02968436c8 (db630a11-b750-4e3d-a391-6686a58e1487) WaitQuorum New New ds_transition to Active"","msg"v:"":0,"name":"crucible"[2] Transition from New to WaitActive","level":,30"v":0,"name":"crucible"","level":30,"time":"[0] 42df254b-7101-46d2-b33a-70cc63b38d0e (4719a10d-d68a-415f-8de5-4a3581ecf476) WaitActive New New ds_transition to WaitQuorum"2023-09-22T23:08:02.533418661Z","hostname":",ip-10-150-1-74.us-west-2.compute.internal"",v"":pid":42910}
,"time":"2023-09-22T23:08:02.533425542Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{{"msg":""msg":"[0] Transition from WaitQuorum to Active","v":[2] 1c3e124b-d8c5-4c0d-9558-10aa827e84d2 (66cadaf0-27ad-428b-882a-29ad9e6a6b14) Active Active WaitActive ds_transition to WaitQuorum"0,,""name"v":":crucible"0,",level":",30name":""crucible"name",:""level":crucible"30,"level":,"30time":"2023-09-22T23:08:02.533479127Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":","time":"[1] 7c6b2b7f-d4b5-4ec2-a67b-4f02968436c8 (db630a11-b750-4e3d-a391-6686a58e1487) Active New New ds_transition to WaitActive","v":02023-09-22T23:08:02.533486207Z","name":","crucible","hostnamelevel""::30","time":"2023-09-22T23:08:02.533510962Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.53352985Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291[1] 7c6b2b7f-d4b5-4ec2-a67b-4f02968436c8 (db630a11-b750-4e3d-a391-6686a58e1487) Active WaitActive New ds_transition to WaitQuorum"},
"v":0,"name":"crucible","level":30{"msg":","[2] Transition from WaitActive to WaitQuorum"time,,"v":0,"name":"crucible","level":30","timetime":"":"2023-09-22T23:08:02.533574869Z"2023-09-22T23:08:02.533559808Z",","hostname"hostname:":""ip-10-150-1-74.us-west-2.compute.internal","pid":ip-10-150-1-74.us-west-2.compute.internal"4291,"}pid"
{":"2023-09-22T23:08:02.533489972Z""msg":","[1] Transition from WaitActive to WaitQuorum"hostname",:""v":0,"name":"crucible"ip-10-150-1-74.us-west-2.compute.internal,""level":,30"pid":4291}
,"time":"2023-09-22T23:08:02.533664139Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{:4291}
"msg":"{[1] 7c6b2b7f-d4b5-4ec2-a67b-4f02968436c8 (db630a11-b750-4e3d-a391-6686a58e1487) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible"","level":msg30":"[2] 1c3e124b-d8c5-4c0d-9558-10aa827e84d2 (66cadaf0-27ad-428b-882a-29ad9e6a6b14) Active Active WaitQuorum ds_transition to Active"{,"v":0,","name":time"":"2023-09-22T23:08:02.53373704Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.533764111Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 7c6b2b7f-d4b5-4ec2-a67b-4f02968436c8 (db630a11-b750-4e3d-a391-6686a58e1487) Active Active New ds_transition to WaitActive"msg":","v":0,"name"[0] Transition from WaitActive to WaitQuorum"crucible","level":30","v":0,,""name":time"":"crucible","2023-09-22T23:08:02.533819725Zlevel"":,30"hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30:"crucible","level",:"30time":"2023-09-22T23:08:02.533847248Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"time"}:
","time":"2023-09-22T23:08:02.533874283Z2023-09-22T23:08:02.533880452Z"",",hostname":""{hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291ip-10-150-1-74.us-west-2.compute.internal"}",
"msg"pid"::4291"{}
[0] 42df254b-7101-46d2-b33a-70cc63b38d0e (4719a10d-d68a-415f-8de5-4a3581ecf476) WaitQuorum New New ds_transition to Active""msg":","v":[2] Transition from New to WaitActive{"0,"v":,0",""msgname"name":"::"""cruciblecrucible"","level":,"30level":1c3e124b-d8c5-4c0d-9558-10aa827e84d2 is now active with session: 66cadaf0-27ad-428b-882a-29ad9e6a6b14"30,"v":0,"name":"crucible","level":,"30time":"2023-09-22T23:08:02.533939541Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":","[2] 7c6b2b7f-d4b5-4ec2-a67b-4f02968436c8 (db630a11-b750-4e3d-a391-6686a58e1487) Active Active WaitActive ds_transition to WaitQuorum"time",":v":,"0","timename":""2023-09-22T23:08:02.533941961Z":crucible"",",level"2023-09-22T23:08:02.533948321Z"hostname",:"30"hostname":":"ip-10-150-1-74.us-west-2.compute.internal"ip-10-150-1-74.us-west-2.compute.internal",","pidpid,""time":":":2023-09-22T23:08:02.53397746Z"42914291,"}hostname
":"}
{ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
"{msg{":""msg":""msg"[2] Transition from WaitActive to WaitQuorum":","v":0,"name":"[1] 1c3e124b-d8c5-4c0d-9558-10aa827e84d2 (66cadaf0-27ad-428b-882a-29ad9e6a6b14) Active Active Active ds_transition to Faultedcrucible"",",level"[0] Transition from WaitQuorum to Active"":v30",:"0v",:"0name":,"","cruciblename"time":"":2023-09-22T23:08:02.534019213Z"",,""cruciblelevelhostname":"":"30ip-10-150-1-74.us-west-2.compute.internal",,""pid":level4291":}
30{"msg":"[2] 7c6b2b7f-d4b5-4ec2-a67b-4f02968436c8 (db630a11-b750-4e3d-a391-6686a58e1487) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.534038211Z","hostname":","time":"ip-10-150-1-74.us-west-2.compute.internal2023-09-22T23:08:02.53405273Z,"","",timehostname""pid":":"":ip-10-150-1-74.us-west-2.compute.internal"4291,"2023-09-22T23:08:02.534044735Z"pid":,4291"}}hostname"

{:""msg":"ip-10-150-1-74.us-west-2.compute.internal"[2] Transition from WaitQuorum to Active,""{pid,""v"::"04291msg,"":name":"}"crucible"
[1] Transition from Active to Faulted","level":,"30v"{:0,"name":"crucible"","msg":level",":time30":""2023-09-22T23:08:02.534107563Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
[1] 42df254b-7101-46d2-b33a-70cc63b38d0e (4719a10d-d68a-415f-8de5-4a3581ecf476) Active New New ds_transition to WaitActive","{v",""msg"time"::0",:""name":"7c6b2b7f-d4b5-4ec2-a67b-4f02968436c8 is now active with session: db630a11-b750-4e3d-a391-6686a58e1487"crucible,2023-09-22T23:08:02.534116736Z"""v,"":,0"hostname":,level"""name:":"30crucible"ip-10-150-1-74.us-west-2.compute.internal,""level":,30"pid":4291}
,"time":"{2023-09-22T23:08:02.534155129Z","hostname":""msg"ip-10-150-1-74.us-west-2.compute.internal":,""pid":,4291"}
time":"[1] 1c3e124b-d8c5-4c0d-9558-10aa827e84d2 (66cadaf0-27ad-428b-882a-29ad9e6a6b14) Active Faulted Active ds_transition to LiveRepairReady"{2023-09-22T23:08:02.534152096Z",","msg":""v"hostname"::"0[1] 7c6b2b7f-d4b5-4ec2-a67b-4f02968436c8 (db630a11-b750-4e3d-a391-6686a58e1487) Active Active Active ds_transition to Faulted",",ip-10-150-1-74.us-west-2.compute.internal""v":,name":""0pid,""crucible"name"::"4291,crucible"","levellevel":"30}:
30,"time":"2023-09-22T23:08:02.534201702Z","hostname":"{ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
"msg{":""msg":"[1] Transition from New to WaitActive"[1] Transition from Active to Faulted",","v":v"0:,"0name",:",crucible"",""level"name"::time""30:"crucible","2023-09-22T23:08:02.534207944Z"level":,30"hostname":","time":"ip-10-150-1-74.us-west-2.compute.internal"2023-09-22T23:08:02.534234224Z",,""hostname":"pid":4291ip-10-150-1-74.us-west-2.compute.internal","pid":4291}}

,"{time{"":msg":"""msg":"2023-09-22T23:08:02.534239249Z","[1] Transition from Faulted to LiveRepairReady"hostname",[1] 7c6b2b7f-d4b5-4ec2-a67b-4f02968436c8 (db630a11-b750-4e3d-a391-6686a58e1487) Active Faulted Active ds_transition to LiveRepairReady":,""v":"0v",":ip-10-150-1-74.us-west-2.compute.internalname":""crucible",,""0pid"level"::4291,"30name"}:
","crucible"time":","level{"2023-09-22T23:08:02.534282787Z":,"30hostname":""msg":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"[1] 42df254b-7101-46d2-b33a-70cc63b38d0e (4719a10d-d68a-415f-8de5-4a3581ecf476) Active WaitActive New ds_transition to WaitQuorummsg":"","[1] Transition from Faulted to LiveRepairReady"v,""v"::0,"0,name"",:"time"crucible":,""level":"2023-09-22T23:08:02.534295614Z30"name":","crucible"hostname",:""level":30,"time":"ip-10-150-1-74.us-west-2.compute.internal","2023-09-22T23:08:02.534321142Z"pid",":hostname":4291"}ip-10-150-1-74.us-west-2.compute.internal
","pid":4291}
{{,"""msg"msg"::""time":"2023-09-22T23:08:02.534327831Z","[1] 7c6b2b7f-d4b5-4ec2-a67b-4f02968436c8 (db630a11-b750-4e3d-a391-6686a58e1487) Active LiveRepairReady Active ds_transition to LiveRepair"hostname",[1] 1c3e124b-d8c5-4c0d-9558-10aa827e84d2 (66cadaf0-27ad-428b-882a-29ad9e6a6b14) Active LiveRepairReady Active ds_transition to LiveRepair":""v":,0"ip-10-150-1-74.us-west-2.compute.internal,"vname":""",:crucible""0pid,,"""level"::304291name":"crucible}"
,,""time":"level":2023-09-22T23:08:02.534371854Z"30,"{hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291"}
msg":"{"msg":"[1] Transition from WaitActive to WaitQuorum"[1] Transition from LiveRepairReady to LiveRepair",,""v":v"0:,,"0"name"time"::,""cruciblename""":,""level":2023-09-22T23:08:02.534383616Z"30crucible",","hostname":"level":30ip-10-150-1-74.us-west-2.compute.internal","pid":,"4291time":"}2023-09-22T23:08:02.534410964Z"
,"hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291{}
"msg":","time"[1] Transition from LiveRepairReady to LiveRepair":","v":2023-09-22T23:08:02.534415194Z0","name":,""crucible"hostname":,""level":30ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":","time":"2023-09-22T23:08:02.534443403Z","hostname":[1] 42df254b-7101-46d2-b33a-70cc63b38d0e (4719a10d-d68a-415f-8de5-4a3581ecf476) Active WaitQuorum New ds_transition to Active"","v":0ip-10-150-1-74.us-west-2.compute.internal",",name"":"pid"crucible":,"4291level":30}
,"time":"2023-09-22T23:08:02.534473845Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.534516968Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{test impacted_blocks::test::union_produces_greater_than_or_equal_block_count ... "okmsg"
:"[2] 42df254b-7101-46d2-b33a-70cc63b38d0e (4719a10d-d68a-415f-8de5-4a3581ecf476) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.534576834Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"test live_repair::repair_test::test_live_repair_deps_repair_rspan_left ... 2023-09-22T23:08:02.534618148Zok
","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 42df254b-7101-46d2-b33a-70cc63b38d0e (4719a10d-d68a-415f-8de5-4a3581ecf476) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.534672953Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.534714472Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":test live_repair::repair_test::test_live_repair_deps_repair_read ... 4291ok
}
{"msg":"[2] 42df254b-7101-46d2-b33a-70cc63b38d0e (4719a10d-d68a-415f-8de5-4a3581ecf476) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.534767125Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.53483521Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"42df254b-7101-46d2-b33a-70cc63b38d0e is now active with session: 4719a10d-d68a-415f-8de5-4a3581ecf476","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.534880365Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 42df254b-7101-46d2-b33a-70cc63b38d0e (4719a10d-d68a-415f-8de5-4a3581ecf476) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.534923549Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.534963368Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 42df254b-7101-46d2-b33a-70cc63b38d0e (4719a10d-d68a-415f-8de5-4a3581ecf476) Active Faulted Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30{"msg":","time":"Crucible stats registered with UUID: 40de3eeb-c819-466e-88b4-e1712ed524bc"2023-09-22T23:08:02.535006432Z",",v"":hostname":0","name":"ip-10-150-1-74.us-west-2.compute.internal"crucible,""pid",:"4291level":}30
{"msg":"[1] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.535037493Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal",,""timepid""::"42912023-09-22T23:08:02.535054287Z"},
"hostname":"ip-10-150-1-74.us-west-2.compute.internal","{pid":4291}
"msg":"{Crucible 40de3eeb-c819-466e-88b4-e1712ed524bc has session id: f1573e91-5db6-4b3a-8324-135a00e99a0d"",msg""v"::"0,"name":"crucible","level":30[1] 42df254b-7101-46d2-b33a-70cc63b38d0e (4719a10d-d68a-415f-8de5-4a3581ecf476) Active LiveRepairReady Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.535106376Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal",,""time"pid"::"42912023-09-22T23:08:02.535113314Z"}
,"hostname":"ip-10-150-1-74.us-west-2.compute.internal",{"pid":4291}
"msg":"{"[0] 40de3eeb-c819-466e-88b4-e1712ed524bc (5b45a21e-af03-4c26-80d4-f98a411c8304) New New New ds_transition to WaitActivemsg"":","v":0[1] Transition from LiveRepairReady to LiveRepair",",name"":"v":crucible"0,","level"name:":30"crucible","level":30,"time,"":"time":"2023-09-22T23:08:02.53516921Z"2023-09-22T23:08:02.535171953Z",","hostname":"hostname":"ip-10-150-1-74.us-west-2.compute.internal","ip-10-150-1-74.us-west-2.compute.internal"pid,"":pid4291":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30{"msg":"Crucible stats registered with UUID: b734394e-ac55-46b3-89f9-8e132a0e6ae7",","time":v"":02023-09-22T23:08:02.535238186Z","name,"":"hostname"crucible:","level":30,"time":"2023-09-22T23:08:02.535273804Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal"","pid":4291ip-10-150-1-74.us-west-2.compute.internal}"
{"msg":}"{
"Crucible stats registered with UUID: aa7556f5-78ad-4859-b5a3-cb4209a0a400msg":"Crucible b734394e-ac55-46b3-89f9-8e132a0e6ae7 has session id: c70bd9ee-29e2-4ee1-bd38-7ddc38a57989","v":0,,""pidname":"crucible","level":30","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.535345236Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":",""time":":4291[0] b734394e-ac55-46b3-89f9-8e132a0e6ae7 (c367f5ab-c4f8-42c0-882b-6f763e0900ac) New New New ds_transition to WaitActive2023-09-22T23:08:02.53535437Z"",,"}v
":0,"name":"crucible","{level":30"msg":"[0] 40de3eeb-c819-466e-88b4-e1712ed524bc (5b45a21e-af03-4c26-80d4-f98a411c8304) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":,"30time":"2023-09-22T23:08:02.535402463Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":","[0] Transition from New to WaitActivetime"":","v":02023-09-22T23:08:02.535419675Z","name",:""hostname"crucible:"","level":30ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{,"time":""msg":"2023-09-22T23:08:02.535451854Z","hostname[0] Transition from WaitActive to WaitQuorum":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
"{hostname":""msg":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
[0] b734394e-ac55-46b3-89f9-8e132a0e6ae7 (c367f5ab-c4f8-42c0-882b-6f763e0900ac) WaitActive New New ds_transition to WaitQuorum","v":0,"{name":"crucible","level":"30msg":"Crucible aa7556f5-78ad-4859-b5a3-cb4209a0a400 has session id: 922807e4-3275-418d-96f2-52bc2d475a21","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.535503085Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
,"time":"2023-09-22T23:08:02.535514244Z","hostname":"{ip-10-150-1-74.us-west-2.compute.internal","pid"":msg4291}
":"{[0] Transition from WaitActive to WaitQuorum","v":"0msg",":name":"crucible"test live_repair::repair_test::test_live_repair_deps_repair_repair_repair ... ,""oklevel
[0] aa7556f5-78ad-4859-b5a3-cb4209a0a400 (df59fbd5-37bd-496c-ad85-1e830208beb9) New New New ds_transition to WaitActive"","v":0,"name":"crucible",""level":30,"v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.535576686Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
,:"time30"{:"2023-09-22T23:08:02.535583651Z""msg",:""hostname":"ip-10-150-1-74.us-west-2.compute.internal[0] Transition from New to WaitActive"",","pid"v:":42910,"name}":
"crucible",",level""{:30time":""msg":"2023-09-22T23:08:02.535604328Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","[0] 40de3eeb-c819-466e-88b4-e1712ed524bc (5b45a21e-af03-4c26-80d4-f98a411c8304) WaitQuorum New New ds_transition to Activepid"":,,"4291v":0,"name":"crucible","level":30}
{,""time":"time":"2023-09-22T23:08:02.535640139Zmsg":"","hostname":"[0] b734394e-ac55-46b3-89f9-8e132a0e6ae7 (c367f5ab-c4f8-42c0-882b-6f763e0900ac) WaitQuorum New New ds_transition to Active"ip-10-150-1-74.us-west-2.compute.internal",,""v"pid"::4291}
0,"name":"{crucible","""level"msg"::2023-09-22T23:08:02.535623832Z30","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{""msg":"[0] Transition from WaitQuorum to Active","v":0,"name":,"[0] aa7556f5-78ad-4859-b5a3-cb4209a0a400 (df59fbd5-37bd-496c-ad85-1e830208beb9) WaitActive New New ds_transition to WaitQuorumcrucible""",,""timevlevel""::0"30,:""name":"crucible"2023-09-22T23:08:02.535681203Z","level,"":30hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
,"time":"2023-09-22T23:08:02.535710264Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal"{,"pid":4291},
""time"msg:{"":"2023-09-22T23:08:02.535704028Z""msg":",[0] Transition from WaitQuorum to Active""hostname,":""ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 40de3eeb-c819-466e-88b4-e1712ed524bc (5b45a21e-af03-4c26-80d4-f98a411c8304) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.53577414Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30[0] Transition from WaitActive to WaitQuorum","v":0,"name":","crucible"time",:""level":302023-09-22T23:08:02.535806691Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{,""timemsg""::""2023-09-22T23:08:02.535819291Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal[1] 40de3eeb-c819-466e-88b4-e1712ed524bc (5b45a21e-af03-4c26-80d4-f98a411c8304) Active WaitActive New ds_transition to WaitQuorum"",",pid"":v"4291:0,"}
name":"crucible","{level":30"msg":"[0] aa7556f5-78ad-4859-b5a3-cb4209a0a400 (df59fbd5-37bd-496c-ad85-1e830208beb9) WaitQuorum New New ds_transition to Active","v":0,"name":",crucible""time,"":"level":302023-09-22T23:08:02.535846134Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{",msg"":"time":"[1] Transition from WaitActive to WaitQuorum2023-09-22T23:08:02.535858245Z"",,""v":hostname"0:","name":"crucible"ip-10-150-1-74.us-west-2.compute.internal",",",pidlevel""::429130"}
time":"{2023-09-22T23:08:02.535854732Z""msg":","hostname":"[0] Transition from WaitQuorum to Active",,""timevip-10-150-1-74.us-west-2.compute.internal"""::0",,"2023-09-22T23:08:02.535879986Z"",name""pid"hostname:""::crucible""4291,"levelip-10-150-1-74.us-west-2.compute.internal"":,30"}pid"
:4291}
,"{time{":""msg":2023-09-22T23:08:02.535902865Z"",""hostname"msg:"":"[1] 40de3eeb-c819-466e-88b4-e1712ed524bc (5b45a21e-af03-4c26-80d4-f98a411c8304) Active WaitQuorum New ds_transition to Active"ip-10-150-1-74.us-west-2.compute.internal",","v"pid:":04291,"}name
"[1] b734394e-ac55-46b3-89f9-8e132a0e6ae7 (c367f5ab-c4f8-42c0-882b-6f763e0900ac) Active New New ds_transition to WaitActive":{",""cruciblemsg"",":level"":v30":0,"name":"crucible","[1] aa7556f5-78ad-4859-b5a3-cb4209a0a400 (df59fbd5-37bd-496c-ad85-1e830208beb9) Active New New ds_transition to WaitActive"level",:"30v":0,",name"":time"":"crucible","level2023-09-22T23:08:02.535933737Z"":30,"hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{,"time"":"msg":","2023-09-22T23:08:02.535946151Z"time"[1] Transition from WaitQuorum to Active,"":hostname,""":v"":02023-09-22T23:08:02.535941426Z,"ip-10-150-1-74.us-west-2.compute.internal"",name,"""pid:"":hostname"crucible4291":"},
"ip-10-150-1-74.us-west-2.compute.internal"level{",:"30msg"":pid"":4291[1] Transition from New to WaitActive"},
","v":time"0:","name":"2023-09-22T23:08:02.535977628Z"crucible",",{hostname"":"level":30"ip-10-150-1-74.us-west-2.compute.internal"msg,"":pid"":4291}
,[1] Transition from New to WaitActive""{,time"":"msg""v":2023-09-22T23:08:02.535994918Z"":,"0hostname,"":name""[2] 40de3eeb-c819-466e-88b4-e1712ed524bc (5b45a21e-af03-4c26-80d4-f98a411c8304) Active Active New ds_transition to WaitActive":",ip-10-150-1-74.us-west-2.compute.internal""crucible",v"",pid:"0:,4291""}name
"level":"{:crucible"",30msg"":level"":30[1] aa7556f5-78ad-4859-b5a3-cb4209a0a400 (df59fbd5-37bd-496c-ad85-1e830208beb9) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible",","level"time:"30:"2023-09-22T23:08:02.536031755Z","hostname":",ip-10-150-1-74.us-west-2.compute.internal"","timepid"":,4291":"}time
"2023-09-22T23:08:02.536030575Z":{",""hostname2023-09-22T23:08:02.536042335Zmsg""",:"":hostname"":[2] Transition from New to WaitActive"","v":ip-10-150-1-74.us-west-2.compute.internal"0ip-10-150-1-74.us-west-2.compute.internal,"","name"pid:"":,"4291crucible"}pid,
"":level{4291":"30msg":}"
[1] Transition from WaitActive to WaitQuorum","v":0,"name":"{crucible,"","time":level"":30"2023-09-22T23:08:02.536077063Zmsg"":","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291[1] b734394e-ac55-46b3-89f9-8e132a0e6ae7 (c367f5ab-c4f8-42c0-882b-6f763e0900ac) Active WaitActive New ds_transition to WaitQuorum"},
","time{"v"::""0msg":,"2023-09-22T23:08:02.536088146Z"",name"":"hostname"crucible":","[2] 40de3eeb-c819-466e-88b4-e1712ed524bc (5b45a21e-af03-4c26-80d4-f98a411c8304) Active Active WaitActive ds_transition to WaitQuorumlevel"ip-10-150-1-74.us-west-2.compute.internal"":,,30""v"pid:":04291,"name}"
:"crucible",{"level":"30msg":"[1] aa7556f5-78ad-4859-b5a3-cb4209a0a400 (df59fbd5-37bd-496c-ad85-1e830208beb9) Active WaitQuorum New ds_transition to Active","v":0,"{name",:"","crucible"time","time"level:":"":"msg30"2023-09-22T23:08:02.536128669Z:2023-09-22T23:08:02.536121796Z""",","hostname":"hostname":Crucible stats registered with UUID: 021f3454-29b2-4ff1-97c2-40bc8a3180a2""ip-10-150-1-74.us-west-2.compute.internal",",pid,ip-10-150-1-74.us-west-2.compute.internal"""":,v"time"4291:pid""::"04291},2023-09-22T23:08:02.536147292Z
}"
,""{{hostname"name"":msg"":":"ip-10-150-1-74.us-west-2.compute.internal"msg""[2] Transition from WaitActive to WaitQuorum",,crucible""pidv":"0:,4291"":}name
",:""{levelcrucible"""",msg""[1] Transition from WaitActive to WaitQuorum:":"level30",:[1] Transition from WaitQuorum to Active30"",v"":v"0:0,,""name"name"::""crucible,"",crucible""time,level""::""30level":2023-09-22T23:08:02.536208499Z"30,"hostname":"ip-10-150-1-74.us-west-2.compute.internal",,""time"pid,""::time""4291:"}
2023-09-22T23:08:02.536223646Z"2023-09-22T23:08:02.536207099Z,{""hostname"":msg"",:","ip-10-150-1-74.us-west-2.compute.internal"hostname"",:"time"":[2] 40de3eeb-c819-466e-88b4-e1712ed524bc (5b45a21e-af03-4c26-80d4-f98a411c8304) Active Active WaitQuorum ds_transition to Activepid"":"ip-10-150-1-74.us-west-2.compute.internal",4291"2023-09-22T23:08:02.536225529Z"v}"
:,,0{,""""msgname""::"pid""hostname:crucible"4291,":[2] aa7556f5-78ad-4859-b5a3-cb4209a0a400 (df59fbd5-37bd-496c-ad85-1e830208beb9) Active Active New ds_transition to WaitActive""level,"":v30"}:"
ip-10-150-1-74.us-west-2.compute.internal"0,",time""name"::"",{crucible2023-09-22T23:08:02.536273588Z"","",pid"hostname"""level:"":msg"::ip-10-150-1-74.us-west-2.compute.internal30"4291,""}pid
Crucible 021f3454-29b2-4ff1-97c2-40bc8a3180a2 has session id: f8ec93e4-add2-454d-a49c-7f44bf328c3f"":,"4291time":"},"
2023-09-22T23:08:02.5362981Z"v"{,{""msg:hostname""::""0","[2] Transition from WaitQuorum to Activeip-10-150-1-74.us-west-2.compute.internal"",,""namevpid""::msg""04291,:}"
name"::""{crucible"""crucible",msg"",level:"":"[1] b734394e-ac55-46b3-89f9-8e132a0e6ae7 (c367f5ab-c4f8-42c0-882b-6f763e0900ac) Active WaitQuorum New ds_transition to Active"30[2] Transition from New to WaitActive"level,"",:"vv"30"::00,,""time"name:"",":name2023-09-22T23:08:02.536342853Z""",crucible"":hostname,""level""::"30crucible","level"ip-10-150-1-74.us-west-2.compute.internal":,"30pid":4291},
,""time"{:time""":msg"2023-09-22T23:08:02.536362516Z"":","2023-09-22T23:08:02.536349838Z"hostname":","hostname":"ip-10-150-1-74.us-west-2.compute.internal"40de3eeb-c819-466e-88b4-e1712ed524bc is now active with session: 5b45a21e-af03-4c26-80d4-f98a411c8304,""ip-10-150-1-74.us-west-2.compute.internal"pid,",:"4291v"}:
0",{"pid""namemsg""::"",:"crucible4291"time",:}"
"[2] aa7556f5-78ad-4859-b5a3-cb4209a0a400 (df59fbd5-37bd-496c-ad85-1e830208beb9) Active Active WaitActive ds_transition to WaitQuorumlevel""2023-09-22T23:08:02.536367263Z",:"30v",:"0{hostname",:""name":"crucible""ip-10-150-1-74.us-west-2.compute.internal,msg,""",time""level""::":"pid"2023-09-22T23:08:02.536407789Z30":,"4291hostname":"}[0] 021f3454-29b2-4ff1-97c2-40bc8a3180a2 (55e767e0-c831-4573-b14d-c7e5c142adea) New New New ds_transition to WaitActive"
ip-10-150-1-74.us-west-2.compute.internal,"",,"timepid"""{::4291"v}"
2023-09-22T23:08:02.536426454Z:"{"0",msg"msg"hostname""::"":,""name"ip-10-150-1-74.us-west-2.compute.internal:"[1] Transition from WaitQuorum to Active"[1] 40de3eeb-c819-466e-88b4-e1712ed524bc (5b45a21e-af03-4c26-80d4-f98a411c8304) Active Active Active ds_transition to Faulted,","",vcrucible""v"pid:"0"",:":name4291"0,}:
"",level""{crucible":",msgname""":30:level""":30crucible"[2] Transition from WaitActive to WaitQuorum",","level"v:":300,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.536489607Z",",hostname"":","time"time":ip-10-150-1-74.us-west-2.compute.internal:"",""2023-09-22T23:08:02.536498468Zpid,2023-09-22T23:08:02.536487138Z""",:"4291"hostname}"
:,time":"{"hostname"":"ip-10-150-1-74.us-west-2.compute.internalmsg"","":pid""2023-09-22T23:08:02.536495066Z":ip-10-150-1-74.us-west-2.compute.internal[1] Transition from Active to Faulted""4291,,""v}"
:pid",0{,:"4291"msg"":"name"}hostname"
:[2] aa7556f5-78ad-4859-b5a3-cb4209a0a400 (df59fbd5-37bd-496c-ad85-1e830208beb9) Active Active WaitQuorum ds_transition to Active"":crucible,"""v",:"0{,level"":name30"ip-10-150-1-74.us-west-2.compute.internal:""",msgcrucible""":"pid",:[0] Transition from New to WaitActive4291",""}timelevel,"
":"30:v"":0{2023-09-22T23:08:02.536563527Z",",name"""hostname:msg"":,"":crucible""time,"":ip-10-150-1-74.us-west-2.compute.internal"""level"2023-09-22T23:08:02.536583309Z,"","pid:hostname""::4291"[2] b734394e-ac55-46b3-89f9-8e132a0e6ae7 (c367f5ab-c4f8-42c0-882b-6f763e0900ac) Active Active New ds_transition to WaitActive"30}ip-10-150-1-74.us-west-2.compute.internal
",,"{"v""pid"msg:"4291:"}:
0,"{name":"[1] 40de3eeb-c819-466e-88b4-e1712ed524bc (5b45a21e-af03-4c26-80d4-f98a411c8304) Active Faulted Active ds_transition to LiveRepairReady""msg",:""crucible"v",:[2] Transition from WaitQuorum to Active"0,,"",v""name:"0time,:"""namecrucible"":,:"""levelcrucible"":,30""level"level":2023-09-22T23:08:02.536608176Z"30:,"30hostname":"ip-10-150-1-74.us-west-2.compute.internal",","pid"time:":"4291,"time2023-09-22T23:08:02.536642624Z"":"},
2023-09-22T23:08:02.536646513Z"",hostname"":"hostname":"ip-10-150-1-74.us-west-2.compute.internal"{,ip-10-150-1-74.us-west-2.compute.internal,"",pid""":pid4291""time"}:msg":4291
}:
{"""{msg":2023-09-22T23:08:02.536648831Z"""msg":,""[1] Transition from Faulted to LiveRepairReady"hostname":[0] 021f3454-29b2-4ff1-97c2-40bc8a3180a2 (55e767e0-c831-4573-b14d-c7e5c142adea) WaitActive New New ds_transition to WaitQuorum""aa7556f5-78ad-4859-b5a3-cb4209a0a400 is now active with session: df59fbd5-37bd-496c-ad85-1e830208beb9,"",,v""v:"0:,0"",name"vname""::"""cruciblecrucible"":,,""0levellevel"":ip-10-150-1-74.us-west-2.compute.internal",30:"30,name"":"pid":crucible"4291,"level":}30
,,""timetime""::""2023-09-22T23:08:02.53671336Z2023-09-22T23:08:02.536712184Z"{",","hostname":hostname"":""msg"ip-10-150-1-74.us-west-2.compute.internalip-10-150-1-74.us-west-2.compute.internal"",:",pid""pid:4291",}"
:"4291[2] Transition from New to WaitActive}{time"
""msg{"::""msg,""":v""2023-09-22T23:08:02.536720344Z":[1] 40de3eeb-c819-466e-88b4-e1712ed524bc (5b45a21e-af03-4c26-80d4-f98a411c8304) Active LiveRepairReady Active ds_transition to LiveRepair",,""0[1] aa7556f5-78ad-4859-b5a3-cb4209a0a400 (df59fbd5-37bd-496c-ad85-1e830208beb9) Active Active Active ds_transition to Faultedv"",hostname":,0""v,"":name0",:"name"name"":":crucible:"",""cruciblelevel"",:ip-10-150-1-74.us-west-2.compute.internal"30",crucible"level","":pid":304291}
,"time":,""time":"2023-09-22T23:08:02.536780715Z"{2023-09-22T23:08:02.536787268Z,"","hostname":hostname"":""msg":"ip-10-150-1-74.us-west-2.compute.internalip-10-150-1-74.us-west-2.compute.internal"",,""pidpid""::42914291[0] Transition from WaitActive to WaitQuorum"}}

{level,""{msg""":msg:"":30v"":[1] Transition from LiveRepairReady to LiveRepair"0,[1] Transition from Active to Faulted",v"",:"0",v""name"name:"0::,"""namecrucible"":crucible"","cruciblelevel"",:"30,"level":level,"30:"30time":"2023-09-22T23:08:02.536818646Z",,""hostnametime""::,"""time":2023-09-22T23:08:02.536834851Z"","ip-10-150-1-74.us-west-2.compute.internal"2023-09-22T23:08:02.536838866Zhostname"",:"",hostname"":ip-10-150-1-74.us-west-2.compute.internalpid""",",:ip-10-150-1-74.us-west-2.compute.internalpid"",:"4291pid"4291}"
:}time"
:4291"}
2023-09-22T23:08:02.536840739Z"{{,"hostname":"""msg":"ip-10-150-1-74.us-west-2.compute.internal"msg,""pid":":4291[1] aa7556f5-78ad-4859-b5a3-cb4209a0a400 (df59fbd5-37bd-496c-ad85-1e830208beb9) Active Faulted Active ds_transition to LiveRepairReady"},
[2] b734394e-ac55-46b3-89f9-8e132a0e6ae7 (c367f5ab-c4f8-42c0-882b-6f763e0900ac) Active Active WaitActive ds_transition to WaitQuorum{","v"":msg0":","name":"crucible","level":30[0] 021f3454-29b2-4ff1-97c2-40bc8a3180a2 (55e767e0-c831-4573-b14d-c7e5c142adea) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.536915928Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
,""time":"v":{02023-09-22T23:08:02.536928533Z",",name"""hostname"msg:":"":"crucible"ip-10-150-1-74.us-west-2.compute.internal",,[2] Transition from WaitActive to WaitQuorum"""level",:"pid"v":304291:0,"}name"
:"crucible","level":30{"msg":"[0] Transition from WaitQuorum to Active","v":,"0time,"":"name":"crucible"2023-09-22T23:08:02.536969915Z","level":,"30,hostname"":"time":"ip-10-150-1-74.us-west-2.compute.internal"2023-09-22T23:08:02.53698014Z",","pid":hostname4291":"}
ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"time"{}:"
"2023-09-22T23:08:02.536995771Z"msg":","hostname":"{[1] Transition from Faulted to LiveRepairReady","ip-10-150-1-74.us-west-2.compute.internal"v",:"0pid",":"4291name"msg"}::"
"crucible"[2] b734394e-ac55-46b3-89f9-8e132a0e6ae7 (c367f5ab-c4f8-42c0-882b-6f763e0900ac) Active Active WaitQuorum ds_transition to Active",",level{"":v"30":msg"0:","name":"crucible","level":30[1] 021f3454-29b2-4ff1-97c2-40bc8a3180a2 (55e767e0-c831-4573-b14d-c7e5c142adea) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.537046497Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal,"","time":"pid":42912023-09-22T23:08:02.537053115Z"},
","hostname":"time":{"ip-10-150-1-74.us-west-2.compute.internal","pid"2023-09-22T23:08:02.537060141Z"":,4291"msg"hostname"::""}
ip-10-150-1-74.us-west-2.compute.internal","[1] aa7556f5-78ad-4859-b5a3-cb4209a0a400 (df59fbd5-37bd-496c-ad85-1e830208beb9) Active LiveRepairReady Active ds_transition to LiveRepairpid"":{4291,"v":}0"
msg":","name":"crucible[2] Transition from WaitQuorum to Active""{,,""level"v"":msg30"::"0,"name":[1] Transition from New to WaitActive"",crucible"",v"":level"0:,"30name":"crucible","level":30,"time":"test live_repair::repair_test::test_live_repair_deps_repair_rspan_right ... 2023-09-22T23:08:02.537117628Z"ok,"
hostname",":",time"ip-10-150-1-74.us-west-2.compute.internal":"",time"":"pid":2023-09-22T23:08:02.537128175Z"2023-09-22T23:08:02.537131316Z"4291,",}hostname""
hostname"::"{""ip-10-150-1-74.us-west-2.compute.internalip-10-150-1-74.us-west-2.compute.internal""msg":",,"[1] Transition from LiveRepairReady to LiveRepair"pid"":,"4291pidv""::0},"4291name"
:"}crucible"
{,"level":30"{msg":""msg":"b734394e-ac55-46b3-89f9-8e132a0e6ae7 is now active with session: c367f5ab-c4f8-42c0-882b-6f763e0900ac","v":0,"name":"crucible"[1] 021f3454-29b2-4ff1-97c2-40bc8a3180a2 (55e767e0-c831-4573-b14d-c7e5c142adea) Active WaitActive New ds_transition to WaitQuorum",,,"""v"timelevel"::"30:0","name":"2023-09-22T23:08:02.537192267Z"crucible",","hostname"level"::"30ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
,"time":"{2023-09-22T23:08:02.53721269Z"","msg"hostname":":","time":ip-10-150-1-74.us-west-2.compute.internal""Write to Extent 0:2:9 under repair,"2023-09-22T23:08:02.53722009Z""pid",:,""4291hostname"v":":}0
ip-10-150-1-74.us-west-2.compute.internal",,""name":"pid"{:crucible"4291,"level""}msg
"::"40{"[1] b734394e-ac55-46b3-89f9-8e132a0e6ae7 (c367f5ab-c4f8-42c0-882b-6f763e0900ac) Active Active Active ds_transition to Faulted"msg":","v":0[1] Transition from WaitActive to WaitQuorum",",name":""v"crucible":,"0level",:,30""name":time"":crucible"","level":2023-09-22T23:08:02.537269921Z"30,"hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
,"time":"2023-09-22T23:08:02.537286276Z","hostname":","ip-10-150-1-74.us-west-2.compute.internal"time":,""pid":42912023-09-22T23:08:02.537292591Z"},
{"msg":"[1] Transition from Active to Faulted","v":0,""name":"crucible"hostname",":level":"30ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{,"time":""msg":"2023-09-22T23:08:02.537380183Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal",[1] 021f3454-29b2-4ff1-97c2-40bc8a3180a2 (55e767e0-c831-4573-b14d-c7e5c142adea) Active WaitQuorum New ds_transition to Active""pid",":v4291":0},"
name":"crucible","level":{30"msg":"[1] b734394e-ac55-46b3-89f9-8e132a0e6ae7 (c367f5ab-c4f8-42c0-882b-6f763e0900ac) Active Faulted Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.537419191Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"}time
":"2023-09-22T23:08:02.537433516Z","{hostname":""ip-10-150-1-74.us-west-2.compute.internal"msg",:""pid":4291[1] Transition from WaitQuorum to Active","}v"
:0,"name":"crucible"{,"level":30"msg":"[1] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.537476549Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"}time
":"2023-09-22T23:08:02.537486141Z","{hostname":""ip-10-150-1-74.us-west-2.compute.internal"msg":,""pid":4291}
[2] 021f3454-29b2-4ff1-97c2-40bc8a3180a2 (55e767e0-c831-4573-b14d-c7e5c142adea) Active Active New ds_transition to WaitActive","v":0{,"name":"crucible",""level":msg30":"[1] b734394e-ac55-46b3-89f9-8e132a0e6ae7 (c367f5ab-c4f8-42c0-882b-6f763e0900ac) Active LiveRepairReady Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30{,"time"":"msg":"2023-09-22T23:08:02.537529744Z","hostname":"Crucible stats registered with UUID: e25c51fe-6bce-4500-9128-4f204e1a9445"ip-10-150-1-74.us-west-2.compute.internal",","pid"v"::42910,"},"
name"time"::""crucible{"2023-09-22T23:08:02.537541991Z",",level"""msghostname"::"30":"ip-10-150-1-74.us-west-2.compute.internal"[2] Transition from New to WaitActive",",pid"":v"4291:0,"}name"
test live_repair::repair_test::test_live_repair_deps_repair_write ... :"{,ok
"crucible""timemsg"",:":""level":302023-09-22T23:08:02.537574802Z"[1] Transition from LiveRepairReady to LiveRepair",","hostname"v"::"0,"name":"ip-10-150-1-74.us-west-2.compute.internal"crucible",",pid"":level"4291:30}
,"time":"2023-09-22T23:08:02.537609141Z"{,"hostname":""msg":"ip-10-150-1-74.us-west-2.compute.internal","pid":,4291"Crucible e25c51fe-6bce-4500-9128-4f204e1a9445 has session id: 15fb88f2-fded-4cce-bdd4-93603c9bc51f"time"}:,"
v"":0,"2023-09-22T23:08:02.53762148Z"name"{:,""crucible"hostname",""msg":":level"":ip-10-150-1-74.us-west-2.compute.internal"30,"pid":4291}
[2] 021f3454-29b2-4ff1-97c2-40bc8a3180a2 (55e767e0-c831-4573-b14d-c7e5c142adea) Active Active WaitActive ds_transition to WaitQuorum","v":0,"{name":"crucible","level"":msg"30,:""time":"Write to Extent 0:2:9 under repair","2023-09-22T23:08:02.537665051Z"v":,"0hostname",:""name":"crucible"ip-10-150-1-74.us-west-2.compute.internal",,""levelpid""::429140,"}
time":"2023-09-22T23:08:02.537681261Z"{,"hostname":""msg":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
[0] e25c51fe-6bce-4500-9128-4f204e1a9445 (04223c12-1924-4b5a-ac33-ec762a7ea271) New New New ds_transition to WaitActive",","timev"":{":0,"2023-09-22T23:08:02.537697881Z"name":",""msg":"hostname"crucible":",[2] Transition from WaitActive to WaitQuorum""ip-10-150-1-74.us-west-2.compute.internallevel,""":,"v30"pid"::04291,"name":"}crucible"
,"level":30{",msg"":"time":"Write 1:0:9 past extent under repair 0"2023-09-22T23:08:02.537739789Z","v,":"0hostname",:""name":"crucible",ip-10-150-1-74.us-west-2.compute.internal"",,time"":"pid"level""::4291402023-09-22T23:08:02.537750983Z"}
,"hostname":"ip-10-150-1-74.us-west-2.compute.internal"{,"pid":4291"}msg
":"[0] Transition from New to WaitActive","v"{:0,"name":""crucible"msg":,""level":30[2] 021f3454-29b2-4ff1-97c2-40bc8a3180a2 (55e767e0-c831-4573-b14d-c7e5c142adea) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.537776463Z",",time"":"hostname":"2023-09-22T23:08:02.537805273Z",ip-10-150-1-74.us-west-2.compute.internal"","hostname":"pid",:"4291ip-10-150-1-74.us-west-2.compute.internal"time,""}:"
2023-09-22T23:08:02.537814609Z"pid":,"4291hostname":"}{
ip-10-150-1-74.us-west-2.compute.internal"","msgpid"":{4291:"}
Write 1:1:9 past extent under repair 0""msg,""v":{":0,"name"":"msg":crucible""[0] e25c51fe-6bce-4500-9128-4f204e1a9445 (04223c12-1924-4b5a-ac33-ec762a7ea271) WaitActive New New ds_transition to WaitQuorum",",level[2] Transition from WaitQuorum to Active""",v""v::"0:,0"40,"name"name"::""crucible"crucible",","level"level"::3030,"time":"2023-09-22T23:08:02.53788561Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal",",pid"":time,4291"":}time
"":"2023-09-22T23:08:02.537894132Z"2023-09-22T23:08:02.537894443Z"{,,""hostname"hostname"":msg":"":"ip-10-150-1-74.us-west-2.compute.internal","pid"ip-10-150-1-74.us-west-2.compute.internal"IO Write 1008 on eur 0 Added deps 1,:""4291pid",:"}4291v"
:}0
,{"name":"crucible"",msg":"{"level[0] Transition from WaitActive to WaitQuorum"":,"40v":"0,"msgname":""crucible":,""level":30021f3454-29b2-4ff1-97c2-40bc8a3180a2 is now active with session: 55e767e0-c831-4573-b14d-c7e5c142adea","v":0,"name":"crucible","level":,","30time":"time":"2023-09-22T23:08:02.537966253Z","hostname":2023-09-22T23:08:02.537959938Z"","ip-10-150-1-74.us-west-2.compute.internal"hostname",":pid":"4291}
ip-10-150-1-74.us-west-2.compute.internal","{pid":"4291msg":"}
[0] e25c51fe-6bce-4500-9128-4f204e1a9445 (04223c12-1924-4b5a-ac33-ec762a7ea271) WaitQuorum New New ds_transition to Active",","v":0time",":name":""crucible","level":302023-09-22T23:08:02.537978013Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":,"4291time":"}2023-09-22T23:08:02.538005773Z"
,"hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}{
"{msg":""msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30[1] 021f3454-29b2-4ff1-97c2-40bc8a3180a2 (55e767e0-c831-4573-b14d-c7e5c142adea) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":,"30time":"2023-09-22T23:08:02.538033115Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":","time":"2023-09-22T23:08:02.538040588Z","[1] e25c51fe-6bce-4500-9128-4f204e1a9445 (04223c12-1924-4b5a-ac33-ec762a7ea271) Active New New ds_transition to WaitActivehostname"":","v":0,ip-10-150-1-74.us-west-2.compute.internal"","name"pid"::"4291crucible","}level{"
":30msg":"{Crucible stats registered with UUID: af820581-d317-46fc-9b93-01ba46c1da49",""msgv""::"0,"name"[1] Transition from Active to Faulted":",crucible,"""time,"":level"":v"302023-09-22T23:08:02.538075872Z":,"0hostname",:""name":"crucible","ip-10-150-1-74.us-west-2.compute.internal"level",:"30pid":4291}
,"{time":""msg":"2023-09-22T23:08:02.53809334Z","[1] Transition from New to WaitActivehostname"":","v":0,"ip-10-150-1-74.us-west-2.compute.internalname"":,""pidcrucible"":,4291"level":}30,"
time":"{2023-09-22T23:08:02.538103523Z","hostname"":msg,"""time:"":"ip-10-150-1-74.us-west-2.compute.internal"2023-09-22T23:08:02.538119971Z,"",pid"Crucible af820581-d317-46fc-9b93-01ba46c1da49 has session id: 77262da8-9084-4487-8fc8-aafdaf5d60a6"":,hostname"4291v""::"0},
"ip-10-150-1-74.us-west-2.compute.internal"name",:""pid"crucible:"4291,"}{
level""msg"{::30""msg":"[1] 021f3454-29b2-4ff1-97c2-40bc8a3180a2 (55e767e0-c831-4573-b14d-c7e5c142adea) Active Faulted Active ds_transition to LiveRepairReady"[1] e25c51fe-6bce-4500-9128-4f204e1a9445 (04223c12-1924-4b5a-ac33-ec762a7ea271) Active WaitActive New ds_transition to WaitQuorum,""v,"",v"":time:"0:,""0name"2023-09-22T23:08:02.53815581Z":,","name"crucible"":hostname,"":"level"":crucible"30ip-10-150-1-74.us-west-2.compute.internal",,""level"pid:"30:4291}
,"time":"{2023-09-22T23:08:02.538179384Z"",msg"":"hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
[0] af820581-d317-46fc-9b93-01ba46c1da49 (4bd2fdd5-afb2-479f-8091-75ec93e8aa7a) New New New ds_transition to WaitActive",",v{":""0time,msg""name:"":"":"crucible[1] Transition from WaitActive to WaitQuorum"",","levelv""2023-09-22T23:08:02.538184389Z"::030,,""hostname"name":test live_repair::repair_test::test_live_repair_deps_repair_wafter ... ":,""oktimecrucible"":,""
2023-09-22T23:08:02.538211347Zlevel"","hostname":"ip-10-150-1-74.us-west-2.compute.internal",":pid30":4291}
{"ip-10-150-1-74.us-west-2.compute.internalmsg","":"time",:"[0] Transition from New to WaitActive"","2023-09-22T23:08:02.538243789Zv""pid",:"0:hostname,"":name""4291:"crucibleip-10-150-1-74.us-west-2.compute.internal"},"",
level"":pid"30:4291}
{{"msg"":"msg":","time":"[1] Transition from Faulted to LiveRepairReady"2023-09-22T23:08:02.538271832Z",",[1] e25c51fe-6bce-4500-9128-4f204e1a9445 (04223c12-1924-4b5a-ac33-ec762a7ea271) Active WaitQuorum New ds_transition to Active""v,hostname""v"::"0",:"ip-10-150-1-74.us-west-2.compute.internal"name"0:,"",pidcrucible"":,""4291name":level}"
:"30{crucible",""msg"level"::"30,"time[0] af820581-d317-46fc-9b93-01ba46c1da49 (4bd2fdd5-afb2-479f-8091-75ec93e8aa7a) WaitActive New New ds_transition to WaitQuorum"":","v":02023-09-22T23:08:02.538302218Z,""name,"":"hostname"crucible:"","level":30ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{","msg":time","":"time":"[1] Transition from WaitQuorum to Active"2023-09-22T23:08:02.538307877Z"2023-09-22T23:08:02.538318214Z,""v,"":,hostname0",:"""name"hostname":ip-10-150-1-74.us-west-2.compute.internal"":,crucible""pid",:"4291"level}"
:ip-10-150-1-74.us-west-2.compute.internal"30{,"pid"":msg"4291:"}
,[0] Transition from WaitActive to WaitQuorum""time,"":"v":0,"2023-09-22T23:08:02.53834631Z"name{,"":"hostname"crucible:"","level"":ip-10-150-1-74.us-west-2.compute.internal30msg"":","pid":4291}
[1] 021f3454-29b2-4ff1-97c2-40bc8a3180a2 (55e767e0-c831-4573-b14d-c7e5c142adea) Active LiveRepairReady Active ds_transition to LiveRepair"{,"",timemsg"""::""v":02023-09-22T23:08:02.53836423Z",","name":hostname""[2] e25c51fe-6bce-4500-9128-4f204e1a9445 (04223c12-1924-4b5a-ac33-ec762a7ea271) Active Active New ds_transition to WaitActive:crucible""",,""levelvip-10-150-1-74.us-west-2.compute.internal"":,0","pid":name""::4291"30crucible}"
,"level":30{"msg":"[0] af820581-d317-46fc-9b93-01ba46c1da49 (4bd2fdd5-afb2-479f-8091-75ec93e8aa7a) WaitQuorum New New ds_transition to Active","v":0,"name",:""time"crucible:"","level":302023-09-22T23:08:02.538396979Z",","hostname":time"":"2023-09-22T23:08:02.538393651Zip-10-150-1-74.us-west-2.compute.internal"","pid",:,4291""}time
"hostname":{":""2023-09-22T23:08:02.538409722Zmsg"":",ip-10-150-1-74.us-west-2.compute.internal"",[2] Transition from New to WaitActivehostname"":,"""v"pid":ip-10-150-1-74.us-west-2.compute.internal0",,"":pidname4291""::"4291crucible"},
"}level
{":30"msg":"{[0] Transition from WaitQuorum to Active","v":0,""name"msg"::",""timecrucible"":","[1] Transition from LiveRepairReady to LiveRepairlevel2023-09-22T23:08:02.53844585Z"":,30"","hostname"v"::"0,"name":"ip-10-150-1-74.us-west-2.compute.internal"crucible",",,pid""time:"4291:"}"
2023-09-22T23:08:02.538458949Z"level":{,"30hostname"":msg"":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
[2] e25c51fe-6bce-4500-9128-4f204e1a9445 (04223c12-1924-4b5a-ac33-ec762a7ea271) Active Active WaitActive ds_transition to WaitQuorum","v{":0,""namemsg""::""crucible","level":30[1] af820581-d317-46fc-9b93-01ba46c1da49 (4bd2fdd5-afb2-479f-8091-75ec93e8aa7a) Active New New ds_transition to WaitActive",",v"":time"0:,""name":,""2023-09-22T23:08:02.538476202Zcrucibletime"",:"""level":302023-09-22T23:08:02.53848912Z",","hostname":hostname"":"ip-10-150-1-74.us-west-2.compute.internal","ip-10-150-1-74.us-west-2.compute.internalpid"":,,""4291timepid""::"4291}
2023-09-22T23:08:02.53850299Z}"
,"hostname":"{"msg"ip-10-150-1-74.us-west-2.compute.internal:"","pid":4291[2] Transition from WaitActive to WaitQuorum}"
,"v":0{,"name":""msg":crucible"","level":30[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.538534993Z","hostname":","time":"ip-10-150-1-74.us-west-2.compute.internal","pid"2023-09-22T23:08:02.538540456Z:"4291,"}
hostname":"{"ip-10-150-1-74.us-west-2.compute.internal"msg",:""pid":4291}
{[2] e25c51fe-6bce-4500-9128-4f204e1a9445 (04223c12-1924-4b5a-ac33-ec762a7ea271) Active Active WaitQuorum ds_transition to Active"","msgv""::"0,"name":"crucible","level":30[1] af820581-d317-46fc-9b93-01ba46c1da49 (4bd2fdd5-afb2-479f-8091-75ec93e8aa7a) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.538569113Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal,"","time"pid:"":4291}
2023-09-22T23:08:02.538576057Z","hostname":{""msg"ip-10-150-1-74.us-west-2.compute.internal:"","pid":4291[2] Transition from WaitQuorum to Active"}
,"v":0{,"name":""msgcrucible"":,""level":30[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.538608779Z","hostname":","time":"ip-10-150-1-74.us-west-2.compute.internal","2023-09-22T23:08:02.538614376Z"pid":,"4291hostname":}"
ip-10-150-1-74.us-west-2.compute.internal","pid":{4291}
"msg":"{"msg":"e25c51fe-6bce-4500-9128-4f204e1a9445 is now active with session: 04223c12-1924-4b5a-ac33-ec762a7ea271","v":0,"name":"[1] af820581-d317-46fc-9b93-01ba46c1da49 (4bd2fdd5-afb2-479f-8091-75ec93e8aa7a) Active WaitQuorum New ds_transition to Active"crucible",","v":level0":,"30name":"crucible","level":30,"time":","time2023-09-22T23:08:02.538645496Z"":","hostname":"2023-09-22T23:08:02.538648301Z","hostname":ip-10-150-1-74.us-west-2.compute.internal"","pid":4291ip-10-150-1-74.us-west-2.compute.internal"},
"pid":4291{}
"msg":"{"msg":"[1] Transition from WaitQuorum to Active","[1] e25c51fe-6bce-4500-9128-4f204e1a9445 (04223c12-1924-4b5a-ac33-ec762a7ea271) Active Active Active ds_transition to Faultedv"":,0",v"":name0":,""namecrucible"":","crucible"level",:"30level":30,,""timetime""::""2023-09-22T23:08:02.538681242Z2023-09-22T23:08:02.538681975Z"",,""hostnamehostname""::""{ip-10-150-1-74.us-west-2.compute.internalip-10-150-1-74.us-west-2.compute.internal"",,""pidpid""::42914291}}

{"{"msgmsg"""msg::"":""[1] Transition from Active to Faulted","Crucible stats registered with UUID: 4b1de70b-0c52-4541-acb2-5a791e8f874cv"":[2] af820581-d317-46fc-9b93-01ba46c1da49 (4bd2fdd5-afb2-479f-8091-75ec93e8aa7a) Active Active New ds_transition to WaitActive0",",",vname""::0"",crucible""name",:""vcruciblelevel"",:"30"level"::300,"name":"crucible","level":30,"time":","time":2023-09-22T23:08:02.538727538Z"","hostname2023-09-22T23:08:02.538729734Z"":"test live_repair::repair_test::test_live_repair_deps_repair_wspan_left ... ,"ip-10-150-1-74.us-west-2.compute.internal"hostname,"":pid""okip-10-150-1-74.us-west-2.compute.internal:"4291,
}"
pid",:{4291"}"
msg"time":{:"""msg":"2023-09-22T23:08:02.5387362Z","[2] Transition from New to WaitActive"hostname",:""v[1] e25c51fe-6bce-4500-9128-4f204e1a9445 (04223c12-1924-4b5a-ac33-ec762a7ea271) Active Faulted Active ds_transition to LiveRepairReady"":,0",v""name:"0:,""ip-10-150-1-74.us-west-2.compute.internalcruciblename"",:"""cruciblelevel"",:,30""levelpid""::304291}
,,""timetime""::""2023-09-22T23:08:02.538783882Z2023-09-22T23:08:02.538787433Z""{,,""hostnamehostname""::"""msg"ip-10-150-1-74.us-west-2.compute.internalip-10-150-1-74.us-west-2.compute.internal:"",,"""pidpid""::42914291}}

{Crucible 4b1de70b-0c52-4541-acb2-5a791e8f874c has session id: d97d444e-ecec-4a18-add5-6352a11d2556","{msg"":""vmsg"":[1] Transition from Faulted to LiveRepairReady"":,"0v":,0","name":"[2] af820581-d317-46fc-9b93-01ba46c1da49 (4bd2fdd5-afb2-479f-8091-75ec93e8aa7a) Active Active WaitActive ds_transition to WaitQuorumname"crucible,"":v"":"0crucible",,"",level"name:"30:""level":crucible"30,"level":30,"time":"2023-09-22T23:08:02.538835163Z","hostname":","time":"ip-10-150-1-74.us-west-2.compute.internal","2023-09-22T23:08:02.538839343Z"pid",:"4291hostname":}"
ip-10-150-1-74.us-west-2.compute.internal",",pid{":""4291msg"}:
"time":"{2023-09-22T23:08:02.538838308Z""msg",:""[1] e25c51fe-6bce-4500-9128-4f204e1a9445 (04223c12-1924-4b5a-ac33-ec762a7ea271) Active LiveRepairReady Active ds_transition to LiveRepair"hostname",:"[2] Transition from WaitActive to WaitQuorum""v",:"0v",:"0ip-10-150-1-74.us-west-2.compute.internalname,"":"name"crucible:"""crucible,"",,""levellevel""::3030pid":4291}
,,""timetime""::""2023-09-22T23:08:02.538881836Z2023-09-22T23:08:02.538882064Z""{,,""hostnamehostname""::"""ip-10-150-1-74.us-west-2.compute.internalip-10-150-1-74.us-west-2.compute.internal"",,""msg"pidpid""::42914291:}}

{"{"msg":""msg":"[1] Transition from LiveRepairReady to LiveRepair"[0] 4b1de70b-0c52-4541-acb2-5a791e8f874c (b63c9270-2f34-4a0f-b0a8-337f9b415156) New New New ds_transition to WaitActive","v",:"[2] af820581-d317-46fc-9b93-01ba46c1da49 (4bd2fdd5-afb2-479f-8091-75ec93e8aa7a) Active Active WaitQuorum ds_transition to Active0",v","":vname":00":,"",name""crucible:"",name"crucible""level,:""level:"":3030crucible","level":30,,""timetime""::""2023-09-22T23:08:02.538938621Z2023-09-22T23:08:02.538938813Z"",,""hostnamehostname""::""ip-10-150-1-74.us-west-2.compute.internalip-10-150-1-74.us-west-2.compute.internal"",,""pidpid""::42914291,"}}

{time":""msg":"2023-09-22T23:08:02.538941965Z","[2] Transition from WaitQuorum to Active"hostname,"":v"":0,"name":"ip-10-150-1-74.us-west-2.compute.internal"crucible",,""levelpid":"30:4291}
{,"time"":"msg":"2023-09-22T23:08:02.538977679Z","[0] Transition from New to WaitActive"hostname",":v"":0,"ip-10-150-1-74.us-west-2.compute.internal"name,"":pid"":crucible"4291,"level}"
:30{"msg":"af820581-d317-46fc-9b93-01ba46c1da49 is now active with session: 4bd2fdd5-afb2-479f-8091-75ec93e8aa7a","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.539011136Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
,"time":"2023-09-22T23:08:02.539024964Z"{,"hostname":""msg":ip-10-150-1-74.us-west-2.compute.internal"","pid":4291}
[0] 4b1de70b-0c52-4541-acb2-5a791e8f874c (b63c9270-2f34-4a0f-b0a8-337f9b415156) WaitActive New New ds_transition to WaitQuorum","v":{0,"name":""msg":"crucible","level":30[1] af820581-d317-46fc-9b93-01ba46c1da49 (4bd2fdd5-afb2-479f-8091-75ec93e8aa7a) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.539062031Z","hostname":","time":"ip-10-150-1-74.us-west-2.compute.internal","2023-09-22T23:08:02.539070084Z"pid",:"4291hostname":"}
ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{{"msg":""msg":"[0] Transition from WaitActive to WaitQuorum","v"[1] Transition from Active to Faulted":,"0v",":name"0:,""crucible"name":","crucible"level",:"30level":30,"time":","time":"2023-09-22T23:08:02.539117656Z","2023-09-22T23:08:02.539118659Z"hostname":,""hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid"ip-10-150-1-74.us-west-2.compute.internal":,"4291pid":4291}
}
{"{msg":""msg":"[0] 4b1de70b-0c52-4541-acb2-5a791e8f874c (b63c9270-2f34-4a0f-b0a8-337f9b415156) WaitQuorum New New ds_transition to Active","v":0,[1] af820581-d317-46fc-9b93-01ba46c1da49 (4bd2fdd5-afb2-479f-8091-75ec93e8aa7a) Active Faulted Active ds_transition to LiveRepairReady"","name":"v"crucible":,0","level"name"::30"crucible","level":30,"time":"2023-09-22T23:08:02.539174922Z",","time":"hostname":"2023-09-22T23:08:02.539177853Z","ip-10-150-1-74.us-west-2.compute.internal"hostname,""pid"::4291"}
ip-10-150-1-74.us-west-2.compute.internal","pid":4291}{
"msg":{"{[0] Transition from WaitQuorum to Active","v"":msg"":0"msg,"":name"[1] Transition from Faulted to LiveRepairReady":"",crucible""v,"test live_repair::repair_test::test_live_repair_deps_repair_wspan_right ... :0ok,""Crucible stats registered with UUID: b4ddac83-8f02-4759-90d0-81ecbf434d0d"level"name
",:"30v"::"0crucible",","name"level"::"30crucible","level":30,"time":"2023-09-22T23:08:02.539247304Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal",","pid":time4291":"}
2023-09-22T23:08:02.539255365Z","hostname":"{,"ip-10-150-1-74.us-west-2.compute.internal"time":,"""pid":msg42912023-09-22T23:08:02.539258682Z""}:
,""hostname":"{ip-10-150-1-74.us-west-2.compute.internal",""pid"[1] 4b1de70b-0c52-4541-acb2-5a791e8f874c (b63c9270-2f34-4a0f-b0a8-337f9b415156) Active New New ds_transition to WaitActive:msg"":",4291"v":0},"
[1] af820581-d317-46fc-9b93-01ba46c1da49 (4bd2fdd5-afb2-479f-8091-75ec93e8aa7a) Active LiveRepairReady Active ds_transition to LiveRepair"name":","cruciblev"":{0,,""name":"levelcrucible""",:msg"30:""level":30Crucible b4ddac83-8f02-4759-90d0-81ecbf434d0d has session id: b1858b94-2183-4402-9eab-f3f8e6e8484a","v":0,"name":"crucible","level":30,"time":","2023-09-22T23:08:02.539322395Z"time":","hostname":"2023-09-22T23:08:02.539324592Z","hostnameip-10-150-1-74.us-west-2.compute.internal"",":pid"":4291},
ip-10-150-1-74.us-west-2.compute.internal"","time"pid":{":42912023-09-22T23:08:02.539333557Z""}msg"
,":"hostname":"[1] Transition from New to WaitActive{"ip-10-150-1-74.us-west-2.compute.internal",",v"":"pid"msg":04291:","}name"
[1] Transition from LiveRepairReady to LiveRepair":",crucible""v",:{"0level",":nameJob is DownstairsIO { ds_id: JobId(1002), guest_id: 1, work: ExtentFlushClose { dependencies: [JobId(1000), JobId(1001)], extent: 1, flush_number: 2, gen_number: 4, source_downstairs: ClientId(1), repair_downstairs: [ClientId(0), ClientId(2)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 1, block: 0 }, ImpactedAddr { extent_id: 1, block: 2 }) }
"msg":"[0] b4ddac83-8f02-4759-90d0-81ecbf434d0d (33b3cc45-a768-4d92-ae35-28fdcf2e3cdf) New New New ds_transition to WaitActive","v":0,"name":"crucible","level""::"crucible"30,"level":3030,"time":","time":2023-09-22T23:08:02.539540937Z"",2023-09-22T23:08:02.539542967Z""hostname",:""hostname":"ip-10-150-1-74.us-west-2.compute.internal"ip-10-150-1-74.us-west-2.compute.internal",,",""pidpid"time""::4291":4291}2023-09-22T23:08:02.53954914Z"}
,
{"hostname":"ip-10-150-1-74.us-west-2.compute.internal"","msg"pid"::"4291[0] Transition from New to WaitActive"}
,"v":0,"name":"crucible","level"{:30"msg":"[1] 4b1de70b-0c52-4541-acb2-5a791e8f874c (b63c9270-2f34-4a0f-b0a8-337f9b415156) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.53961479Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
,"time":"{2023-09-22T23:08:02.539631253Z","hostname"":"msg":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
[0] b4ddac83-8f02-4759-90d0-81ecbf434d0d (33b3cc45-a768-4d92-ae35-28fdcf2e3cdf) WaitActive New New ds_transition to WaitQuorum","v":0,{"name":"crucible"","msg":"level":30[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30{,"time":"2023-09-22T23:08:02.539669188Z",""hostnamemsg""::,"""time":"ip-10-150-1-74.us-west-2.compute.internal","pid"2023-09-22T23:08:02.53967583Z":Crucible stats registered with UUID: ba5b2c47-00db-4ad1-a76f-e40e8bf42e3c"4291,",hostname""}v"
::"0,"{nameip-10-150-1-74.us-west-2.compute.internal"","":pid"":msg4291crucible"":",}"
[0] Transition from WaitActive to WaitQuorum"level":,"30v":0{,"name":"crucible",""level"msg"::"30[1] 4b1de70b-0c52-4541-acb2-5a791e8f874c (b63c9270-2f34-4a0f-b0a8-337f9b415156) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time",:""time":"2023-09-22T23:08:02.539724472Z"2023-09-22T23:08:02.539731055Z",",hostname":""hostname":"ip-10-150-1-74.us-west-2.compute.internal","ip-10-150-1-74.us-west-2.compute.internal"pid,"":pid"4291:,4291}"
time"}:
{"2023-09-22T23:08:02.539742487Z","{"hostname":msg""":"msg"ip-10-150-1-74.us-west-2.compute.internal":,""pid":4291Crucible ba5b2c47-00db-4ad1-a76f-e40e8bf42e3c has session id: 62d9130a-bdf5-43ee-992c-7a4ed3ffed51"}
,"v":0,"name":"{crucible","level":"30msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"[0] b4ddac83-8f02-4759-90d0-81ecbf434d0d (33b3cc45-a768-4d92-ae35-28fdcf2e3cdf) WaitQuorum New New ds_transition to Active"time":","v"2023-09-22T23:08:02.539796062Z:"0,,""hostname"name",:"":time"":crucible""ip-10-150-1-74.us-west-2.compute.internal",2023-09-22T23:08:02.539804233Z"",,""test live_repair::repair_test::test_live_repair_deps_super_spanner ... pidhostname"level":ok"
::"304291}ip-10-150-1-74.us-west-2.compute.internal
","pid":4291{}
"msg":",{"time":"[0] ba5b2c47-00db-4ad1-a76f-e40e8bf42e3c (eb4c23a9-6f3c-467c-9a21-2dac1fc4c32a) New New New ds_transition to WaitActive""2023-09-22T23:08:02.539848645Z"msg",,":v""":hostname":0","name":"ip-10-150-1-74.us-west-2.compute.internal[2] 4b1de70b-0c52-4541-acb2-5a791e8f874c (b63c9270-2f34-4a0f-b0a8-337f9b415156) Active Active New ds_transition to WaitActive""crucible,"",v",:""0level,""pidname:"30"::"4291crucible","}level"
,Job is DownstairsIO { ds_id: JobId(1002), guest_id: 1, work: ExtentLiveReopen { dependencies: [JobId(1000), JobId(1001)], extent: 1 }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 1, block: 0 }, ImpactedAddr { extent_id: 1, block: 2 }) }
{":30msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.540030065Z","hostname":""ip-10-150-1-74.us-west-2.compute.internal"time",":pid"":42912023-09-22T23:08:02.53989306Z"}
,"hostname",:""time":"{ip-10-150-1-74.us-west-2.compute.internal"2023-09-22T23:08:02.540050217Z","pid",":"hostname"msg"4291::""}
[2] Transition from New to WaitActive"ip-10-150-1-74.us-west-2.compute.internal",,""v"pid:{"0:,4291"name":"}"cruciblemsg""
:,""level":30[0] Transition from New to WaitActive",{"v":0,"name":""msg"crucible":","level":30[1] b4ddac83-8f02-4759-90d0-81ecbf434d0d (33b3cc45-a768-4d92-ae35-28fdcf2e3cdf) Active New New ds_transition to WaitActive","v":0,"name":"crucible",,""timelevel""::"302023-09-22T23:08:02.540105266Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal",",pid"":time"4291:"}
2023-09-22T23:08:02.540114405Z","hostname,"":{"time":"ip-10-150-1-74.us-west-2.compute.internal"2023-09-22T23:08:02.54012654Z""msg,,""":hostname"":pid"":4291ip-10-150-1-74.us-west-2.compute.internal"},
[2] 4b1de70b-0c52-4541-acb2-5a791e8f874c (b63c9270-2f34-4a0f-b0a8-337f9b415156) Active Active WaitActive ds_transition to WaitQuorum""pid,"":{v":42910,""name"msg}:"":
crucible"","level":30{[0] ba5b2c47-00db-4ad1-a76f-e40e8bf42e3c (eb4c23a9-6f3c-467c-9a21-2dac1fc4c32a) WaitActive New New ds_transition to WaitQuorum"","msg"v"::"0,"name":"[1] Transition from New to WaitActive"crucible",,""v"level"::030,"name":",crucible"","time"level"::"302023-09-22T23:08:02.540182806Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
,"time":"2023-09-22T23:08:02.540198043Z"{,","hostname":"time":""ip-10-150-1-74.us-west-2.compute.internal"msg2023-09-22T23:08:02.540204777Z,""":pid,"":"4291hostname":"}[2] Transition from WaitActive to WaitQuorum
{ip-10-150-1-74.us-west-2.compute.internal"",",pid{""v:"":4291"0msg},
msg"""::""{name":"crucible"[0] Transition from WaitActive to WaitQuorum"Crucible stats registered with UUID: d9cc3da9-eb70-4040-97da-092ce8ba4fe2,"""msglevel""::"30,,""v"v"::0[1] b4ddac83-8f02-4759-90d0-81ecbf434d0d (33b3cc45-a768-4d92-ae35-28fdcf2e3cdf) Active WaitActive New ds_transition to WaitQuorum",0",,name"""name:"v":"crucible":,",0""level",:"time"name"::crucible"""302023-09-22T23:08:02.540268619Z",crucible"",,level""":levelhostname"":":3030ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
,"time":"2023-09-22T23:08:02.540297779Z"{,"hostname":""ip-10-150-1-74.us-west-2.compute.internal"msg,"":",pid"":time4291",:""[2] 4b1de70b-0c52-4541-acb2-5a791e8f874c (b63c9270-2f34-4a0f-b0a8-337f9b415156) Active Active WaitQuorum ds_transition to Active"time"2023-09-22T23:08:02.540307616Z"}:,"
,2023-09-22T23:08:02.540307424Z"""v"hostname{":,:0""msg,hostname""":"name"::""ip-10-150-1-74.us-west-2.compute.internal""ip-10-150-1-74.us-west-2.compute.internalcrucible,"""pid",:,"[0] ba5b2c47-00db-4ad1-a76f-e40e8bf42e3c (eb4c23a9-6f3c-467c-9a21-2dac1fc4c32a) WaitQuorum New New ds_transition to Active""pid"level,4291:"4291}"
:v"30:}0
,{"name":"crucible","{level"":msg"30:""msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"Crucible d9cc3da9-eb70-4040-97da-092ce8ba4fe2 has session id: de9105d0-8769-44d7-ac42-838041e175c8",name,"""v"time:"::""0crucible",,""time"level":",2023-09-22T23:08:02.540381102Z":"2023-09-22T23:08:02.540393228Z"30,,name"""Job is DownstairsIO { ds_id: JobId(1003), guest_id: 1, work: ExtentLiveNoOp { dependencies: [JobId(1001), JobId(1002)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 1, block: 0 }, ImpactedAddr { extent_id: 1, block: 2 }) }
,":time":"2023-09-22T23:08:02.540417324Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal",""pid":crucible"4291,"}level":
30{"msg":"hostname[1] b4ddac83-8f02-4759-90d0-81ecbf434d0d (33b3cc45-a768-4d92-ae35-28fdcf2e3cdf) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":,"30time":"2023-09-22T23:08:02.5405682Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":","[0] d9cc3da9-eb70-4040-97da-092ce8ba4fe2 (86fd9d03-bd19-4d98-b8bb-45992b454d2b) New New New ds_transition to WaitActive"time":,""v":02023-09-22T23:08:02.540585243Z",":"","name":hostname""ip-10-150-1-74.us-west-2.compute.internal:"crucible"",","level"pidip-10-150-1-74.us-west-2.compute.internal"",:":pid"30:42914291}}

,"{time":"2023-09-22T23:08:02.540623789Z","hostname"":"msg":"{ip-10-150-1-74.us-west-2.compute.internal","pid":4291[1] Transition from WaitQuorum to Active"}
,""{msg":""msg":"[0] Transition from WaitQuorum to Active"[0] Transition from New to WaitActive"v,,"":v"0:","v"0name:,"0name",:""":"namecrucible"",":level":crucible""30,crucible""level":,"30level":30,"time":"2023-09-22T23:08:02.540675359Z",,""time"hostname"::""2023-09-22T23:08:02.540676926Z"ip-10-150-1-74.us-west-2.compute.internal,""time":",,""2023-09-22T23:08:02.540692776Z"pidhostname""hostname,":hostname":""":ip-10-150-1-74.us-west-2.compute.internal4291"ip-10-150-1-74.us-west-2.compute.internal"},
:""pid":,4291"{pidip-10-150-1-74.us-west-2.compute.internal}"
"":msg4291,"{":pid}"""
msg[2] b4ddac83-8f02-4759-90d0-81ecbf434d0d (33b3cc45-a768-4d92-ae35-28fdcf2e3cdf) Active Active New ds_transition to WaitActive:"":"4291,"{v":}0
[0] d9cc3da9-eb70-4040-97da-092ce8ba4fe2 (86fd9d03-bd19-4d98-b8bb-45992b454d2b) WaitActive New New ds_transition to WaitQuorum,""","msg"v:"name":"{:0","name":"crucible""crucible",msg,"[1] ba5b2c47-00db-4ad1-a76f-e40e8bf42e3c (eb4c23a9-6f3c-467c-9a21-2dac1fc4c32a) Active New New ds_transition to WaitActive"levellevel""::"30:""30,"v":[2] Transition from WaitQuorum to Active"0,","name"v"::"0,"crucible"time":",,"2023-09-22T23:08:02.540781246Zlevel""",":hostname30name""::""crucible"ip-10-150-1-74.us-west-2.compute.internal",",pid":"4291level}"
:30{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.540798201Z",,""hostname":time"",":time":""ip-10-150-1-74.us-west-2.compute.internal"2023-09-22T23:08:02.540813576Z",,2023-09-22T23:08:02.54078014Z"""pid",hostname,""":"time"hostname":::""ip-10-150-1-74.us-west-2.compute.internal"4291,ip-10-150-1-74.us-west-2.compute.internal"",pid":}"2023-09-22T23:08:02.540808819Zpid4291
}""
,:{"4291{hostname":"}"
ip-10-150-1-74.us-west-2.compute.internal"","msg":"pidmsg""::[0] d9cc3da9-eb70-4040-97da-092ce8ba4fe2 (86fd9d03-bd19-4d98-b8bb-45992b454d2b) WaitQuorum New New ds_transition to Active"","{v":4291[1] Transition from New to WaitActive"0",}"msg
,"v""::name"0{,"":name"[2] Transition from New to WaitActivecrucible""":msg":"",""cruciblev","":4b1de70b-0c52-4541-acb2-5a791e8f874c is now active with session: b63c9270-2f34-4a0f-b0a8-337f9b4151560level"",:,"30,name"""level"v":::,""30time0crucible"":",,""2023-09-22T23:08:02.540909272Z"level"name,:""30hostname":":"crucible"ip-10-150-1-74.us-west-2.compute.internal",,""pid":level"4291:}
,30{""timemsg":"":"[0] Transition from WaitQuorum to Active","v":2023-09-22T23:08:02.540918208Z"0,"name,":""crucible",,"hostname"level":":30"time":"ip-10-150-1-74.us-west-2.compute.internal"2023-09-22T23:08:02.540927779Z","pid,"":,,hostname"""4291:time"}time"":":
2023-09-22T23:08:02.540950804Z""ip-10-150-1-74.us-west-2.compute.internal",",hostname2023-09-22T23:08:02.540937283Z{"":"",pidip-10-150-1-74.us-west-2.compute.internal""",msg""hostnamepid":":4291":4291}"}:
[1] ba5b2c47-00db-4ad1-a76f-e40e8bf42e3c (eb4c23a9-6f3c-467c-9a21-2dac1fc4c32a) Active WaitActive New ds_transition to WaitQuorum""ip-10-150-1-74.us-west-2.compute.internal","pid"{:4291}
"msg":"{"[2] b4ddac83-8f02-4759-90d0-81ecbf434d0d (33b3cc45-a768-4d92-ae35-28fdcf2e3cdf) Active Active WaitActive ds_transition to WaitQuorummsg"":","v":0,"name":"crucible",[1] 4b1de70b-0c52-4541-acb2-5a791e8f874c (b63c9270-2f34-4a0f-b0a8-337f9b415156) Active Active Active ds_transition to Faulted"",level""v"::300,"name":"crucible",",level"":v"30:0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.541035309Z","hostname":","ip-10-150-1-74.us-west-2.compute.internal"time",:""pid":4291,"}time
2023-09-22T23:08:02.541041939Z"
"{:,""{"hostname2023-09-22T23:08:02.541047554Zmsg""":msg"":",""[2] Transition from WaitActive to WaitQuorum":hostname[1] d9cc3da9-eb70-4040-97da-092ce8ba4fe2 (86fd9d03-bd19-4d98-b8bb-45992b454d2b) Active New New ds_transition to WaitActive"",",v"ip-10-150-1-74.us-west-2.compute.internal":v":"",:"0pid,":"0name4291,""name:"}""crucible"
,:ip-10-150-1-74.us-west-2.compute.internal"""level":crucible",30{,""level"pid"::"30msg":4291"}
,"[1] Transition from Active to Faulted"time":","v"2023-09-22T23:08:02.541118677Z{":,"0hostname":",""name"msg":,:"""crucible"time"[1] Transition from WaitActive to WaitQuorum,":ip-10-150-1-74.us-west-2.compute.internal""level",":",2023-09-22T23:08:02.541123578Z""pid",:30v":"42910hostname",:}"
name""{:""crucible"msg",ip-10-150-1-74.us-west-2.compute.internal"":",level"":[1] Transition from New to WaitActive"pid",,"v":30"0:time4291",":name":"}crucible"
","level":302023-09-22T23:08:02.541156128Z","{hostname":","ip-10-150-1-74.us-west-2.compute.internal"",time",""msgtime":"":":pid"2023-09-22T23:08:02.541178299Z""2023-09-22T23:08:02.541190565Z",:,4291""hostname"}:"
[2] b4ddac83-8f02-4759-90d0-81ecbf434d0d (33b3cc45-a768-4d92-ae35-28fdcf2e3cdf) Active Active WaitQuorum ds_transition to Active"ip-10-150-1-74.us-west-2.compute.internal"hostname,,""pid":":{v""4291:}
ip-10-150-1-74.us-west-2.compute.internal"{"msg,""0"pid"msg":":,"":name4291":[1] d9cc3da9-eb70-4040-97da-092ce8ba4fe2 (86fd9d03-bd19-4d98-b8bb-45992b454d2b) Active WaitActive New ds_transition to WaitQuorum}"
"[1] 4b1de70b-0c52-4541-acb2-5a791e8f874c (b63c9270-2f34-4a0f-b0a8-337f9b415156) Active Faulted Active ds_transition to LiveRepairReady","crucible,{""v":,v""0msg":,"":level"0name",":name"30:":""crucible"crucible",","level":level"30:30[1] ba5b2c47-00db-4ad1-a76f-e40e8bf42e3c (eb4c23a9-6f3c-467c-9a21-2dac1fc4c32a) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible",,""time":"level":2023-09-22T23:08:02.541271229Z30","hostname":","ip-10-150-1-74.us-west-2.compute.internal"time":,""pid":4291}
,2023-09-22T23:08:02.541265558Z""{time",:""msg":"",2023-09-22T23:08:02.541272628Z"[1] Transition from WaitActive to WaitQuorum"hostname":,time""":"hostname"ip-10-150-1-74.us-west-2.compute.internal,"",:"""v":pid"0:,"4291ip-10-150-1-74.us-west-2.compute.internal"2023-09-22T23:08:02.541282086Z,name}":""crucible"
"pid",":,level":4291{30"}hostname"
:",""msg"time":{:""ip-10-150-1-74.us-west-2.compute.internal"2023-09-22T23:08:02.541333721Z"","msg,"hostnamepid[2] Transition from WaitQuorum to Active":"""":,ip-10-150-1-74.us-west-2.compute.internal":,""4291pid":"}v
4291[1] Transition from Faulted to LiveRepairReady""},
{"{v"::0",0",msg":"""namename""msg:[1] d9cc3da9-eb70-4040-97da-092ce8ba4fe2 (86fd9d03-bd19-4d98-b8bb-45992b454d2b) Active WaitQuorum New ds_transition to Active""crucible"":,"":crucible"",v":[1] Transition from WaitQuorum to Active"","level,""0:,level30v":"":name":"0crucible",30,""level":name"30:"crucible","level":30,,""timetime":"":"2023-09-22T23:08:02.541411043Z","hostname"2023-09-22T23:08:02.541401648Z":",,""ip-10-150-1-74.us-west-2.compute.internal"hostname",":time"pid":"4291ip-10-150-1-74.us-west-2.compute.internal"}
,,{""pid":"msg"":":time2023-09-22T23:08:02.541408641Z4291[1] Transition from WaitQuorum to Active"""},:"
,v"":"02023-09-22T23:08:02.541414725Z",hostname","{:name":""""crucible"msgip-10-150-1-74.us-west-2.compute.internal",,"":"hostnamelevel"pid":"30"::"4291}ip-10-150-1-74.us-west-2.compute.internal"
,"[1] 4b1de70b-0c52-4541-acb2-5a791e8f874c (b63c9270-2f34-4a0f-b0a8-337f9b415156) Active LiveRepairReady Active ds_transition to LiveRepair"time":,,"""2023-09-22T23:08:02.541477907Z"{,"pidhostnamev"":"":":msg"ip-10-150-1-74.us-west-2.compute.internal"4291,"0:pid",":"}4291name}b4ddac83-8f02-4759-90d0-81ecbf434d0d is now active with session: 33b3cc45-a768-4d92-ae35-28fdcf2e3cdf"
"
:,"{"crucible"{v"","msg""::levelmsg"0":,"""name":[2] d9cc3da9-eb70-4040-97da-092ce8ba4fe2 (86fd9d03-bd19-4d98-b8bb-45992b454d2b) Active Active New ds_transition to WaitActive30":,""[2] ba5b2c47-00db-4ad1-a76f-e40e8bf42e3c (eb4c23a9-6f3c-467c-9a21-2dac1fc4c32a) Active Active New ds_transition to WaitActive"v":,0",v"crucible"":,name":"0"crucible",,"level""name"level":30,:":"time"crucible":30,"",level""2023-09-22T23:08:02.541541346Z"time":",:"2023-09-22T23:08:02.541559004Z"30,"hostname"hostname":":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291ip-10-150-1-74.us-west-2.compute.internal"}
,"pid{":"4291msg":",}"
[2] Transition from New to WaitActive"time",":v"",{"2023-09-22T23:08:02.541564398Z"time"::0"",msg",2023-09-22T23:08:02.541572171Z"""namehostname:",":"""hostname":":crucible""ip-10-150-1-74.us-west-2.compute.internal"[1] Transition from LiveRepairReady to LiveRepair,ip-10-150-1-74.us-west-2.compute.internal,"""level,""":pid"v"30:,:42910",pid"":name"4291,":}time"}
":"crucible"
{2023-09-22T23:08:02.541622041Z",","level"hostname":"{:"ip-10-150-1-74.us-west-2.compute.internal"msg"30:",""pid":msg"4291[2] Transition from New to WaitActive"}
:","{v":"msg":"0,"name":"[1] b4ddac83-8f02-4759-90d0-81ecbf434d0d (33b3cc45-a768-4d92-ae35-28fdcf2e3cdf) Active Active Active ds_transition to Faulted"crucible"[2] d9cc3da9-eb70-4040-97da-092ce8ba4fe2 (86fd9d03-bd19-4d98-b8bb-45992b454d2b) Active Active WaitActive ds_transition to WaitQuorum",,,""v",""time"v":level0:":,"0name":"":,"2023-09-22T23:08:02.541649886Z"30crucible",name"","hostname"::"level":"30ip-10-150-1-74.us-west-2.compute.internal"crucible",",pid"":level"4291:30},"
,time":""time"2023-09-22T23:08:02.541689102Z":,""hostname":"2023-09-22T23:08:02.541682611Z"ip-10-150-1-74.us-west-2.compute.internal","pid",:"4291hostname"}:,
""{time":""msg":"ip-10-150-1-74.us-west-2.compute.internal"2023-09-22T23:08:02.541695034Z",[2] Transition from WaitActive to WaitQuorum"",,pid"""hostname":v4291"::0","}name":ip-10-150-1-74.us-west-2.compute.internal"
,""crucible"pid",":level":429130{}
","msg"time":":{"2023-09-22T23:08:02.54174225Z","hostname":""ip-10-150-1-74.us-west-2.compute.internal","msg"pid"::[2] ba5b2c47-00db-4ad1-a76f-e40e8bf42e3c (eb4c23a9-6f3c-467c-9a21-2dac1fc4c32a) Active Active WaitActive ds_transition to WaitQuorum"4291"}
,[1] Transition from Active to Faulted""{v,""msg"::""0v",:"0name",[2] d9cc3da9-eb70-4040-97da-092ce8ba4fe2 (86fd9d03-bd19-4d98-b8bb-45992b454d2b) Active Active WaitQuorum ds_transition to Active":""name":",crucible"crucible"",v":"0level,,"":level"30name":"crucible","level":30,"time":,""time":"2023-09-22T23:08:02.541789704Z","2023-09-22T23:08:02.54178692Z"hostname":","hostname":"":ip-10-150-1-74.us-west-2.compute.internal"30,"pid":ip-10-150-1-74.us-west-2.compute.internal"4291,"pid":}4291
}
{"msg"{:","[2] Transition from WaitQuorum to Activetime"":"msg",":""2023-09-22T23:08:02.541812753Z"v":,0",[2] Transition from WaitActive to WaitQuorum""hostnamename,"":":v""":crucible"0,,ip-10-150-1-74.us-west-2.compute.internal"",""levelpid""::name"429130:"}crucible"
,"level":30{"msg":","time":"2023-09-22T23:08:02.541859286Z"[1] b4ddac83-8f02-4759-90d0-81ecbf434d0d (33b3cc45-a768-4d92-ae35-28fdcf2e3cdf) Active Faulted Active ds_transition to LiveRepairReady",",hostname":""v":,"0time,"ip-10-150-1-74.us-west-2.compute.internal":""name",:2023-09-22T23:08:02.541867059Z""",crucible""pid,"hostname""::"4291level":30}ip-10-150-1-74.us-west-2.compute.internal"
,"pid":4291}
{"msg{":",""time"msg"::"d9cc3da9-eb70-4040-97da-092ce8ba4fe2 is now active with session: 86fd9d03-bd19-4d98-b8bb-45992b454d2b"","v":2023-09-22T23:08:02.541899519Z"0,",name"":[2] ba5b2c47-00db-4ad1-a76f-e40e8bf42e3c (eb4c23a9-6f3c-467c-9a21-2dac1fc4c32a) Active Active WaitQuorum ds_transition to Active""hostname",:""crucible"v":ip-10-150-1-74.us-west-2.compute.internal,"0",,level"""name":pid":30:"4291crucible","}level
,{"time":"2023-09-22T23:08:02.541942077Z""msg,"":hostname"":"test live_repair::repair_test::test_live_repair_deps_writes ... [1] Transition from Faulted to LiveRepairReady"okip-10-150-1-74.us-west-2.compute.internal",
",v"":pid0,"name":"crucible","level":30":,"4291time":"}
2023-09-22T23:08:02.542001121Z","hostname":{""ip-10-150-1-74.us-west-2.compute.internal"msg":","pid":4291}
[1] d9cc3da9-eb70-4040-97da-092ce8ba4fe2 (86fd9d03-bd19-4d98-b8bb-45992b454d2b) Active Active Active ds_transition to Faulted",{"v":0,"name":"crucible"","msg":level"":30[1] b4ddac83-8f02-4759-90d0-81ecbf434d0d (33b3cc45-a768-4d92-ae35-28fdcf2e3cdf) Active LiveRepairReady Active ds_transition to LiveRepair","v":,0","timename""::""crucible""2023-09-22T23:08:02.54204875Z",:,""level"hostname30:"30:"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":,,"""time":crucible""time,""2023-09-22T23:08:02.54207645Z"level":,"":hostname"2023-09-22T23:08:02.542075759Z"30:","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":ip-10-150-1-74.us-west-2.compute.internal"4291,"pid":}4291
}
{,"time":""2023-09-22T23:08:02.542107023Z"msg":{","hostname":"[1] Transition from LiveRepairReady to LiveRepair""ip-10-150-1-74.us-west-2.compute.internal"msg",:","pid"":v4291[2] Transition from WaitQuorum to Active"":,}0",
"v"name"::{"0crucible",",name""msg""level"::":30"crucible","level":30[1] d9cc3da9-eb70-4040-97da-092ce8ba4fe2 (86fd9d03-bd19-4d98-b8bb-45992b454d2b) Active Faulted Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":",2023-09-22T23:08:02.542158972Z"","time"hostname"::""2023-09-22T23:08:02.542162035Z","ip-10-150-1-74.us-west-2.compute.internal"time",,:"""pid":hostname"4291:2023-09-22T23:08:02.542169525Z"}",
ip-10-150-1-74.us-west-2.compute.internal"","hostname"pid"::"4291}
ip-10-150-1-74.us-west-2.compute.internal","pid":4291{}
"msg":"{"msg":ba5b2c47-00db-4ad1-a76f-e40e8bf42e3c is now active with session: eb4c23a9-6f3c-467c-9a21-2dac1fc4c32a"","v":0[1] Transition from Faulted to LiveRepairReady",","name"v":":crucible0",","name":level"":crucible"30,"level":30,"time":",2023-09-22T23:08:02.542235198Z"",time":""hostname":"2023-09-22T23:08:02.542236891Z",ip-10-150-1-74.us-west-2.compute.internal"","hostnamepid""::4291"}
ip-10-150-1-74.us-west-2.compute.internal","pid":4291{}
"msg":"{"msg":"[1] ba5b2c47-00db-4ad1-a76f-e40e8bf42e3c (eb4c23a9-6f3c-467c-9a21-2dac1fc4c32a) Active Active Active ds_transition to Faulted","v":0,"name":"crucible",[1] d9cc3da9-eb70-4040-97da-092ce8ba4fe2 (86fd9d03-bd19-4d98-b8bb-45992b454d2b) Active LiveRepairReady Active ds_transition to LiveRepair"",level"":v"30:0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.542291725Z",,""hostnametime":"":"2023-09-22T23:08:02.54229634Z",ip-10-150-1-74.us-west-2.compute.internal""hostname",:""pid":4291}ip-10-150-1-74.us-west-2.compute.internal
","pid":4291}{
"msg":"{[1] Transition from Active to Faulted",""v":msg"0:","name":"crucible"[1] Transition from LiveRepairReady to LiveRepair",",level"":v"30:0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.542344284Z,""time,":""hostname":"2023-09-22T23:08:02.542348595Z","ip-10-150-1-74.us-west-2.compute.internalhostname"":,""pid":4291ip-10-150-1-74.us-west-2.compute.internal}"
,"pid":4291}
{"{msg":""msg":"No repair needed for extent 1","v":[1] ba5b2c47-00db-4ad1-a76f-e40e8bf42e3c (eb4c23a9-6f3c-467c-9a21-2dac1fc4c32a) Active Faulted Active ds_transition to LiveRepairReady"0,,""name"v":":crucible"0,","level"name:":30"crucible","level":30,"time":"2023-09-22T23:08:02.542404334Z",","hostname":"time":"{ip-10-150-1-74.us-west-2.compute.internal"2023-09-22T23:08:02.542407511Z",","pid":hostname4291"":",msg"":"ip-10-150-1-74.us-west-2.compute.internal"":,""Crucible stats registered with UUID: bf05128a-1b6d-44ee-a686-2a0f8c44fb12downstairspid""":}4291
,}"test live_repair::repair_test::test_live_repair_enqueue_close ... 
v":{ok0
","msg"name"::""crucible","[1] Transition from Faulted to LiveRepairReady"level":,"v"30:0,"name":"crucible","level":30,"time":",2023-09-22T23:08:02.542484651Z"",time":""hostname":"2023-09-22T23:08:02.542480088Z","ip-10-150-1-74.us-west-2.compute.internal"hostname",:""pid":4291ip-10-150-1-74.us-west-2.compute.internal"},
"pid":4291{}
"msg":"{[1] ba5b2c47-00db-4ad1-a76f-e40e8bf42e3c (eb4c23a9-6f3c-467c-9a21-2dac1fc4c32a) Active LiveRepairReady Active ds_transition to LiveRepair",""msg"v"::"0,"name":"crucible","level"Crucible bf05128a-1b6d-44ee-a686-2a0f8c44fb12 has session id: a7c67caa-b740-4a8a-abf1-20c5651d758c":,30"v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.542547226Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"time":}"
2023-09-22T23:08:02.542553586Z","hostname":"{ip-10-150-1-74.us-west-2.compute.internal","pid"":msg"4291:"}
[1] Transition from LiveRepairReady to LiveRepair","v":0,"{name":"crucible","level":"30msg":"[0] bf05128a-1b6d-44ee-a686-2a0f8c44fb12 (37037115-02dd-475d-a8b7-528101615ffc) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.542597408Z","hostname"Job is DownstairsIO { ds_id: JobId(1003), guest_id: 1, work: ExtentLiveRepair { dependencies: [JobId(1001), JobId(1002)], extent: 1, source_downstairs: ClientId(0), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(2)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 1, block: 0 }, ImpactedAddr { extent_id: 1, block: 2 }) }
test live_repair::repair_test::test_live_repair_enqueue_repair_noop ... ok
:",ip-10-150-1-74.us-west-2.compute.internal"",time""pid"::"42912023-09-22T23:08:02.542609635Z"}
,"hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.542739175Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] bf05128a-1b6d-44ee-a686-2a0f8c44fb12 (37037115-02dd-475d-a8b7-528101615ffc) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.542780152Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.542815987Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] bf05128a-1b6d-44ee-a686-2a0f8c44fb12 (37037115-02dd-475d-a8b7-528101615ffc) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.542857711Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.542896894Z","hostname"{:"ip-10-150-1-74.us-west-2.compute.internal","pid":4291"}
msg":"test live_repair::repair_test::test_live_repair_enqueue_reopen ... {ok"
msg"Crucible stats registered with UUID: 8d97f7f2-258d-40c3-ad7f-db2027cd44f9":","v":0,"name":"crucible"[1] bf05128a-1b6d-44ee-a686-2a0f8c44fb12 (37037115-02dd-475d-a8b7-528101615ffc) Active New New ds_transition to WaitActive",","level"v"::0,30"name":"crucible","level":30,"time":"2023-09-22T23:08:02.542949584Z",","time"hostname"::""2023-09-22T23:08:02.542946428Zip-10-150-1-74.us-west-2.compute.internal"","pid",:"4291hostname":"}
ip-10-150-1-74.us-west-2.compute.internal","pid":4291{}
"msg":"[1] Transition from New to WaitActive","{v":0,"name":"crucible"","msg"level"::"30Crucible 8d97f7f2-258d-40c3-ad7f-db2027cd44f9 has session id: 6a69c769-4563-43e7-8b68-5571fb482e62","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.543000771Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
,"time":"2023-09-22T23:08:02.543010952Z"{,"hostname":""msg":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
[1] bf05128a-1b6d-44ee-a686-2a0f8c44fb12 (37037115-02dd-475d-a8b7-528101615ffc) Active WaitActive New ds_transition to WaitQuorum","v":0,"name{":"crucible","level":"30msg":"[0] 8d97f7f2-258d-40c3-ad7f-db2027cd44f9 (32d96d38-3745-4ae0-96ff-5afe11fcae22) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.543055415Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
,"time":"2023-09-22T23:08:02.543067649Z"{,"hostname":""msg":"ip-10-150-1-74.us-west-2.compute.internal","pid":[1] Transition from WaitActive to WaitQuorum"4291,"v":}0
,"name":"crucible","level{":30"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.543103816Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
,"time":"2023-09-22T23:08:02.54311389Z"{,"hostname":""msg":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
[1] bf05128a-1b6d-44ee-a686-2a0f8c44fb12 (37037115-02dd-475d-a8b7-528101615ffc) Active WaitQuorum New ds_transition to Active","v":0,{"name":"crucible"","msg"level"::"30[0] 8d97f7f2-258d-40c3-ad7f-db2027cd44f9 (32d96d38-3745-4ae0-96ff-5afe11fcae22) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.543153982Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
,"{time":""msg{2023-09-22T23:08:02.543165393Z"msg":"","hostname":"Crucible stats registered with UUID: 35eecd9f-6208-42ff-b6c0-c4c02caf8b9b"ip-10-150-1-74.us-west-2.compute.internal,""v",":pid":04291,"name"}:"crucible","level":
30{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.543208652Z",""hostname"::,""[1] Transition from WaitQuorum to Active","vip-10-150-1-74.us-west-2.compute.internal":"0,,""pid"name:":4291"crucible",}"
"time":"{2023-09-22T23:08:02.543222229Z","hostname":""msg":"ip-10-150-1-74.us-west-2.compute.internal"level,"":pid"30:Crucible 35eecd9f-6208-42ff-b6c0-c4c02caf8b9b has session id: 12b715e8-407f-4d25-b63b-f62a28f963b8"4291,"v"}:
0,"name":"crucible",{"level":30"msg":","time":"[0] 8d97f7f2-258d-40c3-ad7f-db2027cd44f9 (32d96d38-3745-4ae0-96ff-5afe11fcae22) WaitQuorum New New ds_transition to Active"2023-09-22T23:08:02.543273873Z,""v":,"0hostname":,""name":"crucible"ip-10-150-1-74.us-west-2.compute.internal",,""level,"":time":"2023-09-22T23:08:02.543286687Z","30hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
pid{":4291}"
msg{":""msg":"[0] 35eecd9f-6208-42ff-b6c0-c4c02caf8b9b (63d034ab-cd7a-4ae2-9a0d-89f39e17085b) New New New ds_transition to WaitActive","v":{0,"[2] bf05128a-1b6d-44ee-a686-2a0f8c44fb12 (37037115-02dd-475d-a8b7-528101615ffc) Active Active New ds_transition to WaitActivename"":","crucible""v,"level":30":0,"name":"crucible",",level"":time"30:"2023-09-22T23:08:02.543309484Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal",",pid"":time"4291:"}
2023-09-22T23:08:02.543350477Z",","time":hostname"":{"2023-09-22T23:08:02.543359635Z",""ip-10-150-1-74.us-west-2.compute.internalhostname"":,"pid":4291}
msg":"Crucible stats registered with UUID: 85751bdd-8856-4e5b-8ef6-adad326659c5"{,"v":msg0",":name""msg":"[0] Transition from New to WaitActive",:""v":crucible"0,,"name":"crucible","level":"30level":30"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.543415482Z",","time"hostname"::""2023-09-22T23:08:02.543416734Z"ip-10-150-1-74.us-west-2.compute.internal",,""hostname",:""pid":ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
4291"}
time{"":"msg":"2023-09-22T23:08:02.543427481Z","hostname":"[0] 35eecd9f-6208-42ff-b6c0-c4c02caf8b9b (63d034ab-cd7a-4ae2-9a0d-89f39e17085b) WaitActive New New ds_transition to WaitQuorumip-10-150-1-74.us-west-2.compute.internal"ip-10-150-1-74.us-west-2.compute.internal"",,""pidv":0,"name":"crucible"":,"4291level":30}{
"msg":"{Crucible 85751bdd-8856-4e5b-8ef6-adad326659c5 has session id: 97d2e79d-a968-4554-ac16-de9c428b4ad8"",msg"":v"":,"0time",:[1] 8d97f7f2-258d-40c3-ad7f-db2027cd44f9 (32d96d38-3745-4ae0-96ff-5afe11fcae22) Active New New ds_transition to WaitActive","v":0,""name":"crucible"2023-09-22T23:08:02.543487877Z",",level"",:"30hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
,"time":"{2023-09-22T23:08:02.543522774Z","hostname""msg":":"[0] Transition from WaitActive to WaitQuorum","v":ip-10-150-1-74.us-west-2.compute.internal"0,",pid":4291}"
name"{:"crucible",""level":30msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible"pid,""level":":name30"4291:"crucible"},
,""level"time:":"30{2023-09-22T23:08:02.543567375Z",","hostname":time":"""msg"2023-09-22T23:08:02.543580425Z:ip-10-150-1-74.us-west-2.compute.internal""",,""hostnamepid":4291}
{[2] Transition from New to WaitActive","v":,0","nametime""::""crucible","2023-09-22T23:08:02.543594079Z"level":30,"hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{""msgmsg":"":","time":"2023-09-22T23:08:02.543638046Z"[0] 85751bdd-8856-4e5b-8ef6-adad326659c5 (8cb064bd-4131-4172-9681-8a036ecff523) New New New ds_transition to WaitActive"[0] 35eecd9f-6208-42ff-b6c0-c4c02caf8b9b (63d034ab-cd7a-4ae2-9a0d-89f39e17085b) WaitQuorum New New ds_transition to Active",,,""hostname":""ip-10-150-1-74.us-west-2.compute.internal:""v",:"0ip-10-150-1-74.us-west-2.compute.internalpid",":"4291,name""pid:"":crucible"4291,"level":}30
}
{,"time":"{2023-09-22T23:08:02.543688704Z""msg":",""hostname":"msg"":"ip-10-150-1-74.us-west-2.compute.internalv""[1] 8d97f7f2-258d-40c3-ad7f-db2027cd44f9 (32d96d38-3745-4ae0-96ff-5afe11fcae22) Active WaitActive New ds_transition to WaitQuorum:,"0,[2] bf05128a-1b6d-44ee-a686-2a0f8c44fb12 (37037115-02dd-475d-a8b7-528101615ffc) Active Active WaitActive ds_transition to WaitQuorum""name",:""v"crucible":,0","level"name"::"30crucible","level":30,"time":"pid":2023-09-22T23:08:02.543731132Z"4291,","time":}","hostname"v"::"0,"name":"ip-10-150-1-74.us-west-2.compute.internal"crucible",","pid"level"::304291}
{"msg":","[0] Transition from New to WaitActive
{""msg":"[0] Transition from WaitQuorum to Active","v":2023-09-22T23:08:02.543733978Z"0,"name":"crucible","timehostname""::"","level":302023-09-22T23:08:02.543769778Zip-10-150-1-74.us-west-2.compute.internal"",",pid"":hostname":"4291}ip-10-150-1-74.us-west-2.compute.internal"
,"pid":4291{}
"msg":"","{time":"[2] Transition from WaitActive to WaitQuorum,""2023-09-22T23:08:02.543893133Z"msg,",":"hostname":"[1] Transition from WaitActive to WaitQuorum"ip-10-150-1-74.us-west-2.compute.internal,""v",:"0pid",":name"4291:"crucible"},
{""msg"level"::""30v":0,"name":"crucible","level":[1] 35eecd9f-6208-42ff-b6c0-c4c02caf8b9b (63d034ab-cd7a-4ae2-9a0d-89f39e17085b) Active New New ds_transition to WaitActive"30,"v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.543946224Z",","hostname"time:"":"2023-09-22T23:08:02.543951616Z"ip-10-150-1-74.us-west-2.compute.internal",","hostnamepid""::"4291}
ip-10-150-1-74.us-west-2.compute.internal",,"{""msg":"v":0,"name":"crucible"[1] 8d97f7f2-258d-40c3-ad7f-db2027cd44f9 (32d96d38-3745-4ae0-96ff-5afe11fcae22) Active WaitQuorum New ds_transition to Active",","levelv":":0,"30name":"crucible","level":30,"time":"2023-09-22T23:08:02.543996016Z","hostname,""time:time"":"":ip-10-150-1-74.us-west-2.compute.internal"","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30"pid":4291,"time":"}
2023-09-22T23:08:02.544036364Z"2023-09-22T23:08:02.543957737Z","hostname":","{hostname":ip-10-150-1-74.us-west-2.compute.internal"",""msgpid""::"4291ip-10-150-1-74.us-west-2.compute.internal"},
"pid":2023-09-22T23:08:02.543993477Z[2] bf05128a-1b6d-44ee-a686-2a0f8c44fb12 (37037115-02dd-475d-a8b7-528101615ffc) Active Active WaitQuorum ds_transition to Active{"4291",""}vmsg""::"0,,""
{[2] 8d97f7f2-258d-40c3-ad7f-db2027cd44f9 (32d96d38-3745-4ae0-96ff-5afe11fcae22) Active Active New ds_transition to WaitActive","v":0,"name":""crucible"msg",":"level":30[1] Transition from New to WaitActive","v"name:"0:","name":cruciblehostname""",crucible:"ip-10-150-1-74.us-west-2.compute.internal",,""timepid":":4291"2023-09-22T23:08:02.544127547Z}"
,"hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":{4291"},
""msg":level"":{30"msg":"[0] 85751bdd-8856-4e5b-8ef6-adad326659c5 (8cb064bd-4131-4172-9681-8a036ecff523) WaitActive New New ds_transition to WaitQuorum"[2] Transition from New to WaitActive",,""v"v"::00,,""name"name"::""crucible"crucible",","level"level"::30,30"time":"2023-09-22T23:08:02.544174817Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid"":4291level,"":time"}30
,"time":"2023-09-22T23:08:02.544195456Z","hostname":{":,"""ip-10-150-1-74.us-west-2.compute.internal"time,msg""pid":4291}
{""msg":":"2023-09-22T23:08:02.544207251Z"[0] Transition from WaitActive to WaitQuorum",","hostname"v"::"0,"name":":ip-10-150-1-74.us-west-2.compute.internal"crucible","level":[1] 35eecd9f-6208-42ff-b6c0-c4c02caf8b9b (63d034ab-cd7a-4ae2-9a0d-89f39e17085b) Active WaitActive New ds_transition to WaitQuorum"30,"v":02023-09-22T23:08:02.544193935Z,"","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
,"{time":""2023-09-22T23:08:02.544264475Z"msg":","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291[2] 8d97f7f2-258d-40c3-ad7f-db2027cd44f9 (32d96d38-3745-4ae0-96ff-5afe11fcae22) Active Active WaitActive ds_transition to WaitQuorum"},
"namev""::0","crucible"name,"level":30","pid":4291}
{,"time":""msg":"2023-09-22T23:08:02.544305403Z","[2] Transition from WaitQuorum to Active"hostname":","v":0,"ip-10-150-1-74.us-west-2.compute.internal"name":"",:""crucible","level":30pid":4291}
crucible","{level":30","msg"time"::""2023-09-22T23:08:02.544337534Z"[1] Transition from WaitActive to WaitQuorum",","{v,"time":"2023-09-22T23:08:02.54435019Z""msg",":"hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291[0] 85751bdd-8856-4e5b-8ef6-adad326659c5 (8cb064bd-4131-4172-9681-8a036ecff523) WaitQuorum New New ds_transition to Active"}
,"v":0,{"name":""crucible"msg":,""level":30bf05128a-1b6d-44ee-a686-2a0f8c44fb12 is now active with session: 37037115-02dd-475d-a8b7-528101615ffc","v":0,"name":"crucible"",":level"0:30,"name":"crucible","level":30hostname":","time":ip-10-150-1-74.us-west-2.compute.internal,"""time,"":"pid":2023-09-22T23:08:02.54438621Z"42912023-09-22T23:08:02.544394773Z","},"
hostname"hostname":":"{,"ip-10-150-1-74.us-west-2.compute.internal"time"":msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
,"time":"2023-09-22T23:08:02.544432075Z",""hostname"{:2023-09-22T23:08:02.544399642Z"",""ip-10-150-1-74.us-west-2.compute.internal"msg,""pid"::"hostname[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,""time":":"2023-09-22T23:08:02.544468952Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","ip-10-150-1-74.us-west-2.compute.internal"pid":,4291"pid":}
4291{}
{4291"msg":"[1] 35eecd9f-6208-42ff-b6c0-c4c02caf8b9b (63d034ab-cd7a-4ae2-9a0d-89f39e17085b) Active WaitQuorum New ds_transition to Active",,""v":pid":04291,"name":"}
crucible","level":30{"msg":"}
[1] bf05128a-1b6d-44ee-a686-2a0f8c44fb12 (37037115-02dd-475d-a8b7-528101615ffc) Active Active Active ds_transition to Faulted"{,"v":0","msg"name:"":"crucible","level":30[2] 8d97f7f2-258d-40c3-ad7f-db2027cd44f9 (32d96d38-3745-4ae0-96ff-5afe11fcae22) Active Active WaitQuorum ds_transition to Active",,""v"":time"0msg,:""name":"crucible","level":30,"time":"2023-09-22T23:08:02.544606792Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
,"time":"{2023-09-22T23:08:02.544620596Z"","msg":hostname"":"[1] Transition from Active to Faulted"ip-10-150-1-74.us-west-2.compute.internal,"",v"":pid"0:,"4291name":"}
crucible","level"{:30"msg":"2023-09-22T23:08:02.544591187Z"[2] Transition from WaitQuorum to Active",",v":"0,hostname"":name","::"""time""crucibleip-10-150-1-74.us-west-2.compute.internal","pid":[1] 85751bdd-8856-4e5b-8ef6-adad326659c5 (8cb064bd-4131-4172-9681-8a036ecff523) Active New New ds_transition to WaitActive"4291},
","level":30,"{time":"2023-09-22T23:08:02.544695071Z",""hostname":msg"":"ip-10-150-1-74.us-west-2.compute.internal"[1] Transition from WaitQuorum to Active","pid"::,"4291"v":}2023-09-22T23:08:02.54464569Z0","name":""crucible","v"level"::030,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.54472645Z","hostname":",,""ip-10-150-1-74.us-west-2.compute.internal"time":"2023-09-22T23:08:02.544731603Z","hostname":","pid":ip-10-150-1-74.us-west-2.compute.internal4291
"},"pid":4291{}
"msg":"8d97f7f2-258d-40c3-ad7f-db2027cd44f9 is now active with session: 32d96d38-3745-4ae0-96ff-5afe11fcae22"{,"v":0,"name":"crucible",""msg"level"::30"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.544789323Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
,"time":"{2023-09-22T23:08:02.544797338Z"","msg":hostname"":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291[1] 8d97f7f2-258d-40c3-ad7f-db2027cd44f9 (32d96d38-3745-4ae0-96ff-5afe11fcae22) Active Active Active ds_transition to Faulted}"
,"v":0,"name":"{crucible","level":30"msg":"[1] 85751bdd-8856-4e5b-8ef6-adad326659c5 (8cb064bd-4131-4172-9681-8a036ecff523) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible",,""
{level":30hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":","time":"2023-09-22T23:08:02.544871349Z"","msg"hostname:[1] bf05128a-1b6d-44ee-a686-2a0f8c44fb12 (37037115-02dd-475d-a8b7-528101615ffc) Active Faulted Active ds_transition to LiveRepairReady"","v":0,"name":"crucible","level":30time":"2023-09-22T23:08:02.544836794Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
,"time":"{2023-09-22T23:08:02.544901131Z","hostname"":"msg":"ip-10-150-1-74.us-west-2.compute.internal"[1] Transition from Active to Faulted,""pid",:"4291v":0}
,[2] 35eecd9f-6208-42ff-b6c0-c4c02caf8b9b (63d034ab-cd7a-4ae2-9a0d-89f39e17085b) Active Active New ds_transition to WaitActive{"msg":"[1] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.544938452Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal"","pid":,4291}
"v":{0,""namemsg""::""crucible"","level"::"30[1] bf05128a-1b6d-44ee-a686-2a0f8c44fb12 (37037115-02dd-475d-a8b7-528101615ffc) Active LiveRepairReady Active ds_transition to LiveRepair","v":ip-10-150-1-74.us-west-2.compute.internal"0,","name"pid:"":crucible"4291,"level":30}
,{,""time":time""msg":"name":"crucible"[1] Transition from WaitActive to WaitQuorum","level":,30"v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.545002093Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
",":"time":{"2023-09-22T23:08:02.544973779Z""2023-09-22T23:08:02.545007872Zmsg",","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
"2023-09-22T23:08:02.544982589Z"{,"hostname":""msg":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
[1] 85751bdd-8856-4e5b-8ef6-adad326659c5 (8cb064bd-4131-4172-9681-8a036ecff523) Active WaitQuorum New ds_transition to Active","v":{0,""name"hostname"::"ip-10-150-1-74.us-west-2.compute.internal","pid":"4291"crucible":}",
[1] 8d97f7f2-258d-40c3-ad7f-db2027cd44f9 (32d96d38-3745-4ae0-96ff-5afe11fcae22) Active Faulted Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30"level":30,"time":"2023-09-22T23:08:02.545086327Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}{
","msg"time{"::"""msg":[2] Transition from New to WaitActive"2023-09-22T23:08:02.545091532Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0msg,"":"name":"crucible","[1] Transition from LiveRepairReady to LiveRepair",level",:"30v"":v"","time"[1] Transition from Faulted to LiveRepairReady":","v"2023-09-22T23:08:02.545152171Z":0,,""hostnamename""::""crucible","ip-10-150-1-74.us-west-2.compute.internal"level",:"30pid"::04291,"}
name":"crucible","level":{30,"time":""msg":"2023-09-22T23:08:02.545182518Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","[2] 85751bdd-8856-4e5b-8ef6-adad326659c5 (8cb064bd-4131-4172-9681-8a036ecff523) Active Active New ds_transition to WaitActive"pid":,4291"v":}0
,,""time"{:"2023-09-22T23:08:02.545194208Z""msg",":"hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid"[1] 8d97f7f2-258d-40c3-ad7f-db2027cd44f9 (32d96d38-3745-4ae0-96ff-5afe11fcae22) Active LiveRepairReady Active ds_transition to LiveRepair":,"4291}
name":"crucible","level":300,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.545256216Z","hostname":"{ip-10-150-1-74.us-west-2.compute.internal","pid"":msg"4291,":"time"}:
"[2] 35eecd9f-6208-42ff-b6c0-c4c02caf8b9b (63d034ab-cd7a-4ae2-9a0d-89f39e17085b) Active Active WaitActive ds_transition to WaitQuorum",2023-09-22T23:08:02.545268365Z""v{",":hostname"":0msg,"name":"crucible","level":30v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.54530405Z","hostname":","ip-10-150-1-74.us-west-2.compute.internal"time,":""pid""::42912023-09-22T23:08:02.545315816Z}
""[2] Transition from New to WaitActive"ip-10-150-1-74.us-west-2.compute.internal","pid":,4291"v":}
0,"name":"crucible"{,"level":"30msg":"Repair for extent 1 s:0 d:[ClientId(2)]","v":0,"name":"{crucible","level":30"msg":""[2] Transition from WaitActive to WaitQuorum,"",hostname"":v"":0,"name,"","timetime""::""2023-09-22T23:08:02.545375326Z"2023-09-22T23:08:02.545367602Z","hostname",:""hostname":"ip-10-150-1-74.us-west-2.compute.internal","ip-10-150-1-74.us-west-2.compute.internal"pid,"":pid4291":,4291"":"}downstairs
"}
{:"crucible",""level"msg":ip-10-150-1-74.us-west-2.compute.internal:"30,"time":"2023-09-22T23:08:02.54542421Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
,"pid":4291{}
"{msg":""msg":"[1] Transition from LiveRepairReady to LiveRepair","[2] 35eecd9f-6208-42ff-b6c0-c4c02caf8b9b (63d034ab-cd7a-4ae2-9a0d-89f39e17085b) Active Active WaitQuorum ds_transition to Activev":"0,,""v"name"::""0crucible",","name"level:[2] 85751bdd-8856-4e5b-8ef6-adad326659c5 (8cb064bd-4131-4172-9681-8a036ecff523) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30"crucible","level":,30"time":"2023-09-22T23:08:02.545499486Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291"}:
30,"time":"2023-09-22T23:08:02.545521708Z","{hostname":"ip-10-150-1-74.us-west-2.compute.internal",""pid"msg:"4291,"time":"2023-09-22T23:08:02.545539191Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
}
:"{[2] Transition from WaitActive to WaitQuorum"",msg"":v"":0,"[2] Transition from WaitQuorum to Active"name":","crucible"v":,"0level":,"30name"test live_repair::repair_test::test_live_repair_enqueue_repair_repair ... :"crucibleok"
,"level":30,"time":"2023-09-22T23:08:02.545606296Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
,"time":"2023-09-22T23:08:02.545621756Z","{hostname":"ip-10-150-1-74.us-west-2.compute.internal"","msg"pid":4291}
{:""msg":"35eecd9f-6208-42ff-b6c0-c4c02caf8b9b is now active with session: 63d034ab-cd7a-4ae2-9a0d-89f39e17085b"[2] 85751bdd-8856-4e5b-8ef6-adad326659c5 (8cb064bd-4131-4172-9681-8a036ecff523) Active Active WaitQuorum ds_transition to Active","v":0,",v"":name":0","cruciblename"":","crucible"level":,"30level":30,"time":"2023-09-22T23:08:02.545686817Z",","hostname":time"":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{2023-09-22T23:08:02.545688373Z""msg":","hostname":"[1] 35eecd9f-6208-42ff-b6c0-c4c02caf8b9b (63d034ab-cd7a-4ae2-9a0d-89f39e17085b) Active Active Active ds_transition to Faultedip-10-150-1-74.us-west-2.compute.internal"",,""pidv""::42910,"name":}"
crucible","level":30{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.545751608Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{,"time":"2023-09-22T23:08:02.545766926Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":test live_repair::repair_test::test_live_repair_flush_is_flush ... "ok
85751bdd-8856-4e5b-8ef6-adad326659c5 is now active with session: 8cb064bd-4131-4172-9681-8a036ecff523","v":0,"name":"crucible","level":30",msg""time":":"2023-09-22T23:08:02.545820642Z"[1] Transition from Active to Faulted","hostname","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.545883049Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 35eecd9f-6208-42ff-b6c0-c4c02caf8b9b (63d034ab-cd7a-4ae2-9a0d-89f39e17085b) Active Faulted Active ds_transition to LiveRepairReady":,"v"":0,"name":"ip-10-150-1-74.us-west-2.compute.internal"crucible","level,":"30pid":4291}
{"msg":","time":"2023-09-22T23:08:02.545975161Z","[1] 85751bdd-8856-4e5b-8ef6-adad326659c5 (8cb064bd-4131-4172-9681-8a036ecff523) Active Active Active ds_transition to Faulted"hostname":","v":0ip-10-150-1-74.us-west-2.compute.internal",,""pid"name":":crucible4291","}level"
{":msg":"30[1] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30{,"time"":","msg"time:2023-09-22T23:08:02.546023976Z""":"Crucible stats registered with UUID: 182bd4a1-263e-4c1e-9802-54d965519b48","v":02023-09-22T23:08:02.546030697Z,""name":,""crucible"hostname":","level":ip-10-150-1-74.us-west-2.compute.internal"30,"pid,":"4291hostname"}:
,"time":"2023-09-22T23:08:02.546065963Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{{"msg"":"msg":"Crucible 182bd4a1-263e-4c1e-9802-54d965519b48 has session id: 8d0a0614-4017-41fa-99e2-ce0e155fa6af","v":0[1] 35eecd9f-6208-42ff-b6c0-c4c02caf8b9b (63d034ab-cd7a-4ae2-9a0d-89f39e17085b) Active LiveRepairReady Active ds_transition to LiveRepair","v":0,"name":"crucible"","level":30ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":","time":"[1] Transition from Active to Faulted","2023-09-22T23:08:02.546131337Z"v":,"0hostname",:","name"ip-10-150-1-74.us-west-2.compute.internal:""crucible","pid":,"4291level":}30
"name{":"crucible",""msg"level:":"30[1] Transition from LiveRepairReady to LiveRepair","v":0,",name"":"time"crucible":","level":2023-09-22T23:08:02.546168776Z"30,"hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
,"time":"2023-09-22T23:08:02.546182556Z"{,",hostname":"""time"msg":ip-10-150-1-74.us-west-2.compute.internal"","pid":4291}
2023-09-22T23:08:02.546193179Z","hostname":"{ip-10-150-1-74.us-west-2.compute.internal","pid":"4291msg":"}
[1] 85751bdd-8856-4e5b-8ef6-adad326659c5 (8cb064bd-4131-4172-9681-8a036ecff523) Active Faulted Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30:"[0] 182bd4a1-263e-4c1e-9802-54d965519b48 (1ab74527-7a63-44a1-a8c0-319deaa158d7) New New New ds_transition to WaitActive","v":0,"name":"crucible",,""timelevel"":":302023-09-22T23:08:02.546241921Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{",msg":""time":"[1] Transition from Faulted to LiveRepairReady"2023-09-22T23:08:02.546262663Z","v",:"0hostname":,""name":"crucible",ip-10-150-1-74.us-west-2.compute.internal"","level"pid"::304291}
{"msg":"[0] Transition from New to WaitActive","v":,"0time",":"name":"crucible2023-09-22T23:08:02.546296012Z"",,""level"hostname"::"30ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"{msg":","time":""msg":2023-09-22T23:08:02.546322089Z"","[1] 85751bdd-8856-4e5b-8ef6-adad326659c5 (8cb064bd-4131-4172-9681-8a036ecff523) Active LiveRepairReady Active ds_transition to LiveRepair"hostname":",Crucible stats registered with UUID: 3b2f825e-b910-4529-8739-ae82806d8b76""v":0,ip-10-150-1-74.us-west-2.compute.internal",""vname""::0",,crucible"""name,"":level""pidcrucible:"",30":level"4291:30}
{"msg":","time":"2023-09-22T23:08:02.546358173Z",",hostname"":"time":[0] 182bd4a1-263e-4c1e-9802-54d965519b48 (1ab74527-7a63-44a1-a8c0-319deaa158d7) WaitActive New New ds_transition to WaitQuorum""ip-10-150-1-74.us-west-2.compute.internal",,""pidv"2023-09-22T23:08:02.546360756Z"":,:"42910hostname},
":""name":"{ip-10-150-1-74.us-west-2.compute.internal"crucible,""msg"pid:"":"4291,"}[1] Transition from LiveRepairReady to LiveRepair
"level",:"{30v":0","msg"name:"":"crucible","level":30Crucible 3b2f825e-b910-4529-8739-ae82806d8b76 has session id: 51a953b0-5569-419e-a249-0b56135d74c9","v":0,"name":"crucible","level":30,","time":time"":"2023-09-22T23:08:02.546402583Z"2023-09-22T23:08:02.546407325Z,"",time,"":""hostname"hostname:2023-09-22T23:08:02.546413754Z""":,""ip-10-150-1-74.us-west-2.compute.internal"hostname,"":"pid":ip-10-150-1-74.us-west-2.compute.internal4291"ip-10-150-1-74.us-west-2.compute.internal}"
,,test live_repair::repair_test::test_live_repair_no_repair_yet ... {"""pidpid":msg4291"ok:"}":
4291Create read repair deps for extent 1
"}
{,"v":0","msg":name"":"{crucible","level":40"msg":"[0] 3b2f825e-b910-4529-8739-ae82806d8b76 (2fa4702d-3c78-4506-b8ee-ef9259c4e868) New New New ds_transition to WaitActive","v":0,"[0] Transition from WaitActive to WaitQuorum"name":","crucible"v,"":level"0:,30,""time"name"::""crucible"2023-09-22T23:08:02.546477609Z",","level":hostname"30:,""time":"ip-10-150-1-74.us-west-2.compute.internal",2023-09-22T23:08:02.546488799Z""pid",:"4291hostname":"}
ip-10-150-1-74.us-west-2.compute.internal","pid":{----------------------------------------------------------------
,"time":"2023-09-22T23:08:02.546497599Z" Crucible gen:0 GIO:true work queues:  Upstairs:1  downstairs:1
4291}
GW_ID      ACK  DSID    TYPE  BLOCKS  DS:0  DS:1  DS:2  REPLAY
    1 AckReady  1000   Write       1   New   New{"msg":","[0] Transition from New to WaitActive"hostname":,""v":0,"ip-10-150-1-74.us-west-2.compute.internal"name":",crucible"",pid"":level":429130   New}  false
   STATES      DS:0   DS:1   DS:2   TOTAL
    New           1      1      1   
,   3
{    Sent          0      0      0      0
    Done          0      0      0   "   0
"    Skipped       0      0   time"   0   :   0
"msg":    Error      "   0      0      0      0
msg":"Last Flush: 0 0 0 
[0] 182bd4a1-263e-4c1e-9802-54d965519b48 (1ab74527-7a63-44a1-a8c0-319deaa158d7) WaitQuorum New New ds_transition to Active"","v":2023-09-22T23:08:02.546648104Z"0,"name",:""cruciblehostname"":",Downstairs last five completed:"
IO Read  1004 extent 0 added deps 1ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
Upstairs last five completed:  {
","v":0,"name":"crucible","level":level40":30"msg":"[0] 3b2f825e-b910-4529-8739-ae82806d8b76 (2fa4702d-3c78-4506-b8ee-ef9259c4e868) WaitActive New New ds_transition to WaitQuorum","v":0,",name"":"time":crucible"","level":2023-09-22T23:08:02.546765873Z"30,"hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
,"time":",2023-09-22T23:08:02.546767696Z""time":,""hostname":"2023-09-22T23:08:02.546792486Z","ip-10-150-1-74.us-west-2.compute.internal"hostname,"":pid"":4291ip-10-150-1-74.us-west-2.compute.internal"},
"pid":4291}
{{"msg":"{[0] Transition from WaitQuorum to Active"",""msgvmsg"":""::"0[0] Transition from WaitActive to WaitQuorum,""name"Crucible stats registered with UUID: 231d0792-4fa5-43d8-bed8-61976ebe4426:",""crucible"v",,:""v0level"",:":030name,"":"name":crucible"","crucible"level":,"30level":30,"time":"2023-09-22T23:08:02.546868264Z","hostname":","time":"ip-10-150-1-74.us-west-2.compute.internal","pid"2023-09-22T23:08:02.546875076Z":4291,"hostname}"
,:""time":"{ip-10-150-1-74.us-west-2.compute.internal",2023-09-22T23:08:02.546876832Z""pid"",:msg""4291:hostname"}:"
"[1] 182bd4a1-263e-4c1e-9802-54d965519b48 (1ab74527-7a63-44a1-a8c0-319deaa158d7) Active New New ds_transition to WaitActive"ip-10-150-1-74.us-west-2.compute.internal,""{,v":"0",pid"msg"::""4291name":"crucible"},
"[0] 3b2f825e-b910-4529-8739-ae82806d8b76 (2fa4702d-3c78-4506-b8ee-ef9259c4e868) WaitQuorum New New ds_transition to Activelevel":"30,"v":0{,"name":"crucible","level":"30msg":"Crucible 231d0792-4fa5-43d8-bed8-61976ebe4426 has session id: daddcc97-92dc-410b-8580-428379cd0b09","v":0,"name":"crucible",,""level":time"30,:""time":"2023-09-22T23:08:02.546940974Z","2023-09-22T23:08:02.546946821Z"hostname":","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":ip-10-150-1-74.us-west-2.compute.internal4291","}pid"
:4291}
,{"time":"{"2023-09-22T23:08:02.546960331Z"msg":,"""msg"hostname"[1] Transition from New to WaitActive"::",""v":ip-10-150-1-74.us-west-2.compute.internal"0,[0] Transition from WaitQuorum to Active""pid",,:"4291v":"test live_repair::repair_test::test_live_repair_repair_read_push ... 0}
ok
{"msg":,name""":name"":"cruciblecrucible"",,[0] 231d0792-4fa5-43d8-bed8-61976ebe4426 (311603ce-e15d-47ce-bbc6-29c87d03dcc9) New New New ds_transition to WaitActive""level"level""::,"30v"30:0,"name":"crucible","level":30,"time":","2023-09-22T23:08:02.547047453Z"time":","hostname":",2023-09-22T23:08:02.547048681Z""ip-10-150-1-74.us-west-2.compute.internal,time"","hostname":""pid:"":ip-10-150-1-74.us-west-2.compute.internal"42912023-09-22T23:08:02.547053692Z",}"
,pid"":hostname"4291:"{}
ip-10-150-1-74.us-west-2.compute.internal"","msgpid""::4291{"}
"[1] 182bd4a1-263e-4c1e-9802-54d965519b48 (1ab74527-7a63-44a1-a8c0-319deaa158d7) Active WaitActive New ds_transition to WaitQuorummsg"":","v"{:0,"name":"crucible""[1] 3b2f825e-b910-4529-8739-ae82806d8b76 (2fa4702d-3c78-4506-b8ee-ef9259c4e868) Active New New ds_transition to WaitActivemsg","",:""v"level"::[0] Transition from New to WaitActive0","30name":,""v"crucible":,"0level",:"30name":"crucible","level":30,"time":"2023-09-22T23:08:02.547128256Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid",:"4291time":"}
2023-09-22T23:08:02.547135768Z",",time"":{hostname"":""2023-09-22T23:08:02.547138944Z"msgip-10-150-1-74.us-west-2.compute.internal","",:hostname"""pid"::"[1] Transition from WaitActive to WaitQuorum"4291,"v"ip-10-150-1-74.us-west-2.compute.internal:"0},
,""pid"name"::"4291{crucible",}"
level"":msg"30:"{[1] Transition from New to WaitActive","v":0","msg"name"::""crucible","level":30[0] 231d0792-4fa5-43d8-bed8-61976ebe4426 (311603ce-e15d-47ce-bbc6-29c87d03dcc9) WaitActive New New ds_transition to WaitQuorum","v":0,",name"":"time"crucible":","level":2023-09-22T23:08:02.54719662Z"30,"hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid",:"4291time":"}
2023-09-22T23:08:02.547207564Z","hostname":"{ip-10-150-1-74.us-west-2.compute.internal,"""time"msg"::"","pid"2023-09-22T23:08:02.547219247Z":4291,"hostname":}"
ip-10-150-1-74.us-west-2.compute.internal"[1] 182bd4a1-263e-4c1e-9802-54d965519b48 (1ab74527-7a63-44a1-a8c0-319deaa158d7) Active WaitQuorum New ds_transition to Active,""pid{,"":v":"0msg"4291,:""}name"
[1] 3b2f825e-b910-4529-8739-ae82806d8b76 (2fa4702d-3c78-4506-b8ee-ef9259c4e868) Active WaitActive New ds_transition to WaitQuorum":","v":crucible"{0,,""name":""crucible"msg","level":level"::30"30[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.547277518Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
,"{time":""msg":"2023-09-22T23:08:02.547278566Z"[1] Transition from WaitActive to WaitQuorum",","hostnamev":",:"0"time",:""ip-10-150-1-74.us-west-2.compute.internal"name"2023-09-22T23:08:02.547286638Z:"",crucible",pid"":",4291"hostname"level"}:
:"30ip-10-150-1-74.us-west-2.compute.internal"{,"pid":4291"}msg"
:","[1] Transition from WaitQuorum to Active"time":",{"v"2023-09-22T23:08:02.547325799Z":,"0"msg":,"hostname"":"name":"crucible"ip-10-150-1-74.us-west-2.compute.internal",,""levelpid"[0] 231d0792-4fa5-43d8-bed8-61976ebe4426 (311603ce-e15d-47ce-bbc6-29c87d03dcc9) WaitQuorum New New ds_transition to Active"":,4291:"30}v
":0,"name"{:"crucible",""level"msg"::"30,[1] 3b2f825e-b910-4529-8739-ae82806d8b76 (2fa4702d-3c78-4506-b8ee-ef9259c4e868) Active WaitQuorum New ds_transition to Active"",time"":v":"0,"name":"crucible"2023-09-22T23:08:02.547360142Z","level":,"30hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"}time"
:,""time":"2023-09-22T23:08:02.547384682Z"2023-09-22T23:08:02.547371939Z{","hostname":",""ip-10-150-1-74.us-west-2.compute.internalhostname""msg":":","pid":4291ip-10-150-1-74.us-west-2.compute.internal"}
,"pid[2] 182bd4a1-263e-4c1e-9802-54d965519b48 (1ab74527-7a63-44a1-a8c0-319deaa158d7) Active Active New ds_transition to WaitActive""{,:"4291"v"msg":"}:
0[1] Transition from WaitQuorum to Active",,""v":0name":","crucible"name","{level":":crucible","30"level":30msg":"[0] Transition from WaitQuorum to Active","v":0,"name":","crucible"time":","level"2023-09-22T23:08:02.547436503Z":,"30hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
,"time"{:""msg":"2023-09-22T23:08:02.547435743Z","hostname":",[2] 3b2f825e-b910-4529-8739-ae82806d8b76 (2fa4702d-3c78-4506-b8ee-ef9259c4e868) Active Active New ds_transition to WaitActive"","timevip-10-150-1-74.us-west-2.compute.internal""":,""pid":2023-09-22T23:08:02.547447982Z0":,4291,""name"hostname"::}""
ip-10-150-1-74.us-west-2.compute.internal"crucible",","pidlevel{"":{:429130"msg"}:""
Crucible stats registered with UUID: 1a8122c3-953b-42de-a373-2c666ddd65a5"msg":","v":[2] Transition from New to WaitActive0"{,,"",nametime""::"""crucible"msg":v"":"2023-09-22T23:08:02.547492901Z",0,""level","hostname:name[1] 231d0792-4fa5-43d8-bed8-61976ebe4426 (311603ce-e15d-47ce-bbc6-29c87d03dcc9) Active New New ds_transition to WaitActive""30:","":vip-10-150-1-74.us-west-2.compute.internal""":,0"crucible","pidname",:"":level"4291"crucible:}",
"30,time""{:level""":msg":2023-09-22T23:08:02.547523165Z""30,"hostname"[2] Transition from New to WaitActive:"","v":0,ip-10-150-1-74.us-west-2.compute.internal""name,"":"pid":crucible"4291,"level"}:
30,"{time":""2023-09-22T23:08:02.54754485Z"msg":","hostname":","time":ip-10-150-1-74.us-west-2.compute.internal"","Crucible 1a8122c3-953b-42de-a373-2c666ddd65a5 has session id: 53637ab1-3cfb-4773-8699-b0612dee3265,""2023-09-22T23:08:02.547550606Ztime,"":v""pid":2023-09-22T23:08:02.547560785Z0",:",""name"hostname:"":,crucible""4291",ip-10-150-1-74.us-west-2.compute.internal"hostname}"level
:"":,""30pid":4291{ip-10-150-1-74.us-west-2.compute.internal"},
""pid":msg"4291:,{""}"time
"msg:"":[2] 182bd4a1-263e-4c1e-9802-54d965519b48 (1ab74527-7a63-44a1-a8c0-319deaa158d7) Active Active WaitActive ds_transition to WaitQuorum""2023-09-22T23:08:02.547596024Z",,{""v":hostname"0:[2] 3b2f825e-b910-4529-8739-ae82806d8b76 (2fa4702d-3c78-4506-b8ee-ef9259c4e868) Active Active WaitActive ds_transition to WaitQuorum"",",msg"":name"ip-10-150-1-74.us-west-2.compute.internal""v,""::pid0",:""4291name"}:
""{crucible"crucible,""msg"level:"":"[1] Transition from New to WaitActive30","level",:"[0] 1a8122c3-953b-42de-a373-2c666ddd65a5 (7c39838b-aea6-4a95-a4cb-86170299d0c5) New New New ds_transition to WaitActivev"30:"0,",v":"0name,"",:name"":time"""crucible:""crucible",2023-09-22T23:08:02.547646361Z""level,,"":"hostname30"level":":,30"ip-10-150-1-74.us-west-2.compute.internal"time":,""pid":42912023-09-22T23:08:02.547651541Z"}
,,""time{hostname"""::msg"":""2023-09-22T23:08:02.54766642Z","ip-10-150-1-74.us-west-2.compute.internal"hostname[2] Transition from WaitActive to WaitQuorum,",pid""":time"4291:",""}:2023-09-22T23:08:02.547670162Z"v"",:
"ip-10-150-1-74.us-west-2.compute.internal0",,""hostnamepidname""::"{4291":"}
crucible"ip-10-150-1-74.us-west-2.compute.internal"{,"""level"msg:,30""msg"pid":::"4291"[0] Transition from New to WaitActive}"
[2] Transition from WaitActive to WaitQuorum",,"",timev"""::0",{"2023-09-22T23:08:02.547715904Z"name",v"":hostname:""msg0":,":"""name":crucible"",ip-10-150-1-74.us-west-2.compute.internal""crucible",level""[1] 231d0792-4fa5-43d8-bed8-61976ebe4426 (311603ce-e15d-47ce-bbc6-29c87d03dcc9) Active WaitActive New ds_transition to WaitQuorum"pid:":304291,,""}level"
v"::{300,"",time"msg""::""name":"2023-09-22T23:08:02.547753507Zcrucible"",","level":hostname"[2] 3b2f825e-b910-4529-8739-ae82806d8b76 (2fa4702d-3c78-4506-b8ee-ef9259c4e868) Active Active WaitQuorum ds_transition to Active:"30,""v":0,"ip-10-150-1-74.us-west-2.compute.internalname"":,""pidcrucible"":,4291"level"}:
,30"{time":""msg":"2023-09-22T23:08:02.547762515Z","[0] 1a8122c3-953b-42de-a373-2c666ddd65a5 (7c39838b-aea6-4a95-a4cb-86170299d0c5) WaitActive New New ds_transition to WaitQuorum",,,"""vtime""::0",time":"2023-09-22T23:08:02.547781092Z"name",:"""hostnamecrucible"":","2023-09-22T23:08:02.547773487Z"level":ip-10-150-1-74.us-west-2.compute.internal30",","hostname"pid":":hostname4291":ip-10-150-1-74.us-west-2.compute.internal}"
,ip-10-150-1-74.us-west-2.compute.internal""{,""time",msg""::"""pidpid"2023-09-22T23:08:02.547802108Z[2] Transition from WaitQuorum to Active"":",,"4291hostname""v:""::}0ip-10-150-1-74.us-west-2.compute.internal",,""namepid"
"{"msg":"[1] Transition from WaitActive to WaitQuorum"4291,"v":0,"}name
":"crucible","level":30{"msg"::"4291}
[2] 182bd4a1-263e-4c1e-9802-54d965519b48 (1ab74527-7a63-44a1-a8c0-319deaa158d7) Active Active WaitQuorum ds_transition to Active","{v":0","msg"name,":"time":":"crucible"","[0] Transition from WaitActive to WaitQuorumlevel""2023-09-22T23:08:02.547873142Z":,,""v"hostname":"30:0,"ip-10-150-1-74.us-west-2.compute.internalname":"crucible","level":30,"time",:""time":"2023-09-22T23:08:02.547911704Z","2023-09-22T23:08:02.547906494Z"hostname":","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":ip-10-150-1-74.us-west-2.compute.internal"4291,"pid}"
:4291}"
{,"pid"":msg":4291{"}
[0] 1a8122c3-953b-42de-a373-2c666ddd65a5 (7c39838b-aea6-4a95-a4cb-86170299d0c5) WaitQuorum New New ds_transition to Active"","msg"v:""{:[2] Transition from WaitQuorum to Active",0""msg"v:,":"0name","":name":"crucible"","crucible"[1] 231d0792-4fa5-43d8-bed8-61976ebe4426 (311603ce-e15d-47ce-bbc6-29c87d03dcc9) Active WaitQuorum New ds_transition to Active"level",:",level:"""30v":crucible30":0,","level"name"::"30crucible","level":30,"time":","2023-09-22T23:08:02.547985294Z",,""time":hostname""time:"":"2023-09-22T23:08:02.547983739Z",2023-09-22T23:08:02.547990417Zip-10-150-1-74.us-west-2.compute.internal""",,"hostname"pid"":hostname4291",:}"
ip-10-150-1-74.us-west-2.compute.internal"",:time{""pid""":msg:"4291:"ip-10-150-1-74.us-west-2.compute.internal""},
[0] Transition from WaitQuorum to Active"2023-09-22T23:08:02.547993227Z""pid"{,,"""v"msg:"0:,:""name"hostname"4291::""}3b2f825e-b910-4529-8739-ae82806d8b76 is now active with session: 2fa4702d-3c78-4506-b8ee-ef9259c4e868crucible""ip-10-150-1-74.us-west-2.compute.internal,,
""",levelv""::300"{pid",":name":4291""crucible}"msg,
"",level""::time30{"":"2023-09-22T23:08:02.548057734Z""msg",:"182bd4a1-263e-4c1e-9802-54d965519b48 is now active with session: 1ab74527-7a63-44a1-a8c0-319deaa158d7"hostname"":,""[1] Transition from WaitQuorum to Active"timeip-10-150-1-74.us-west-2.compute.internal"":,",,"2023-09-22T23:08:02.548074747Z""pid""v"v"::,:"42910hostname}"
:,"{"0name","ip-10-150-1-74.us-west-2.compute.internalmsg"",:":""pid"name""::"4291crucible"}[1] 1a8122c3-953b-42de-a373-2c666ddd65a5 (7c39838b-aea6-4a95-a4cb-86170299d0c5) Active New New ds_transition to WaitActive"
,,crucible""{",v""msg:"0:,""levelname"":"":crucible"level"[1] 3b2f825e-b910-4529-8739-ae82806d8b76 (2fa4702d-3c78-4506-b8ee-ef9259c4e868) Active Active Active ds_transition to Faulted,""30:,level""v:"30:300,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.548136235Z",,"",hostnametime""::""",time"ip-10-150-1-74.us-west-2.compute.internal:"2023-09-22T23:08:02.548140417Z,"",pid"":hostname4291"":}
"time""{:"ip-10-150-1-74.us-west-2.compute.internal""2023-09-22T23:08:02.548134121Zmsg,":2023-09-22T23:08:02.548137199Z""""pid",[1] Transition from New to WaitActive:",4291"hostname,}"
v"":{0hostname"",msg"""::""name"::[1] Transition from Active to Faulted""ip-10-150-1-74.us-west-2.compute.internal,crucible""v,"":"0level,"":pid30,"""name":ip-10-150-1-74.us-west-2.compute.internal":4291,""}crucible"pid,,"""leveltime:"":
":4291302023-09-22T23:08:02.548201056Z",}"
hostname{":","time":"ip-10-150-1-74.us-west-2.compute.internal"{,2023-09-22T23:08:02.548220803Z"""msgpid",:"":msg""4291":}hostname
"":[2] 231d0792-4fa5-43d8-bed8-61976ebe4426 (311603ce-e15d-47ce-bbc6-29c87d03dcc9) Active Active New ds_transition to WaitActive""{,"v":"ip-10-150-1-74.us-west-2.compute.internal0msg"",,:""pid"[1] 182bd4a1-263e-4c1e-9802-54d965519b48 (1ab74527-7a63-44a1-a8c0-319deaa158d7) Active Active Active ds_transition to Faulted":"4291,name""}:"v"
[1] 1a8122c3-953b-42de-a373-2c666ddd65a5 (7c39838b-aea6-4a95-a4cb-86170299d0c5) Active WaitActive New ds_transition to WaitQuorum"crucible"{,,""v"msg:":0"",level":name""::0[1] 3b2f825e-b910-4529-8739-ae82806d8b76 (2fa4702d-3c78-4506-b8ee-ef9259c4e868) Active Faulted Active ds_transition to LiveRepairReady""30,,crucible""v",:"0"level,"":name"30name"::""crucible"crucible",,""level"level"::3030,"time":",2023-09-22T23:08:02.548280296Z"","time":hostname"":","2023-09-22T23:08:02.548274497Z"timeip-10-150-1-74.us-west-2.compute.internal"":,"",pid"2023-09-22T23:08:02.548287802Z""hostname,:""4291:hostname}"
:"","{ip-10-150-1-74.us-west-2.compute.internal"ip-10-150-1-74.us-west-2.compute.internal",time""msg"",::pid""""pid":2023-09-22T23:08:02.548288574Z"4291:[1] Transition from WaitActive to WaitQuorum"4291,},
"}"
hostname"{v":"0:,msg""name:""{":"crucible[1] Transition from Faulted to LiveRepairReady"",",ip-10-150-1-74.us-west-2.compute.internal"level""v:":"030,msg"":name"",:""pid"[2] Transition from New to WaitActive"crucible":,,"4291,level"":time30""}:v"":
,2023-09-22T23:08:02.548354368Z""time",:""0hostname":2023-09-22T23:08:02.548367548Z"",,{"ip-10-150-1-74.us-west-2.compute.internal""name",hostname""":"pidcrucible"::"4291"msg"}ip-10-150-1-74.us-west-2.compute.internal
":,{,""""msgpid""::4291"level}"
:[1] Transition from Active to Faulted"30{[1] 1a8122c3-953b-42de-a373-2c666ddd65a5 (7c39838b-aea6-4a95-a4cb-86170299d0c5) Active WaitQuorum New ds_transition to Active","",msg""v:v"""::00,","name"name"::[1] 3b2f825e-b910-4529-8739-ae82806d8b76 (2fa4702d-3c78-4506-b8ee-ef9259c4e868) Active LiveRepairReady Active ds_transition to LiveRepair""",crucible"v"",:"0level,"":crucible"30name":,""level"crucible,":",30time":""level":,"30time":2023-09-22T23:08:02.548409167Z"","2023-09-22T23:08:02.548425369Z"hostname",:""hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid"ip-10-150-1-74.us-west-2.compute.internal":,,""pid"time:"4291:4291"}
2023-09-22T23:08:02.548434436Z"},{
""msg",hostname:"":""{time"[1] Transition from WaitQuorum to Active"ip-10-150-1-74.us-west-2.compute.internal:,"",""vpid""::04291,msg}"
name"":{"":crucible""msg,2023-09-22T23:08:02.548431931Z"""":",level":"30[1] Transition from LiveRepairReady to LiveRepairhostname"":","[2] 231d0792-4fa5-43d8-bed8-61976ebe4426 (311603ce-e15d-47ce-bbc6-29c87d03dcc9) Active Active WaitActive ds_transition to WaitQuorum"v"ip-10-150-1-74.us-west-2.compute.internal:,0","",,v"name"time:""""crucible:"",:pid0"2023-09-22T23:08:02.5484838Z:""level,"":4291hostname30","}name
":":","crucible"timeip-10-150-1-74.us-west-2.compute.internal"":,""{pid,2023-09-22T23:08:02.548508294Z"":,4291""hostname}"level":
":"30{ip-10-150-1-74.us-west-2.compute.internal"msg",msg""pid:"":"4291:"}
[2] 1a8122c3-953b-42de-a373-2c666ddd65a5 (7c39838b-aea6-4a95-a4cb-86170299d0c5) Active Active New ds_transition to WaitActive","v":0,"[1] 182bd4a1-263e-4c1e-9802-54d965519b48 (1ab74527-7a63-44a1-a8c0-319deaa158d7) Active Faulted Active ds_transition to LiveRepairReady"name":","crucible"v",:"0level",:",30name"":"time":crucible"","level":2023-09-22T23:08:02.54853145Z"30,"hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"time"}:
"2023-09-22T23:08:02.548551187Z","hostname":"{ip-10-150-1-74.us-west-2.compute.internal,"","msg"time":":""pid":[2] Transition from WaitActive to WaitQuorum4291"2023-09-22T23:08:02.548556881Z"},,
""v"hostname":{:0",""name"msg"::ip-10-150-1-74.us-west-2.compute.internal"","crucible"",pid[2] Transition from New to WaitActive""":level"4291:,"30}v":
0,"name":"crucible","level":{30"msg":"[1] Transition from Faulted to LiveRepairReady",",v"":time"0:","name":"2023-09-22T23:08:02.548606176Z"crucible,,"""hostname"time":,"":"level":ip-10-150-1-74.us-west-2.compute.internal302023-09-22T23:08:02.548616292Z"",","hostname":"pid":4291ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
}
{{,"time":"""msg":"msg2023-09-22T23:08:02.548639017Z"":","hostname":"[2] 1a8122c3-953b-42de-a373-2c666ddd65a5 (7c39838b-aea6-4a95-a4cb-86170299d0c5) Active Active WaitActive ds_transition to WaitQuorum","v":0ip-10-150-1-74.us-west-2.compute.internal",",name[2] 231d0792-4fa5-43d8-bed8-61976ebe4426 (311603ce-e15d-47ce-bbc6-29c87d03dcc9) Active Active WaitQuorum ds_transition to Active"""pid:",""cruciblev:"4291,""level"::30}0
,"name":"crucible","{time":","level"2023-09-22T23:08:02.548675836Z":,"30hostname":""msg":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 182bd4a1-263e-4c1e-9802-54d965519b48 (1ab74527-7a63-44a1-a8c0-319deaa158d7) Active LiveRepairReady Active ds_transition to LiveRepair",[2] Transition from WaitActive to WaitQuorum"","v"v"::0,0"name,":"",namecrucible"","timelevel"""::30:""crucible"2023-09-22T23:08:02.548689608Z",",level"":hostname"30,":time":""2023-09-22T23:08:02.54871486Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","ip-10-150-1-74.us-west-2.compute.internal"pid",":pid":42914291}
}
,{"time":""msg":"{2023-09-22T23:08:02.548721977Z","hostname":""msg":"ip-10-150-1-74.us-west-2.compute.internal","[2] 1a8122c3-953b-42de-a373-2c666ddd65a5 (7c39838b-aea6-4a95-a4cb-86170299d0c5) Active Active WaitQuorum ds_transition to Activepid"[2] Transition from WaitQuorum to Active"":,,4291""v}v":
"0:,"0name",:{""name":crucible"""crucible"msg",:"","level"level":[1] Transition from LiveRepairReady to LiveRepair"30:,"30v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.548776723Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
,"{time":""msg":"2023-09-22T23:08:02.548778741Z","hostname":"[2] Transition from WaitQuorum to Active","v":ip-10-150-1-74.us-west-2.compute.internal"0,,""pid"name"::4291"crucible"},
"level":30test live_repair::repair_test::test_live_repair_send_io_write_below ... {ok
,"",time"":msg"":time""2023-09-22T23:08:02.548783846Z":","hostname":2023-09-22T23:08:02.548818038Z""231d0792-4fa5-43d8-bed8-61976ebe4426 is now active with session: 311603ce-e15d-47ce-bbc6-29c87d03dcc9",",hostname"ip-10-150-1-74.us-west-2.compute.internal"v",":":0pid"",:"4291name"ip-10-150-1-74.us-west-2.compute.internal":},"
pid"":crucible4291","{}level"
:"30msg":"{Write to Extent 0:2:9 under repair","v"":msg"0:","name":"crucible","level":401a8122c3-953b-42de-a373-2c666ddd65a5 is now active with session: 7c39838b-aea6-4a95-a4cb-86170299d0c5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.548869939Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,}"
,"time":time":""{2023-09-22T23:08:02.548886389Z"2023-09-22T23:08:02.548880671Z","hostname":",""hostname"ip-10-150-1-74.us-west-2.compute.internalmsg:""",:ip-10-150-1-74.us-west-2.compute.internal""",pid":"4291pid":}
[1] 231d0792-4fa5-43d8-bed8-61976ebe4426 (311603ce-e15d-47ce-bbc6-29c87d03dcc9) Active Active Active ds_transition to Faulted"4291,"{v"}:"0msg","
:"name":"crucible","level"{:[1] 1a8122c3-953b-42de-a373-2c666ddd65a5 (7c39838b-aea6-4a95-a4cb-86170299d0c5) Active Active Active ds_transition to Faulted"30,"v":0,""name":msg"":"crucible","level":30Write 1:0:9 past extent under repair 0","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:02.548942143Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"}
time":"{"msg":"2023-09-22T23:08:02.54893712Z"[1] Transition from Active to Faulted",,""v":0hostname",":name",:"""crucible"time","ip-10-150-1-74.us-west-2.compute.internal":level":"30,"pid":2023-09-22T23:08:02.548949333Z"4291,"hostname"}:
","time":"2023-09-22T23:08:02.548973089Z"ip-10-150-1-74.us-west-2.compute.internal",",{"hostname":pid"":4291"ip-10-150-1-74.us-west-2.compute.internal",msg"}":"pid":
4291[1] Transition from Active to Faulted"}
,{"{v":"msg0":",""msg"name"::[1] 1a8122c3-953b-42de-a373-2c666ddd65a5 (7c39838b-aea6-4a95-a4cb-86170299d0c5) Active Faulted Active ds_transition to LiveRepairReady"""crucible","v,Write 1:1:9 past extent under repair 0""":,0",v"level""name"::":crucible"300,","level":name"30:"crucible","level":40,"time":"2023-09-22T23:08:02.549029518Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
,"{time":""msg":"2023-09-22T23:08:02.549026445Z"[1] Transition from Faulted to LiveRepairReady",",v":0","timename",""::hostname""":crucible"","2023-09-22T23:08:02.549033148Z"level"ip-10-150-1-74.us-west-2.compute.internal":,30,""pid":hostname":4291"}
ip-10-150-1-74.us-west-2.compute.internal",",time":""pid":2023-09-22T23:08:02.549064822Z{"4291,"hostname":"}
"ip-10-150-1-74.us-west-2.compute.internal",msg"":pid":"4291{}
"{msg":""msg":"[1] 231d0792-4fa5-43d8-bed8-61976ebe4426 (311603ce-e15d-47ce-bbc6-29c87d03dcc9) Active Faulted Active ds_transition to LiveRepairReady"IO Write 1004 on eur 0 Added deps 1",",v""[1] 1a8122c3-953b-42de-a373-2c666ddd65a5 (7c39838b-aea6-4a95-a4cb-86170299d0c5) Active LiveRepairReady Active ds_transition to LiveRepair":,"v"v0":0:,","0name"name",:""name"::"cruciblecrucible""",crucible"",level","level""::level30"30:40,"time":"2023-09-22T23:08:02.549124336Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal",","time":pid,""":2023-09-22T23:08:02.549125439Z"4291time","}hostname:""
2023-09-22T23:08:02.549126296Z:""{,"ip-10-150-1-74.us-west-2.compute.internal"hostname","msg""::pid""":4291ip-10-150-1-74.us-west-2.compute.internal"[1] Transition from LiveRepairReady to LiveRepair",}"
pid",:"4291v":0},"{name":
""crucible"msg":,""level":30[1] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.549194119Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
,"time":"2023-09-22T23:08:02.549202312Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 231d0792-4fa5-43d8-bed8-61976ebe4426 (311603ce-e15d-47ce-bbc6-29c87d03dcc9) Active LiveRepairReady Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.549252745Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{{"msg":""[1] Transition from LiveRepairReady to LiveRepair"msg",":"v":0,"name":"crucibleCrucible stats registered with UUID: 6220a0c3-d619-4944-bb16-c06d6f7a401f"","level",:"30v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.549296486Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
,"time":"{2023-09-22T23:08:02.549302079Z","hostname":""msg":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
[1] 231d0792-4fa5-43d8-bed8-61976ebe4426 (311603ce-e15d-47ce-bbc6-29c87d03dcc9) Active LiveRepair Active ds_transition to Faulted","v":0,"name"{:"crucible","level":"30msg":"Crucible 6220a0c3-d619-4944-bb16-c06d6f7a401f has session id: e3f8e9de-d6e0-4089-ac3a-675d8ee0da5c","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.549349067Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
,"time":"2023-09-22T23:08:02.549359598Z"{,"hostname":""msg":"ip-10-150-1-74.us-west-2.compute.internal","pid":[1] Transition from LiveRepair to Faulted"4291,"v":}0
,"name":"crucible","{level":30"msg":"[0] 6220a0c3-d619-4944-bb16-c06d6f7a401f (a4be66d9-994e-49cf-8c96-b2599509ea94) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"test live_repair::repair_test::test_live_repair_update ... time":"ok
2023-09-22T23:08:02.54939765Z",",time"":"hostname":"2023-09-22T23:08:02.549410787Z","ip-10-150-1-74.us-west-2.compute.internal"hostname,""pid"::"4291}ip-10-150-1-74.us-west-2.compute.internal"
,"pid":4291}
{{""msg"msg":":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30[1] 231d0792-4fa5-43d8-bed8-61976ebe4426 (311603ce-e15d-47ce-bbc6-29c87d03dcc9) Active Faulted Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.549455454Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 6220a0c3-d619-4944-bb16-c06d6f7a401f (a4be66d9-994e-49cf-8c96-b2599509ea94) WaitActive New New ds_transition to WaitQuorum","v":,0","time"name":"test live_repair::repair_test::test_live_repair_repair_write_push ... :2023-09-22T23:08:02.549463638Z"ok
,""crucible"hostname",:""level":30ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{,"time":""msg"2023-09-22T23:08:02.549498231Z":","hostname":"[1] Transition from Faulted to LiveRepairReady","ip-10-150-1-74.us-west-2.compute.internalv":"0,",pid"":name"4291:"crucible"},
"level":30{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.549531084Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
,"time":"2023-09-22T23:08:02.549542968Z"{,"hostname":""msg":ip-10-150-1-74.us-west-2.compute.internal"","pid":4291}
[1] 231d0792-4fa5-43d8-bed8-61976ebe4426 (311603ce-e15d-47ce-bbc6-29c87d03dcc9) Active LiveRepairReady Active ds_transition to LiveRepair","v":0{,"name":"crucible"",msg"":"level":30[0] 6220a0c3-d619-4944-bb16-c06d6f7a401f (a4be66d9-994e-49cf-8c96-b2599509ea94) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.549583239Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"time"}:"
2023-09-22T23:08:02.549593231Z","hostname":"{ip-10-150-1-74.us-west-2.compute.internal","pid":4291"msg}"
:"[1] Transition from LiveRepairReady to LiveRepair{","v":"0msg":","name":"crucible"[0] Transition from WaitQuorum to Active",",level":"30v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.549644825Z",",time":""hostname":"2023-09-22T23:08:02.549649568Z","ip-10-150-1-74.us-west-2.compute.internal"hostname",:""pid":4291ip-10-150-1-74.us-west-2.compute.internal","}pid
":4291}
{{"msg":""msg":"Write to Extent 0:2:9 under repair","v":0,"name":"crucible","[1] 6220a0c3-d619-4944-bb16-c06d6f7a401f (a4be66d9-994e-49cf-8c96-b2599509ea94) Active New New ds_transition to WaitActive"level",:"40v":0,"name":"crucible","level":30,"time":","time":2023-09-22T23:08:02.549700248Z"","hostname"2023-09-22T23:08:02.549704959Z:"","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid"ip-10-150-1-74.us-west-2.compute.internal":,4291"pid":}4291
}
{{"msg"":"msg":"Write 1:0:9 past extent under repair 0"[1] Transition from New to WaitActive",",v"":v"0:,0","name":name"":"crucible"crucible",","level"level"::4030,"time":","time":"2023-09-22T23:08:02.54975879Z","2023-09-22T23:08:02.54975902Z"hostname":,""hostname":"ip-10-150-1-74.us-west-2.compute.internal","ip-10-150-1-74.us-west-2.compute.internalpid"":,4291"pid":}4291
}
{"{msg":""Write 1:1:9 past extent under repair 0"msg":","v":0,"name":"crucible","level":[1] 6220a0c3-d619-4944-bb16-c06d6f7a401f (a4be66d9-994e-49cf-8c96-b2599509ea94) Active WaitActive New ds_transition to WaitQuorum"40,"v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.549811126Z",","time":hostname"":"2023-09-22T23:08:02.549816109Z"ip-10-150-1-74.us-west-2.compute.internal",,""pid":hostname4291":"}
ip-10-150-1-74.us-west-2.compute.internal","pid":4291{}
"msg":"{IO Write 1004 on eur 0 Added deps 1","v":"0msg",":"name":"crucible","[1] Transition from WaitActive to WaitQuorum"level":,"40v":0,"name":"crucible","level":30,"time":","2023-09-22T23:08:02.549865138Z"time":","hostname":"2023-09-22T23:08:02.549870138Z","hostname"ip-10-150-1-74.us-west-2.compute.internal":,""pid":4291ip-10-150-1-74.us-west-2.compute.internal","}pid
"{:4291}
"msg":{""msg":"Crucible stats registered with UUID: fce41f2d-d98f-4004-81f1-42f5f6597ea7","v":0,"name"[1] 6220a0c3-d619-4944-bb16-c06d6f7a401f (a4be66d9-994e-49cf-8c96-b2599509ea94) Active WaitQuorum New ds_transition to Active":","crucible","level":30{"msg":"Crucible stats registered with UUID: 4e3cf648-2b3b-4d4d-8069-b689dc174a75","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.549927207Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
v":0,"name":{"crucible","level":"30msg":"Crucible fce41f2d-d98f-4004-81f1-42f5f6597ea7 has session id: d91cfece-2dd3-44d1-b808-deff3c07fda2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.549978944Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal",,""pid":time4291":"}2023-09-22T23:08:02.549989699Z"
,"hostname":"ip-10-150-1-74.us-west-2.compute.internal","{pid":4291}
{",msg":""msg"[0] fce41f2d-d98f-4004-81f1-42f5f6597ea7 (76c19d0f-1c30-4091-9c4d-921b4ab00458) New New New ds_transition to WaitActive:"","v":[1] Transition from WaitQuorum to Active"0,",name"":v"":crucible"0,,"level":30"time":"2023-09-22T23:08:02.549949711Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
,"time":"2023-09-22T23:08:02.550066151Z"{,"hostname":""ip-10-150-1-74.us-west-2.compute.internal"msg":,""pid":4291}
{"msg":"[0] Transition from New to WaitActive"Crucible 4e3cf648-2b3b-4d4d-8069-b689dc174a75 has session id: 4f35d8e7-db1f-4b21-be79-84172e32182f,""vtest live_repair::repair_test::test_live_repair_span_write_write ... ",ok":0v",:0,"name":"crucible"",name":""level":crucible"30,"level":"30name":"crucible","level":30
,"time":"2023-09-22T23:08:02.550143778Z",","hostname":time":""2023-09-22T23:08:02.550150389Z"ip-10-150-1-74.us-west-2.compute.internal",",pid"":hostname":"4291}
{"msg":"[0] fce41f2d-d98f-4004-81f1-42f5f6597ea7 (76c19d0f-1c30-4091-9c4d-921b4ab00458) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level,"":time30":"2023-09-22T23:08:02.550141553Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
,"time":"{2023-09-22T23:08:02.550206853Z","hostname"ip-10-150-1-74.us-west-2.compute.internal:"""msg":"ip-10-150-1-74.us-west-2.compute.internal",,""pid":4291pid":}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":42910,"name":"crucible"},
"level"[0] 4e3cf648-2b3b-4d4d-8069-b689dc174a75 (e740b9e2-acbd-4260-8e40-b3b2459d8372) New New New ds_transition to WaitActive:"30{,"v":"msg0":","name":"crucible","level":30[2] 6220a0c3-d619-4944-bb16-c06d6f7a401f (a4be66d9-994e-49cf-8c96-b2599509ea94) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":,30"time":"2023-09-22T23:08:02.550264181Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{,"time":"2023-09-22T23:08:02.55028015Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30"msg":","time":"[0] fce41f2d-d98f-4004-81f1-42f5f6597ea7 (76c19d0f-1c30-4091-9c4d-921b4ab00458) WaitQuorum New New ds_transition to Active"2023-09-22T23:08:02.550326127Z",","v":hostname":0","name":"crucible","level":30ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":","time":","[2] 6220a0c3-d619-4944-bb16-c06d6f7a401f (a4be66d9-994e-49cf-8c96-b2599509ea94) Active Active WaitActive ds_transition to WaitQuorum"2023-09-22T23:08:02.550272985Z","timev,"":":0"2023-09-22T23:08:02.550352094Z",,""name":"hostname":"crucible","level":ip-10-150-1-74.us-west-2.compute.internal"30,"pid":4291}
hostname"{:""ip-10-150-1-74.us-west-2.compute.internal"msg":,,"""time[0] Transition from WaitQuorum to Active"","v":0,"name":"crucible",pid"":level4291":30}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level",":time":"2023-09-22T23:08:02.550414176Z","hostname":":"ip-10-150-1-74.us-west-2.compute.internal"2023-09-22T23:08:02.550386023Z,""pid",:"4291hostname":"}
30{"msg"ip-10-150-1-74.us-west-2.compute.internal:"","pid":4291}
,"[1] fce41f2d-d98f-4004-81f1-42f5f6597ea7 (76c19d0f-1c30-4091-9c4d-921b4ab00458) Active New New ds_transition to WaitActive"time":,""v":{2023-09-22T23:08:02.550456811Z"0,,"msg":"[2] Transition from WaitActive to WaitQuorum"",name""v":"crucible","level":30"hostname":"ip-10-150-1-74.us-west-2.compute.internal",":pid":04291,"}
name":"crucible","level{":30"msg":","time":"2023-09-22T23:08:02.550490171Z","hostname":[0] 4e3cf648-2b3b-4d4d-8069-b689dc174a75 (e740b9e2-acbd-4260-8e40-b3b2459d8372) WaitActive New New ds_transition to WaitQuorum"","v":0,"ip-10-150-1-74.us-west-2.compute.internal"name,"":"pid":crucible"4291,","level"time:"30:}"
,"time":"2023-09-22T23:08:02.550520884Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30{,"time":""2023-09-22T23:08:02.550553282Zmsg","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{{"msg":""msg":"Crucible stats registered with UUID: 278316f9-9926-4cf9-908b-c5ecd8db3aac"[0] 4e3cf648-2b3b-4d4d-8069-b689dc174a75 (e740b9e2-acbd-4260-8e40-b3b2459d8372) WaitQuorum New New ds_transition to Active",,""vv""::00,","name"name:"":"crucible"crucible",","level"level:":3030":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":,,""30timetime""::""2023-09-22T23:08:02.550590671Z"2023-09-22T23:08:02.550590968Z",","hostname"hostname:"":"2023-09-22T23:08:02.550503853Zip-10-150-1-74.us-west-2.compute.internalip-10-150-1-74.us-west-2.compute.internal"",,"""pidpid""::42914291}}

{,,"{""msg"":time"":"[0] Transition from WaitQuorum to Active"2023-09-22T23:08:02.550604179Z","v,":"0,"hostnamename"":"msghostnameip-10-150-1-74.us-west-2.compute.internal"",":pid"""::"4291ip-10-150-1-74.us-west-2.compute.internal"},
{:"msg":""crucible","level"[1] fce41f2d-d98f-4004-81f1-42f5f6597ea7 (76c19d0f-1c30-4091-9c4d-921b4ab00458) Active WaitActive New ds_transition to WaitQuorum"Crucible 278316f9-9926-4cf9-908b-c5ecd8db3aac has session id: 6ec92630-da8a-4479-b77d-86eef1bf4579","v":0,,""v"name:":0:30,"name":"crucible","level":30"crucible","level":30,"time"":"pid",":2023-09-22T23:08:02.550751895Z,"4291",timetime""}:
"{"msg"::""2023-09-22T23:08:02.550761825Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":[2] 6220a0c3-d619-4944-bb16-c06d6f7a401f (a4be66d9-994e-49cf-8c96-b2599509ea94) Active Active WaitQuorum ds_transition to Active"4291}
,"v":0,"name":"2023-09-22T23:08:02.550755541Zcrucible{"","level,""msg":"hostname":"[0] 278316f9-9926-4cf9-908b-c5ecd8db3aac (504852f5-4e78-4bd4-9def-dcf505c09e8e) New New New ds_transition to WaitActive""ip-10-150-1-74.us-west-2.compute.internal",hostname""v:"":,0"ip-10-150-1-74.us-west-2.compute.internal,""pid",name:""30:"crucible","level"::429130}
{,"time":"""2023-09-22T23:08:02.550842181Zpid"",:,4291""msg"}time"
hostname"::""{2023-09-22T23:08:02.550838095Z"",msg"":[1] Transition from WaitActive to WaitQuorumhostname"":"ip-10-150-1-74.us-west-2.compute.internal:"",,""ip-10-150-1-74.us-west-2.compute.internal"v,""pid":4291}
":[1] 4e3cf648-2b3b-4d4d-8069-b689dc174a75 (e740b9e2-acbd-4260-8e40-b3b2459d8372) Active New New ds_transition to WaitActive"0,",{"namev"":"0:msg,"":name""":"pid[0] Transition from New to WaitActivecrucible""crucible","level":,"30level":30":4291,"v"}:
,0"time",:""name":"crucible2023-09-22T23:08:02.550912176Z"",","level":{hostname30":","time":"ip-10-150-1-74.us-west-2.compute.internal"",2023-09-22T23:08:02.550913553Z"msg,"hostname":"ip-10-150-1-74.us-west-2.compute.internal"",:"",pid"":[2] Transition from WaitQuorum to Active4291"}
time":"2023-09-22T23:08:02.550931076Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 278316f9-9926-4cf9-908b-c5ecd8db3aac (504852f5-4e78-4bd4-9def-dcf505c09e8e) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30{"msg",:""v":,0"time"",:"[1] fce41f2d-d98f-4004-81f1-42f5f6597ea7 (76c19d0f-1c30-4091-9c4d-921b4ab00458) Active WaitQuorum New ds_transition to Active"name":"crucible"2023-09-22T23:08:02.550979658Z","",level"":,hostname"30":v""ip-10-150-1-74.us-west-2.compute.internal",":pid":04291,"}name":pid":4291"crucible"}
,"level":30,"{time"msg":"":[1] Transition from New to WaitActive"","v2023-09-22T23:08:02.551008026Z"":0,,"
name"":"hostname"crucible"{,:","levelmsg""::30""[0] Transition from WaitActive to WaitQuorum","v":0time",:""name":"crucible2023-09-22T23:08:02.551027926Z"","level,"":30hostname"",:""time":ip-10-150-1-74.us-west-2.compute.internal""ip-10-150-1-74.us-west-2.compute.internal",",2023-09-22T23:08:02.551050104Z""pid,pid""::42914291}
,"hostname"{:"ip-10-150-1-74.us-west-2.compute.internal""msg},""
:time":"2023-09-22T23:08:02.551061933Z","{hostname":"ip-10-150-1-74.us-west-2.compute.internal",""pid":4291msg":}"
6220a0c3-d619-4944-bb16-c06d6f7a401f is now active with session: a4be66d9-994e-49cf-8c96-b2599509ea94"{,"v":"0msg":","name":"crucible","[0] 278316f9-9926-4cf9-908b-c5ecd8db3aac (504852f5-4e78-4bd4-9def-dcf505c09e8e) WaitQuorum New New ds_transition to Active"level",":v":030,"name":"crucible","level":30"[1] Transition from WaitQuorum to Active----------------------------------------------------------------
,"",time"pid:" Crucible gen:0 GIO:true work queues:  Upstairs:6  downstairs:6
"GW_ID      ACK  DSID    TYPE  BLOCKS  DS:0  DS:1  DS:2  REPLAY
2023-09-22T23:08:02.551129591Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291    1 AckReady  1000   Write       1  Done  Done  Done  false
}
    2 NotAcked  1001    Read       1  Done  Done  Done  false
    3 NotAcked  1002  WriteU       1":4291}
,"{v":0"msg":,""name":"crucible","level":[1] 4e3cf648-2b3b-4d4d-8069-b689dc174a75 (e740b9e2-acbd-4260-8e40-b3b2459d8372) Active WaitActive New ds_transition to WaitQuorum"30,"v":0,"name":"crucible","level":30"time":"2023-09-22T23:08:02.551127669Z","time":,  Done  Done"  Done,2023-09-22T23:08:02.551211437Z"","time"hostname  false
    4 AckReady  1003   Write       1"   Newhostname":"{ip-10-150-1-74.us-west-2.compute.internal"",msg":""pid":[0] Transition from WaitQuorum to Active"4291,"v":0,"}name"
::"crucible"","level":302023-09-22T23:08:02.551208531Z"{  Skip   New,  false"msg":"
",":"time":"2023-09-22T23:08:02.551308397Z""ip-10-150-1-74.us-west-2.compute.internal"    5 NotAcked  1004    Read       1   New  Skip[1] 6220a0c3-d619-4944-bb16-c06d6f7a401f (a4be66d9-994e-49cf-8c96-b2599509ea94) Active Active Active ds_transition to Faulted","hostnamepid",:"4291":"}v"
:ip-10-150-1-74.us-west-2.compute.internal"0,,""{name":pid""":crucible"4291msg":","}level"[1] Transition from WaitActive to WaitQuorum":   New30,  false
,"hostname":    6 NotAcked  1005  WriteU       1   New,
{"msg":"[2] fce41f2d-d98f-4004-81f1-42f5f6597ea7 (76c19d0f-1c30-4091-9c4d-921b4ab00458) Active Active New ds_transition to WaitActive","v":0  Skip,   New"  falsename":
"v":0,   STATES      DS:0   DS:1   DS:2   TOTAL
    New        "crucible"","level":ip-10-150-1-74.us-west-2.compute.internal"30,"pid":4291}
{"msg":""[1] 278316f9-9926-4cf9-908b-c5ecd8db3aac (504852f5-4e78-4bd4-9def-dcf505c09e8e) Active New New ds_transition to WaitActive","v":0time,""name"::","crucible"2023-09-22T23:08:02.551368709Z"   6   ,","level":hostname"30:"   3      6   ip-10-150-1-74.us-west-2.compute.internal"  15
    Sent       "time":","time"2023-09-22T23:08:02.551436779Z":,"hostname"":"2023-09-22T23:08:02.551463329Z",ip-10-150-1-74.us-west-2.compute.internal"","hostname"pid"::"4291}
ip-10-150-1-74.us-west-2.compute.internal","pid":"4291name}"
:"   0   crucible{","level"":msg":30"   0      0      0
    Done          0   {   0      0   ,   0
    Skipped    ,"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30   0      3      0      3
"pid":4291    Error         0      0   [1] Transition from New to WaitActive   0   "   0
,Last Flush: ,"v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.551589278Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 278316f9-9926-4cf9-908b-c5ecd8db3aac (504852f5-4e78-4bd4-9def-dcf505c09e8e) Active WaitActive New ds_transition to WaitQuorum"0 ,0 "time":"0 
Downstairs last five completed:"time":"2023-09-22T23:08:02.551507331Z2023-09-22T23:08:02.55154613Z"","hostname":","ip-10-150-1-74.us-west-2.compute.internal"hostname,}""pid
:":"4291}
{
{"msg":""msg":"[1] Transition from Active to Faulted","v":0,""name"v[1] 4e3cf648-2b3b-4d4d-8069-b689dc174a75 (e740b9e2-acbd-4260-8e40-b3b2459d8372) Active WaitQuorum New ds_transition to Active"":Upstairs last five completed:  ,:"
0,"v":crucible0","name":",crucible"",level""level"::3030"name":"crucible","level":30ip-10-150-1-74.us-west-2.compute.internal","time":,"",time"2023-09-22T23:08:02.551742237Z"":"pid":,2023-09-22T23:08:02.551746509Z"",hostname,"4291""timehostname:"":"}"ip-10-150-1-74.us-west-2.compute.internal"
,ip-10-150-1-74.us-west-2.compute.internal""pid",:":pid4291":"}4291
}{
{"msg":{""msg"":"msg":"[1] Transition from WaitQuorum to Active"[1] Transition from WaitActive to WaitQuorum,""v,"":v"0:[2] fce41f2d-d98f-4004-81f1-42f5f6597ea7 (76c19d0f-1c30-4091-9c4d-921b4ab00458) Active Active WaitActive ds_transition to WaitQuorum"0,",",name""name:"":v""crucible"crucible,"":level,0"":,30level"":name"30:"crucible","level":30,"time":,""time":"2023-09-22T23:08:02.551811011Z","2023-09-22T23:08:02.551813609Z"hostname",:""hostname":"ip-10-150-1-74.us-west-2.compute.internal","ip-10-150-1-74.us-west-2.compute.internalpid"",:"4291pid":}4291
}
{2023-09-22T23:08:02.551742673Z,"""{timemsg"":msg"",""::hostname""":"[2] 4e3cf648-2b3b-4d4d-8069-b689dc174a75 (e740b9e2-acbd-4260-8e40-b3b2459d8372) Active Active New ds_transition to WaitActive"2023-09-22T23:08:02.551817566Z","ip-10-150-1-74.us-west-2.compute.internal,""[1] 278316f9-9926-4cf9-908b-c5ecd8db3aac (504852f5-4e78-4bd4-9def-dcf505c09e8e) Active WaitQuorum New ds_transition to Activehostname",""v,"":v:"0:,0",name""name:""crucible:""pid"":,crucible"4291"level,"":level30":}30ip-10-150-1-74.us-west-2.compute.internal"
,,"",timetime""::"""{pid"2023-09-22T23:08:02.551869234Z2023-09-22T23:08:02.551867573Z"":,,"4291"hostname"":msg""hostname"}:ip-10-150-1-74.us-west-2.compute.internal:"
"","pid"[1] 6220a0c3-d619-4944-bb16-c06d6f7a401f (a4be66d9-994e-49cf-8c96-b2599509ea94) Active Faulted Active ds_transition to LiveRepairReady":ip-10-150-1-74.us-west-2.compute.internal"4291,",pid}"
:{4291{"}v""
msg""::"{0msg","[2] Transition from New to WaitActive"":",msg""v:"":name"0:"[2] Transition from WaitActive to WaitQuorum"crucible",[1] Transition from WaitQuorum to Active",,""","vv""name:"0:,""namecrucible""::level",:""level"crucible:"30,"0level30",:"30name":"crucible","level":30,"time":"2023-09-22T23:08:02.551945492Z","hostname":","time":"ip-10-150-1-74.us-west-2.compute.internal","2023-09-22T23:08:02.551950294Zpid"":,4291"hostname}"
:"{ip-10-150-1-74.us-west-2.compute.internal"",,msg""pid:"":",4291"time"time":}[2] 4e3cf648-2b3b-4d4d-8069-b689dc174a75 (e740b9e2-acbd-4260-8e40-b3b2459d8372) Active Active WaitActive ds_transition to WaitQuorum
"":,{""2023-09-22T23:08:02.551948544Z""v"msg:"0:,,""2023-09-22T23:08:02.551954107Z""hostname"name",::"[2] 278316f9-9926-4cf9-908b-c5ecd8db3aac (504852f5-4e78-4bd4-9def-dcf505c09e8e) Active Active New ds_transition to WaitActive"""crucible,"",v"ip-10-150-1-74.us-west-2.compute.internal"hostname"level:,""::030,"""name":pid"":ip-10-150-1-74.us-west-2.compute.internal"crucible4291",",pid"}",level""
:time:"30:"4291}2023-09-22T23:08:02.552002631Z"{
,",""msgtimehostname"{:"":"":""2023-09-22T23:08:02.552020306Z"ip-10-150-1-74.us-west-2.compute.internal",,""[1] Transition from Faulted to LiveRepairReady"pidhostname""::4291"msg,""}ip-10-150-1-74.us-west-2.compute.internal
":,{""pid""msg:"4291:v"}"
[2] Transition from WaitActive to WaitQuorum"[2] fce41f2d-d98f-4004-81f1-42f5f6597ea7 (76c19d0f-1c30-4091-9c4d-921b4ab00458) Active Active WaitQuorum ds_transition to Active":{,0"",msg""vv""::"0:,,""name[2] Transition from New to WaitActive"":0,,"""vcrucible"":name0,,"""name:"namelevel""::"30crucible"crucible"",":level,""":level"30:,30"crucible"time":","level":2023-09-22T23:08:02.552083104Z"30,"hostname":"ip-10-150-1-74.us-west-2.compute.internal",","pid"time:":4291"}
2023-09-22T23:08:02.552092085Z","{hostname":""msg":"ip-10-150-1-74.us-west-2.compute.internal",,""pid":time"4291:"}[2] 4e3cf648-2b3b-4d4d-8069-b689dc174a75 (e740b9e2-acbd-4260-8e40-b3b2459d8372) Active Active WaitQuorum ds_transition to Active
",,{"""vmsg""::0",time2023-09-22T23:08:02.552091582Z""":,""name":hostname2023-09-22T23:08:02.552096834Z[2] 278316f9-9926-4cf9-908b-c5ecd8db3aac (504852f5-4e78-4bd4-9def-dcf505c09e8e) Active Active WaitActive ds_transition to WaitQuorum"""",crucible"",v"",:"level:"0"ip-10-150-1-74.us-west-2.compute.internal:,30""name"hostname"::"",crucible"",pid"ip-10-150-1-74.us-west-2.compute.internal","leveltime""::"":302023-09-22T23:08:02.552139346Z"4291,,""}hostname
pid","":time""::4291"ip-10-150-1-74.us-west-2.compute.internal"{2023-09-22T23:08:02.55215359Z,""}pid,"":
"hostname4291":}"
ip-10-150-1-74.us-west-2.compute.internal"{msg"{,":"pidmsg""::"4291""msg"[2] Transition from WaitQuorum to Active}":,
""v"[1] 6220a0c3-d619-4944-bb16-c06d6f7a401f (a4be66d9-994e-49cf-8c96-b2599509ea94) Active LiveRepairReady Active ds_transition to LiveRepair":{0[2] Transition from WaitQuorum to Active"",msg"",name:""",v":[2] Transition from WaitActive to WaitQuorum:"",0",vcrucible""v,"":""level0",:name"":name30"0::"",cruciblecrucible"""name,,""level,"":time30"level":""::"302023-09-22T23:08:02.552214305Z"crucible",","hostname"level",:"":time":"30ip-10-150-1-74.us-west-2.compute.internal"2023-09-22T23:08:02.552225558Z,""pid,"":4291hostname":"}
ip-10-150-1-74.us-west-2.compute.internal","pid":{4291}
",msg{"":""time"msg":":"2023-09-22T23:08:02.552230633Z"4e3cf648-2b3b-4d4d-8069-b689dc174a75 is now active with session: e740b9e2-acbd-4260-8e40-b3b2459d8372",,"",v""hostname"[2] 278316f9-9926-4cf9-908b-c5ecd8db3aac (504852f5-4e78-4bd4-9def-dcf505c09e8e) Active Active WaitQuorum ds_transition to Active:"0:time,,"""namev"""::0":crucible,"",ip-10-150-1-74.us-west-2.compute.internal"name"""2023-09-22T23:08:02.552238062Z",:,"level"crucible"":hostname"30,:"""pidlevel"":ip-10-150-1-74.us-west-2.compute.internal:"429130,"},
pid"",:time"4291time""::""}{
"2023-09-22T23:08:02.552280733Z2023-09-22T23:08:02.552288301Z"",msg","":{hostnamehostname""::""""msg":"ip-10-150-1-74.us-west-2.compute.internalip-10-150-1-74.us-west-2.compute.internal"",,""pidpid""::4291fce41f2d-d98f-4004-81f1-42f5f6597ea7 is now active with session: 76c19d0f-1c30-4091-9c4d-921b4ab004584291}"
},
{"[1] Transition from LiveRepairReady to LiveRepair"v""{msg",":msg""":v"::"00,"[2] Transition from WaitQuorum to Active[1] 4e3cf648-2b3b-4d4d-8069-b689dc174a75 (e740b9e2-acbd-4260-8e40-b3b2459d8372) Active Active Active ds_transition to Faulted""name,,"",""namevv""::00,,"""namename"::"":""cruciblecrucible"":,,""level"cruciblecrucible"level""::3030",","level"level"::3030,,""timetime""::""2023-09-22T23:08:02.552369459Z2023-09-22T23:08:02.552369576Z"",,""hostnamehostname""::""ip-10-150-1-74.us-west-2.compute.internalip-10-150-1-74.us-west-2.compute.internal"",,""pidpid""::42914291}}

,{,{""""msgmsg"":time""::""time":"2023-09-22T23:08:02.552374457Z"[1] Transition from Active to Faulted"278316f9-9926-4cf9-908b-c5ecd8db3aac is now active with session: 504852f5-4e78-4bd4-9def-dcf505c09e8e",,""vv""::0,,0",name""name":":hostname"2023-09-22T23:08:02.552374589Z""":,"cruciblecrucible"",,""level"level:"30":hostname":ip-10-150-1-74.us-west-2.compute.internal""30,"pid":ip-10-150-1-74.us-west-2.compute.internal"4291,"pid":}4291
,,""timetime""::""}{
2023-09-22T23:08:02.552423047Z2023-09-22T23:08:02.552426814Z""",,""msg"hostnamehostname""::""{:"ip-10-150-1-74.us-west-2.compute.internalip-10-150-1-74.us-west-2.compute.internal"",,""pid""pid:"4291:msg"4291}:}

{"[1] fce41f2d-d98f-4004-81f1-42f5f6597ea7 (76c19d0f-1c30-4091-9c4d-921b4ab00458) Active Active Active ds_transition to Faulted""{msg",":msg""":"[0] 6220a0c3-d619-4944-bb16-c06d6f7a401f (a4be66d9-994e-49cf-8c96-b2599509ea94) Active LiveRepair Active ds_transition to Faulted"v":,"0v",:"0[1] 278316f9-9926-4cf9-908b-c5ecd8db3aac (504852f5-4e78-4bd4-9def-dcf505c09e8e) Active Active Active ds_transition to Faulted[1] 4e3cf648-2b3b-4d4d-8069-b689dc174a75 (e740b9e2-acbd-4260-8e40-b3b2459d8372) Active Faulted Active ds_transition to LiveRepairReady"",,,"""vvname""::00"name"::,,""name"name:"":""cruciblecrucible"crucible","",level"crucible"":"30,,""level"level:"30:level"30:30,,""timetime""::"","2023-09-22T23:08:02.552503334Z2023-09-22T23:08:02.552502151Z""time,,"",hostnamehostname"":"":"time":ip-10-150-1-74.us-west-2.compute.internal"":,"""pidip-10-150-1-74.us-west-2.compute.internal"":,4291"2023-09-22T23:08:02.552504549Z}pid
"2023-09-22T23:08:02.552499948Z""{:,4291""}msghostname","
::""{"hostname":""ip-10-150-1-74.us-west-2.compute.internalmsg[1] Transition from Active to Faulted""ip-10-150-1-74.us-west-2.compute.internal":,"",v""[1] Transition from Faulted to LiveRepairReady:"0,pid",v""name:"0:,"""",:namecrucible"",:"""levelcrucible"":,30"4291level"pid":}:
4291,30"time":}"{
"2023-09-22T23:08:02.55257189Z"msg,,"""timehostname""::"":{"2023-09-22T23:08:02.552581626Zip-10-150-1-74.us-west-2.compute.internal"",,""pid"hostname:"4291:"[1] Transition from Active to Faulted""}ip-10-150-1-74.us-west-2.compute.internal
"msg,{",":""pidmsg""::4291"[0] Transition from Active to Faulted"}"
,"vv""{[1] 278316f9-9926-4cf9-908b-c5ecd8db3aac (504852f5-4e78-4bd4-9def-dcf505c09e8e) Active Faulted Active ds_transition to LiveRepairReady":,":msg""00:v"",:"0name",:,"[1] 4e3cf648-2b3b-4d4d-8069-b689dc174a75 (e740b9e2-acbd-4260-8e40-b3b2459d8372) Active LiveRepairReady Active ds_transition to LiveRepair""name",:""v"crucible:"0,,""name"cruciblelevel""name"::"30crucible"",:",level"":level"30:"30crucible",",time"":level"":302023-09-22T23:08:02.552643459Z",,""time"hostname:"":"2023-09-22T23:08:02.552650352Z",ip-10-150-1-74.us-west-2.compute.internal"",hostname"":pid"":4291}
ip-10-150-1-74.us-west-2.compute.internal,,"{"","timemsg""pid"time":"":::2023-09-22T23:08:02.552657773Z"4291""},
[1] Transition from Faulted to LiveRepairReady""2023-09-22T23:08:02.552652134Zhostname,{""v""msg:"0:,""",:name[1] Transition from LiveRepairReady to LiveRepair"":,"""cruciblev"",:"0"hostname,level""":ip-10-150-1-74.us-west-2.compute.internal:name""30:""ip-10-150-1-74.us-west-2.compute.internal",crucible,"",pid"""level"pid:",:"304291time":}:4291
"},
2023-09-22T23:08:02.552713054Z""time",:""{hostname":"2023-09-22T23:08:02.552724386Z"{,"ip-10-150-1-74.us-west-2.compute.internalhostname"",:""pid"""msg:ip-10-150-1-74.us-west-2.compute.internal4291""msg",}"
pid:"{::"4291""msg}"
:"{[2] 6220a0c3-d619-4944-bb16-c06d6f7a401f (a4be66d9-994e-49cf-8c96-b2599509ea94) Faulted LiveRepair Active ds_transition to Faulted""[1] fce41f2d-d98f-4004-81f1-42f5f6597ea7 (76c19d0f-1c30-4091-9c4d-921b4ab00458) Active Faulted Active ds_transition to LiveRepairReady"msg",[1] 278316f9-9926-4cf9-908b-c5ecd8db3aac (504852f5-4e78-4bd4-9def-dcf505c09e8e) Active LiveRepairReady Active ds_transition to LiveRepair,:"",""Write to Extent 0:0:9 under repair""v",:v"0",v"":name:"0:,""v"crucible0name:"",:,""level"crucible:"30,0",level"":name"40:"crucible","level":30,"time":"2023-09-22T23:08:02.552788191Z","hostname",:""time":"ip-10-150-1-74.us-west-2.compute.internal"2023-09-22T23:08:02.552792721Z,"",pid"":hostname4291":"}
ip-10-150-1-74.us-west-2.compute.internal""{,"pid"":,"msg4291"name"time":::}"
"[1] Transition from LiveRepairReady to LiveRepair"{"crucible",""msgv""::0",,2023-09-22T23:08:02.552796235Z""Write to Extent 0:0:9 under repairname"",:""v"crucible:,0",,"""levelname"""::30"level"hostname"crucible":,:""30level":40ip-10-150-1-74.us-west-2.compute.internal","pid,"":time":4291"}
,2023-09-22T23:08:02.552840179Z""time",:""hostname":"2023-09-22T23:08:02.552847131Z"{,"ip-10-150-1-74.us-west-2.compute.internal"hostname",:""pid":"4291msg",}ip-10-150-1-74.us-west-2.compute.internal
"","timepid{":":"4291"}msg
":[1] Transition from Faulted to LiveRepairReady{:"""msg""2023-09-22T23:08:02.552846792Z",[1] 0 final dependency list []:""",v",:[1] client skip 3 in process jobs because fault"""v,""hostname"0:v:"0:,0",name"",:name""""crucible:"name","crucible:ip-10-150-1-74.us-west-2.compute.internal"",""crucible""level,,"pid"level"""level:"30::30:429130}
,",time"":time"":"2023-09-22T23:08:02.55291739Z"2023-09-22T23:08:02.552918617Z,""{hostname,"":"hostname":"ip-10-150-1-74.us-west-2.compute.internal""ip-10-150-1-74.us-west-2.compute.internalmsg","",pid"":pid4291":",:,4291""[2] Transition from Active to Faulted"time",:"",:""":"v"downstairs""downstairs":}}

{02023-09-22T23:08:02.552920144Z",",""name":msg{"hostname":"msg""":[1] job 1000 middle: AckReady""crucible,:[1] 0 final dependency list []""v",:"0v,"":""0name,"":,ip-10-150-1-74.us-west-2.compute.internal"name""level"":crucible"",crucible,"""pid:",level""level::"4291:303030}
{,,"""timetime""::""msg":"2023-09-22T23:08:02.552998024Z2023-09-22T23:08:02.552997908Z"",,""hostnamehostname""::"","time"[1] fce41f2d-d98f-4004-81f1-42f5f6597ea7 (76c19d0f-1c30-4091-9c4d-921b4ab00458) Active LiveRepairReady Active ds_transition to LiveRepair:ip-10-150-1-74.us-west-2.compute.internalip-10-150-1-74.us-west-2.compute.internal"",,""pid"pid:"4291:"4291,",""2023-09-22T23:08:02.5529991Z""::"",""downstairsdownstairs"",}}hostname"":

{"v":"{0ip-10-150-1-74.us-west-2.compute.internal","msg"msg:":""",name"":[1] changed 3 jobs to fault skippedpid"""[1] 0 final dependency list []",,""vv""::00,,""crucible":namename""::""4291,cruciblecrucible"","},""level"level:"30:level"
:3030,"time"{:"2023-09-22T23:08:02.553073971Z",""hostname":","msg"time"::ip-10-150-1-74.us-west-2.compute.internal""","2023-09-22T23:08:02.55308023Z"pid":,"4291hostname",:"""Write to Extent 0:0:9 under repair",ip-10-150-1-74.us-west-2.compute.internal,:"",""downstairspid"":"}time"
4291:v",{":"""msg:""2023-09-22T23:08:02.553080944Z"downstairs:""}0
,,""name"[1] 4e3cf648-2b3b-4d4d-8069-b689dc174a75 (e740b9e2-acbd-4260-8e40-b3b2459d8372) Active LiveRepair Active ds_transition to Faulted":,hostname"""crucible":v":,""0level,"ip-10-150-1-74.us-west-2.compute.internal""name",:":"crucible"40pid",:"4291level":30}
,"{time":","2023-09-22T23:08:02.553143692Z"time,"""msg"::hostname""":"2023-09-22T23:08:02.553140658Z",[1] Transition from LiveRepairReady to LiveRepair"ip-10-150-1-74.us-west-2.compute.internal"",hostname""pid"::"4291,"v":ip-10-150-1-74.us-west-2.compute.internal"}0,,
""pid"name"::4291{"crucible"}",
"msg"level":":30{[1] Transition from LiveRepair to Faulted","v":0","msg"name:":""crucible","levelWrite to Extent 0:0:9 under repair"":,"30v":0,"name":"crucible","level":,"40time":"2023-09-22T23:08:02.55319907Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"time"}:"
2023-09-22T23:08:02.553210278Z","hostname",:""{time":"ip-10-150-1-74.us-west-2.compute.internal","2023-09-22T23:08:02.553216017Z""pid":,msg""4291:hostname"}:
""Write to Extent 0:0:9 under repair"ip-10-150-1-74.us-west-2.compute.internal"{,,""pid":"4291v"msg"}::"
Extent 0 Aborting repair"0,",v":"{name"0:,"""name"msg":":crucible"",crucible""[1] client skip 3 in process jobs because fault,level""level""::,4040"v":0,"name":"crucible","level":30,"time":","2023-09-22T23:08:02.553282532Z"time":","hostname":"2023-09-22T23:08:02.553282931Z",",time":ip-10-150-1-74.us-west-2.compute.internal""",hostname"":"2023-09-22T23:08:02.5532883Z"pid":4291ip-10-150-1-74.us-west-2.compute.internal","},
hostname":""pid":4291{ip-10-150-1-74.us-west-2.compute.internal","}pid""
msg"::"4291,Extent 0 Create and send noop jobs""{,"":v""":downstairs0",msg}"
:""name":"Write to Extent 0:0:9 under repair"crucible"{,,""v""levelmsg":"0:,:40""name":"[1] job 1000 middle: AckReady"crucible",,""v"level:":040,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.553362379Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal",",pid":"4291time":"}
,"2023-09-22T23:08:02.553369719Z"time":","{test live_repair::repair_test::test_repair_dep_cleanup_done ... 2023-09-22T23:08:02.553373652Z"hostname":,"""msg"hostname"ip-10-150-1-74.us-west-2.compute.internal"::ok""ip-10-150-1-74.us-west-2.compute.internalAbort repair on extent 0 starting with id 1003 deps:[JobId(1002), JobId(1001), JobId(1000)]",
,"v""":,pid0",pid"":"4291name":":crucible,""":","4291downstairs"level":}}

{40{"msg":""[1] notify = true for 1001"msg":","v":0,[1] client skip 3 in process jobs because fault,"""time":,"name":v""crucible:"0",,""2023-09-22T23:08:02.55344461Zlevel"":,"name30hostname":":""crucible","level":ip-10-150-1-74.us-west-2.compute.internal"30,"pid":4291}
,"time":"2023-09-22T23:08:02.553468431Z",","time"hostname:"":"2023-09-22T23:08:02.553474358Z"ip-10-150-1-74.us-west-2.compute.internal,"","hostname"pid:"":4291,"ip-10-150-1-74.us-west-2.compute.internal"":,""pid"downstairs:"4291}
,"":"downstairs"}
{"{msg":"[1] notify = true for 1002"",msg"":v"":0[1] job 1000 middle: AckReady",",name"":v"":crucible"0,,""name"level"::"30crucible","level":30,,""time":"time":"2023-09-22T23:08:02.55354077Z"2023-09-22T23:08:02.553538593Z","hostname",:""hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid"ip-10-150-1-74.us-west-2.compute.internal:"4291,"pid,":"4291":","downstairs"":"}
downstairs"}
{"{msg":""[1] changed 3 jobs to fault skipped"msg":","v":0[1] changed 3 jobs to fault skipped",","namev""::"0crucible",","namelevel""::30"crucible","level":30,,""time":time":""2023-09-22T23:08:02.55360386Z"2023-09-22T23:08:02.553601089Z",","hostname":hostname"":"ip-10-150-1-74.us-west-2.compute.internal"ip-10-150-1-74.us-west-2.compute.internal",",pid""pid"::42914291,,""":"":"downstairs"downstairs"}
}
{"msg":"{"msg":"[1] 6220a0c3-d619-4944-bb16-c06d6f7a401f (a4be66d9-994e-49cf-8c96-b2599509ea94) Faulted LiveRepair Faulted ds_transition to Faulted","v":0,"name":"crucible","[1] fce41f2d-d98f-4004-81f1-42f5f6597ea7 (76c19d0f-1c30-4091-9c4d-921b4ab00458) Active LiveRepair Active ds_transition to Faulted"level":,"30v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.553663312Z","hostname":","time":"ip-10-150-1-74.us-west-2.compute.internal","pid"2023-09-22T23:08:02.553668636Z":4291,"hostname"}:
"ip-10-150-1-74.us-west-2.compute.internal","pid"{:4291}
"msg":"{[1] Transition from LiveRepair to Faulted","v":"0msg",:""name":"crucible"[1] Transition from LiveRepair to Faulted",","level":v":300,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.553724737Z,"","time":hostname"":"2023-09-22T23:08:02.553728729Z"ip-10-150-1-74.us-west-2.compute.internal,"","hostname":pid"":4291}ip-10-150-1-74.us-west-2.compute.internal"
,"pid":4291}{
test live_repair::repair_test::test_repair_abort_reserved_jobs ... "msgok{
"":"msg":"Extent 0 Aborting repair"Extent 0 Aborting repair",",v""v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:02.553791812Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible stats registered with UUID: b0e3c5c3-cd28-4727-a05c-bad7123639b4","v"::0,0"name":,""crucible"name",":"level":crucible"30,"level":40,"time":","time"2023-09-22T23:08:02.553869769Z":","hostname2023-09-22T23:08:02.553871771Z"":","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid"ip-10-150-1-74.us-west-2.compute.internal":,"4291pid":4291}
}
{"msg":"{"msg":"Crucible b0e3c5c3-cd28-4727-a05c-bad7123639b4 has session id: 3db7f790-4d53-4e7a-ade8-e826a5d38e48","v":0Abort repair on extent 0: All downstairs are Faulted",",name"":v"":crucible"0,,""name":level"":crucible"30,"level":50,"time":"2023-09-22T23:08:02.553936916Z",,""time":"hostname":"2023-09-22T23:08:02.553938703Z","ip-10-150-1-74.us-west-2.compute.internal"hostname,"":"pid":4291ip-10-150-1-74.us-west-2.compute.internal}"
,"pid":4291{}
"msg":"[0] b0e3c5c3-cd28-4727-a05c-bad7123639b4 (3c586b7d-7750-46c5-8dd5-c7e1fc78978a) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.553988783Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30test live_repair::repair_test::test_repair_abort_basic ... ok
,"time":"2023-09-22T23:08:02.554030144Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] b0e3c5c3-cd28-4727-a05c-bad7123639b4 (3c586b7d-7750-46c5-8dd5-c7e1fc78978a) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.554074139Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.554114264Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] b0e3c5c3-cd28-4727-a05c-bad7123639b4 (3c586b7d-7750-46c5-8dd5-c7e1fc78978a) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.554156174Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.55419623Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"test live_repair::repair_test::test_repair_abort_all_failed_reserved_jobs ... [1] b0e3c5c3-cd28-4727-a05c-bad7123639b4 (3c586b7d-7750-46c5-8dd5-c7e1fc78978a) Active New New ds_transition to WaitActiveok"
,"v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.554245758Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level{":30"msg":"Crucible stats registered with UUID: a67c3619-5a95-4827-9e5d-6fcdacafd685","v":0,"name":"crucible","level":,"30time":"2023-09-22T23:08:02.55428634Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":","time":"2023-09-22T23:08:02.554302254Z"[1] b0e3c5c3-cd28-4727-a05c-bad7123639b4 (3c586b7d-7750-46c5-8dd5-c7e1fc78978a) Active WaitActive New ds_transition to WaitQuorum",","hostname"v"::"0,"name":"ip-10-150-1-74.us-west-2.compute.internal"crucible",",pid"":level":429130}
{"msg":","time":"Crucible a67c3619-5a95-4827-9e5d-6fcdacafd685 has session id: ea4e2b43-abd0-4b45-9a7a-b02971269ea52023-09-22T23:08:02.554336473Z"",",v":"0hostname,"":"name":"crucible","ip-10-150-1-74.us-west-2.compute.internal"level":,"30pid":4291}
{,"time":""msg":"2023-09-22T23:08:02.554365936Z","[1] Transition from WaitActive to WaitQuorumhostname":"","v":ip-10-150-1-74.us-west-2.compute.internal"0,","pidname""::4291"crucible"},
"level":30{"msg":"[0] a67c3619-5a95-4827-9e5d-6fcdacafd685 (29dc6a8c-0001-4949-b734-4fa4c3de49de) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.554404672Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
,"time":{"2023-09-22T23:08:02.554421125Z"","msg"hostname":":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
[1] b0e3c5c3-cd28-4727-a05c-bad7123639b4 (3c586b7d-7750-46c5-8dd5-c7e1fc78978a) Active WaitQuorum New ds_transition to Active","v":0,"{name":"crucible","level"":msg"30:"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.554459478Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal",",pid"":time4291":"}
2023-09-22T23:08:02.554467216Z","hostname":"{ip-10-150-1-74.us-west-2.compute.internal","pid"":msg"4291:"}
[1] Transition from WaitQuorum to Active","v":0,"{name":"crucible",""levelmsg"":":30[0] a67c3619-5a95-4827-9e5d-6fcdacafd685 (29dc6a8c-0001-4949-b734-4fa4c3de49de) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.554511435Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291},
"time":"{2023-09-22T23:08:02.554521353Z",""hostname"msg"::""ip-10-150-1-74.us-west-2.compute.internal","pid":4291}[2] b0e3c5c3-cd28-4727-a05c-bad7123639b4 (3c586b7d-7750-46c5-8dd5-c7e1fc78978a) Active Active New ds_transition to WaitActive"
,"v":0,"name":"{crucible","level":30"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.554563299Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
,"time":"{2023-09-22T23:08:02.554572026Z","hostname"":msg"":"[2] Transition from New to WaitActiveip-10-150-1-74.us-west-2.compute.internal"",",pid"":v4291":0},
"name":"crucible","level{":30"msg":"{"msg"[0] a67c3619-5a95-4827-9e5d-6fcdacafd685 (29dc6a8c-0001-4949-b734-4fa4c3de49de) WaitQuorum New New ds_transition to Active":","v":0,"name":"Crucible stats registered with UUID: f238b930-a5e0-4425-a639-165303326233crucible"","level,,""time""v":0,"name":"crucible","level":30:"2023-09-22T23:08:02.554613521Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":","time":"2023-09-22T23:08:02.554635552Z","hostname":"[2] b0e3c5c3-cd28-4727-a05c-bad7123639b4 (3c586b7d-7750-46c5-8dd5-c7e1fc78978a) Active Active WaitActive ds_transition to WaitQuorum","v"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
:{30"msg":"Crucible f238b930-a5e0-4425-a639-165303326233 has session id: ea6a7783-cfdd-4381-839d-ebe9eec602ed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.554673341Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,}"
time":"{2023-09-22T23:08:02.554685028Z",""hostname"msg:"":"ip-10-150-1-74.us-west-2.compute.internal"[0] Transition from WaitQuorum to Active",","pid"v:":42910,"}
{"msg":":0,"name":"crucible","level"[0] f238b930-a5e0-4425-a639-165303326233 (45eb49ee-4fdc-4ffa-8697-e0375c977186) New New New ds_transition to WaitActive","v":0,"name":"crucible","namelevel""::"30crucible","level":30,,""timetime""::""2023-09-22T23:08:02.554729383Z2023-09-22T23:08:02.554727364Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{:30"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.554761074Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal,"","time"pid:"":4291"}2023-09-22T23:08:02.554768817Z"
,,""{hostname":"hostname""msg"ip-10-150-1-74.us-west-2.compute.internal":"ip-10-150-1-74.us-west-2.compute.internal",",pid""pid"::42914291}
:"{[2] Transition from WaitActive to WaitQuorum"","msg"v:"":0,"name":"crucible","level":30[0] f238b930-a5e0-4425-a639-165303326233 (45eb49ee-4fdc-4ffa-8697-e0375c977186) WaitActive New New ds_transition to WaitQuorum","}v":0
{,"time":"2023-09-22T23:08:02.554811544Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] b0e3c5c3-cd28-4727-a05c-bad7123639b4 (3c586b7d-7750-46c5-8dd5-c7e1fc78978a) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30"msg":"[1] a67c3619-5a95-4827-9e5d-6fcdacafd685 (29dc6a8c-0001-4949-b734-4fa4c3de49de) Active New New ds_transition to WaitActive","v":0,,""name":name,"":"time"crucible"":crucible","","level"2023-09-22T23:08:02.554905557Zlevel":"30,:"30hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{,"time"":msg"":"2023-09-22T23:08:02.554929777Z",[2] Transition from WaitQuorum to Active""hostname,"":"v":0,"name"ip-10-150-1-74.us-west-2.compute.internal:"","cruciblepid"":,"4291level":}30
,"time":"{2023-09-22T23:08:02.55493174Z"",msg"":"hostname":","time"[0] Transition from WaitActive to WaitQuorum:""ip-10-150-1-74.us-west-2.compute.internal",2023-09-22T23:08:02.554951994Z""v":,0",,""hostnamename""::""pid":crucible"ip-10-150-1-74.us-west-2.compute.internal,"",4291level"":pid"30:4291}
}
,"time":{{""2023-09-22T23:08:02.554975317Zmsg"":",""hostname"msg":":"b0e3c5c3-cd28-4727-a05c-bad7123639b4 is now active with session: 3c586b7d-7750-46c5-8dd5-c7e1fc78978a","ip-10-150-1-74.us-west-2.compute.internal[1] Transition from New to WaitActive""v",,""pid:"0:v"4291,":name}"0:
","crucible"{name":",""crucible"msglevel""::30","level":30[0] f238b930-a5e0-4425-a639-165303326233 (45eb49ee-4fdc-4ffa-8697-e0375c977186) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible",","time"level:"":302023-09-22T23:08:02.555009776Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
,","time"{:time"""msg"::2023-09-22T23:08:02.555020899Z""","hostname":2023-09-22T23:08:02.555012098Z"","hostname":"ip-10-150-1-74.us-west-2.compute.internal[1] b0e3c5c3-cd28-4727-a05c-bad7123639b4 (3c586b7d-7750-46c5-8dd5-c7e1fc78978a) Active Active Active ds_transition to Faulted"",","pid"v:"4291:ip-10-150-1-74.us-west-2.compute.internal"}0,,
""name"pid":{":crucible""4291,msg"":level""}:
[0] Transition from WaitQuorum to Active30","v":0,"name":"{crucible","level":30"msg":","time":"2023-09-22T23:08:02.555059959Z","[1] a67c3619-5a95-4827-9e5d-6fcdacafd685 (29dc6a8c-0001-4949-b734-4fa4c3de49de) Active WaitActive New ds_transition to WaitQuorum"hostname":",","time"v:ip-10-150-1-74.us-west-2.compute.internal""",2023-09-22T23:08:02.555065316Z"":,pid"":hostname4291"0:}"
ip-10-150-1-74.us-west-2.compute.internal",,"{"name"pid""::msg4291"":}"
[1] Transition from Active to Faultedcrucible""{,,"""level"msgv""::"0:,"30name":"crucible","level":30[1] f238b930-a5e0-4425-a639-165303326233 (45eb49ee-4fdc-4ffa-8697-e0375c977186) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.555106996Z",","time,hostname""::"""time":2023-09-22T23:08:02.555111999Z"ip-10-150-1-74.us-west-2.compute.internal"",,""2023-09-22T23:08:02.555104233Z"pidhostname""::4291",}"
ip-10-150-1-74.us-west-2.compute.internal"hostname"{,:"""pidmsg""::"4291ip-10-150-1-74.us-west-2.compute.internal"},
"pid":{[1] b0e3c5c3-cd28-4727-a05c-bad7123639b4 (3c586b7d-7750-46c5-8dd5-c7e1fc78978a) Active Faulted Active ds_transition to LiveRepairReady4291""msg",:""v}"
[1] Transition from New to WaitActive:"0,,""v":name"0:","cruciblename"":,""{levelcrucible"":,30"level":30"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible",,""time"time,"::"""level":2023-09-22T23:08:02.555156146Z2023-09-22T23:08:02.555157254Z""30,,""hostnamehostname""::""ip-10-150-1-74.us-west-2.compute.internalip-10-150-1-74.us-west-2.compute.internal"",,""pidpid""::42914291}}

{"{msg":""msg":"[1] Transition from Faulted to LiveRepairReady","v":,0","time":[1] f238b930-a5e0-4425-a639-165303326233 (45eb49ee-4fdc-4ffa-8697-e0375c977186) Active WaitActive New ds_transition to WaitQuorumname"":","crucible""v",:"02023-09-22T23:08:02.555172631Zlevel,"":"30name":","crucible"hostname",:""level":30ip-10-150-1-74.us-west-2.compute.internal","pid":4291},
,""timetime""::""2023-09-22T23:08:02.555198398Z2023-09-22T23:08:02.555203362Z""{,,""hostname"hostname:"":""msg"ip-10-150-1-74.us-west-2.compute.internalip-10-150-1-74.us-west-2.compute.internal"":,,""pidpid":"4291:"}4291
}
{[1] a67c3619-5a95-4827-9e5d-6fcdacafd685 (29dc6a8c-0001-4949-b734-4fa4c3de49de) Active WaitQuorum New ds_transition to Active""{,msg""":msg"":"v":0,[1] Transition from WaitActive to WaitQuorum"",name""v"::[1] b0e3c5c3-cd28-4727-a05c-bad7123639b4 (3c586b7d-7750-46c5-8dd5-c7e1fc78978a) Active LiveRepairReady Active ds_transition to LiveRepair0",",""crucible"namev""::"0,crucible,"",""namelevel"level":"::"3030crucible","level":30,"time":","time2023-09-22T23:08:02.55526151Z"":","hostname":2023-09-22T23:08:02.555263922Z"","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291ip-10-150-1-74.us-west-2.compute.internal",}"
pid",:{"4291"time"msg}"
:":"{2023-09-22T23:08:02.555262513Z""msg":","hostname":"[1] f238b930-a5e0-4425-a639-165303326233 (45eb49ee-4fdc-4ffa-8697-e0375c977186) Active WaitQuorum New ds_transition to Active"[1] Transition from LiveRepairReady to LiveRepair",","vv""::00,,""ip-10-150-1-74.us-west-2.compute.internal"namename""::"",cruciblecrucible"",,"""levellevel""::3030pid":4291}
,,""timetime""::""{2023-09-22T23:08:02.555309599Z2023-09-22T23:08:02.555309376Z"",,"""hostnamehostname""::""msg":"ip-10-150-1-74.us-west-2.compute.internalip-10-150-1-74.us-west-2.compute.internal"",",pid""pid:"4291:[1] Transition from WaitQuorum to Active}"
4291,"v}{
""msg:{":0"",msg"":[1] Transition from WaitQuorum to Activename""":","Write to Extent 0:1:9 under repairv""crucible":,",v0","":level"name:"030,:""name"crucible:"","crucible"level",:"30level":40,,""timetime""::""2023-09-22T23:08:02.555360428Z2023-09-22T23:08:02.555359687Z"",,""hostnamehostname""::"","timeip-10-150-1-74.us-west-2.compute.internalip-10-150-1-74.us-west-2.compute.internal""",,""pidpid""::42914291:"}}

{2023-09-22T23:08:02.555356841Z"{{,"hostname":"ip-10-150-1-74.us-west-2.compute.internal",""pid"msg""msg":"::"4291}
Crucible stats registered with UUID: 47522d2c-b387-4d7d-a896-16a922aa39ba{","v":0,"name":"crucible","level":30[2] f238b930-a5e0-4425-a639-165303326233 (45eb49ee-4fdc-4ffa-8697-e0375c977186) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time,":"""time2023-09-22T23:08:02.555432394Zmsg""":,"":"hostname":"2023-09-22T23:08:02.555423295Z"ip-10-150-1-74.us-west-2.compute.internal,"hostname"[2] a67c3619-5a95-4827-9e5d-6fcdacafd685 (29dc6a8c-0001-4949-b734-4fa4c3de49de) Active Active New ds_transition to WaitActive":","v":ip-10-150-1-74.us-west-2.compute.internal"0,","pid"name":4291}
{:"crucible""msg",":"level":"30,"pid":4291Crucible 47522d2c-b387-4d7d-a896-16a922aa39ba has session id: 05d7f4c3-b11e-4eed-a687-3a04e8a5e796"},"
v":0,"name":"crucible"{,"level":30"msg":"[2] Transition from New to WaitActive","v":0,"name":",""crucible"time,msg""level"::"30":"Write to Extent 0:1:9 under repair"2023-09-22T23:08:02.555489878Z",",v":"0,,""time":"2023-09-22T23:08:02.555503876Z","hostname":","ip-10-150-1-74.us-west-2.compute.internal"time",:""pid":42912023-09-22T23:08:02.555515227Z","}hostname
{"msg":"":"ip-10-150-1-74.us-west-2.compute.internal"name",:""[0] 47522d2c-b387-4d7d-a896-16a922aa39ba (704620be-2e8a-473a-8f41-8d5ed8675af2) New New New ds_transition to WaitActive"pidcrucible"",:,""levelv":0,"name":"":crucible"40,"level":304291}
{,"time":","2023-09-22T23:08:02.555562656Ztime":"2023-09-22T23:08:02.555564084Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid"hostname:"4291:"}
{"msg":"[0] Transition from New to WaitActiveip-10-150-1-74.us-west-2.compute.internal"","v",":"0,,pid""hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Write to Extent 0:2:9 under repair","v":0,"name":"crucible","level":40"msg":"[2] f238b930-a5e0-4425-a639-165303326233 (45eb49ee-4fdc-4ffa-8697-e0375c977186) Active Active WaitActive ds_transition to WaitQuorum","v":,0","time"name:"":name"2023-09-22T23:08:02.5556352Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":""Write 1:0:9 past extent under repair 0":,"v4291":0,"name"}:
":"crucible"crucible"{""crucible"msg":","level":[2] Transition from New to WaitActive"30,"v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.55569827Z",","hostname":"level":30ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
,{""msg":"time":"[2] Transition from WaitActive to WaitQuorum"2023-09-22T23:08:02.555704878Z","v":,0,,""name"level:"":,40crucible"",time":"2023-09-22T23:08:02.55571875Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":","time":""level2023-09-22T23:08:02.555741178Z"":30[0] 47522d2c-b387-4d7d-a896-16a922aa39ba (704620be-2e8a-473a-8f41-8d5ed8675af2) WaitActive New New ds_transition to WaitQuorum",","hostname"v":":0,"nameip-10-150-1-74.us-west-2.compute.internal"":""crucible",","timelevel"":,30"pid":4291}
{"msg":"IO Write 1013 on eur 0 Added deps 1","v":,0","time"name:":2023-09-22T23:08:02.55579352Z"hostname",:""hostname":""ip-10-150-1-74.us-west-2.compute.internal",ip-10-150-1-74.us-west-2.compute.internal"2023-09-22T23:08:02.555773116Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}"
:"crucible","level":40{"msg":"[2] f238b930-a5e0-4425-a639-165303326233 (45eb49ee-4fdc-4ffa-8697-e0375c977186) Active Active WaitQuorum ds_transition to Active",",pid"":v4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":""crucible"pid",":level"4291:30}
{":0","msg":name":""crucible","level":30,"[2] a67c3619-5a95-4827-9e5d-6fcdacafd685 (29dc6a8c-0001-4949-b734-4fa4c3de49de) Active Active WaitActive ds_transition to WaitQuorumtime"":","v":02023-09-22T23:08:02.555884256Z","name,",:"""hostname"time":"2023-09-22T23:08:02.555844646Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid,"":time4291":"}
2023-09-22T23:08:02.555900152Z","{hostname":""msg":ip-10-150-1-74.us-west-2.compute.internal"","pid":[1] 1007 final dependency list []"4291,"v":0},
":name"":"crucible","ip-10-150-1-74.us-west-2.compute.internal{""msg":"crucible[2] Transition from WaitQuorum to Active"",,""level"v"::300,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.555959165Z",","pid"hostname"::"4291,"timeip-10-150-1-74.us-west-2.compute.internal"},
{"msg":"[0] 47522d2c-b387-4d7d-a896-16a922aa39ba (704620be-2e8a-473a-8f41-8d5ed8675af2) WaitQuorum New New ds_transition to Active","v":level0",:"30name":"crucible","level":30"pid":4291}
,"time":"2023-09-22T23:08:02.556004207Z"{",,:""time":"2023-09-22T23:08:02.556007587Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291"}
{hostname""msg":":"ip-10-150-1-74.us-west-2.compute.internal"[0] Transition from WaitQuorum to Active",","pid"v"2023-09-22T23:08:02.555963769Z",":hostname0:4291,"":"downstairs"}
{"msg":"[1] 1008 final dependency list [JobId(1004)]","v":0,"name":"crucible","level":30","msg"name:","time":"2023-09-22T23:08:02.556092635Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":""[1] 1013 final dependency list [JobId(1008), JobId(1004), JobId(1012)]":,"":v":"0ip-10-150-1-74.us-west-2.compute.internal,"crucible",""name",pid"":4291}
{""msg":[2] Transition from WaitActive to WaitQuorum"","v":0,"name":"cruciblef238b930-a5e0-4425-a639-165303326233 is now active with session: 45eb49ee-4fdc-4ffa-8697-e0375c977186"",","level"v:"30:level"0:,"30name":"crucible","level":30:"crucible",","level"time:":30"2023-09-22T23:08:02.556166399Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal",,""timepid""::4291,"}"
time,"2023-09-22T23:08:02.55616924Z{:"""2023-09-22T23:08:02.556181095Z"msg":","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291[2] a67c3619-5a95-4827-9e5d-6fcdacafd685 (29dc6a8c-0001-4949-b734-4fa4c3de49de) Active Active WaitQuorum ds_transition to Active","",:""v":downstairs0","}
name":"crucible","level":30,"time":"2023-09-22T23:08:02.556215827Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal",,""pid":hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"time":""msg"2023-09-22T23:08:02.556172897Z":","hostname":"4291ip-10-150-1-74.us-west-2.compute.internal[1] 47522d2c-b387-4d7d-a896-16a922aa39ba (704620be-2e8a-473a-8f41-8d5ed8675af2) Active New New ds_transition to WaitActive"","},pid"":4291}
{"msg":"[1] f238b930-a5e0-4425-a639-165303326233 (45eb49ee-4fdc-4ffa-8697-e0375c977186) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30v":0,"name":"crucible","level":30
,"time":"{2023-09-22T23:08:02.556343206Z","hostname":"",msg""ip-10-150-1-74.us-west-2.compute.internal":,""timepid"":[2] Transition from WaitQuorum to Active"4291:",}"2023-09-22T23:08:02.55635025Z
"v":0,,"{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.556395633Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
hostname":"{ip-10-150-1-74.us-west-2.compute.internal","pid":"4291msg":"}
[1] f238b930-a5e0-4425-a639-165303326233 (45eb49ee-4fdc-4ffa-8697-e0375c977186) Active Faulted Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.556438302Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{{"msg":"""msg"[1] Transition from New to WaitActive:"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.556481286Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 47522d2c-b387-4d7d-a896-16a922aa39ba (704620be-2e8a-473a-8f41-8d5ed8675af2) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30name":,""timecrucible"":",[1] Transition from Faulted to LiveRepairReady2023-09-22T23:08:02.556533757Z"","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
"level":{30"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,",time"":"v":2023-09-22T23:08:02.55660298Z"0,","hostname"name"::""crucibletest live_repair::repair_test::test_repair_dep_cleanup_repair ... ,","oklevel"
:30"time":"2023-09-22T23:08:02.55661369Z",","time"hostname":":"ip-10-150-1-74.us-west-2.compute.internal"2023-09-22T23:08:02.556643946Z","pid":,"4291hostname":"}
{"msg":"ip-10-150-1-74.us-west-2.compute.internal[1] 47522d2c-b387-4d7d-a896-16a922aa39ba (704620be-2e8a-473a-8f41-8d5ed8675af2) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible"","level",:"30pid":4291ip-10-150-1-74.us-west-2.compute.internal}"
,"pid":4291}{
","msg"time":"{2023-09-22T23:08:02.556704205Z"","msg":hostname":""ip-10-150-1-74.us-west-2.compute.internal","pid"::4291"[1] f238b930-a5e0-4425-a639-165303326233 (45eb49ee-4fdc-4ffa-8697-e0375c977186) Active LiveRepairReady Active ds_transition to LiveRepair"}
{a67c3619-5a95-4827-9e5d-6fcdacafd685 is now active with session: 29dc6a8c-0001-4949-b734-4fa4c3de49de","v":0,"name":"crucible","level":30"msg":"[1] Transition from WaitQuorum to Active","v":,"0time":","name"2023-09-22T23:08:02.556799686Z:"","crucible"hostname":","level"ip-10-150-1-74.us-west-2.compute.internal":,"v":0,"name":"crucible","level":3030,"time":"2023-09-22T23:08:02.556832004Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
,"time":",2023-09-22T23:08:02.556851418Z""{,pid":""msg":"hostname":[1] Transition from LiveRepairReady to LiveRepair"","v":0ip-10-150-1-74.us-west-2.compute.internal",,"pid":4291}
4291}
{"msg":"[1] a67c3619-5a95-4827-9e5d-6fcdacafd685 (29dc6a8c-0001-4949-b734-4fa4c3de49de) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.556909698Z"{,"hostname":""ip-10-150-1-74.us-west-2.compute.internal"msg,"":pid":""name":"crucible","level":30[2] 47522d2c-b387-4d7d-a896-16a922aa39ba (704620be-2e8a-473a-8f41-8d5ed8675af2) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"4291time":"}
2023-09-22T23:08:02.556965994Z,""time{,"":"hostnamemsg"":"ip-10-150-1-74.us-west-2.compute.internal","pid":42912023-09-22T23:08:02.556975141Z"},
"hostname":"{ip-10-150-1-74.us-west-2.compute.internal","pid":"4291msg":"}[1] 0 final dependency list []","v":0,"
{":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.557053286Z","hostname":""ip-10-150-1-74.us-west-2.compute.internal","msg"pid":4291:"}
[2] Transition from New to WaitActive","v":{0,"name"":msg":""namecrucible"",":level"[1] a67c3619-5a95-4827-9e5d-6fcdacafd685 (29dc6a8c-0001-4949-b734-4fa4c3de49de) Active Faulted Active ds_transition to LiveRepairReady"",:crucible30,"time":"2023-09-22T23:08:02.557096468Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":""[2] 47522d2c-b387-4d7d-a896-16a922aa39ba (704620be-2e8a-473a-8f41-8d5ed8675af2) Active Active WaitActive ds_transition to WaitQuorum",",""vlevel"v"::0,"name":"crucible","level":030,"name":"crucible","level":30":30,"time":"2023-09-22T23:08:02.557155027Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":,4291"time":}"
{2023-09-22T23:08:02.557159392Z","{,hostname"""msgtime"":""::"2023-09-22T23:08:02.557164099Z","[1] Transition from Faulted to LiveRepairReady"hostname"",:""v"ip-10-150-1-74.us-west-2.compute.internalip-10-150-1-74.us-west-2.compute.internalmsg"",":"","pid":4291}
:Crucible stats registered with UUID: ddd0b289-599b-47c8-9e88-afd221589469"0,","name"v:"":{crucible0",","name":level"":"30crucible"msg,":"[2] Transition from WaitActive to WaitQuorum"","level":v"30:0,"name":"crucible","level"pid:"30:,"4291time":",2023-09-22T23:08:02.557218759Z"","":"hostname":"downstairs","}ip-10-150-1-74.us-west-2.compute.internaltime"",:""pid
2023-09-22T23:08:02.557226531Z"":,4291,"}"
hostname"time"{"msg:"":":"ip-10-150-1-74.us-west-2.compute.internal"2023-09-22T23:08:02.557232588Z,""pid":,4291[1] a67c3619-5a95-4827-9e5d-6fcdacafd685 (29dc6a8c-0001-4949-b734-4fa4c3de49de) Active LiveRepairReady Active ds_transition to LiveRepair"}",
"{hostname"{"msg":"":"msg":"ip-10-150-1-74.us-west-2.compute.internal"[1] 0 final dependency list [JobId(1006)]"Crucible ddd0b289-599b-47c8-9e88-afd221589469 has session id: 6162f099-e560-430c-abf9-296ea61a199b",,,""vpid":4291":0,"name}"
v":0,"{name":"crucible","level"":msg"30:":"crucible","time":","2023-09-22T23:08:02.557310463Z"level",:"hostname"30:"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"[2] 47522d2c-b387-4d7d-a896-16a922aa39ba (704620be-2e8a-473a-8f41-8d5ed8675af2) Active Active WaitQuorum ds_transition to Active"time":","v"2023-09-22T23:08:02.557328946Z":0,","hostname"name,:"""time":"ip-10-150-1-74.us-west-2.compute.internal","2023-09-22T23:08:02.557344011Z":","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid"crucible:4291}
","level"{:30"msg":""pidWrite to Extent 0:1:9 under repairv""":,":v0":42910,","name"name:"",:crucible""",crucible""",level"":":level":30downstairs40"},
"time":"2023-09-22T23:08:02.557377513Z","hostname":,{",ip-10-150-1-74.us-west-2.compute.internal"",""timepid":4291}
"time":"2023-09-22T23:08:02.557393803Z","hostname":"{msgip-10-150-1-74.us-west-2.compute.internal"":msg"":"","pid":[1] 0 final dependency list [JobId(1007), JobId(1006)]"4291[2] Transition from WaitQuorum to Active"},
,"v":0{,"name":""crucible"msg":","level":30[1] 1004 final dependency list [JobId(1001), JobId(1002), JobId(1003)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.557454546Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":,""time":"downstairs""2023-09-22T23:08:02.557449957Z"}v",:0,"name":"crucible"","hostnamelevel"":":"30ip-10-150-1-74.us-west-2.compute.internal":","pid":42912023-09-22T23:08:02.55739335Z,"":"downstairs""}
,"hostname":","ip-10-150-1-74.us-west-2.compute.internal"time,""pid":":
42912023-09-22T23:08:02.557485605Z"}
{,"hostname":"{"msg":"ip-10-150-1-74.us-west-2.compute.internal"","msgpid[1] 1005 final dependency list [JobId(1004), JobId(1001)]"":","":v"4291:0,"}name[0] ddd0b289-599b-47c8-9e88-afd221589469 (9d011751-8e3e-40ec-9a26-e2c509116656) New New New ds_transition to WaitActive"":",
{"msg":"crucible"",v"47522d2c-b387-4d7d-a896-16a922aa39ba is now active with session: 704620be-2e8a-473a-8f41-8d5ed8675af2level":30","v":0,"name":"crucible","level":30",":time":"02023-09-22T23:08:02.557585526Z",","hostname":name"":"crucible"ip-10-150-1-74.us-west-2.compute.internal,",,""pid":4291,"":"downstairs"}
time":"2023-09-22T23:08:02.557594145Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 47522d2c-b387-4d7d-a896-16a922aa39ba (704620be-2e8a-473a-8f41-8d5ed8675af2) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30"level":30,"time":"2023-09-22T23:08:02.557678645Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{,"time"":"msg":"2023-09-22T23:08:02.557694139Z"[0] Transition from Active to Faulted",",hostname":""v":0,ip-10-150-1-74.us-west-2.compute.internal"",name""pid":"crucible","level":30:4291}
,"time":"2023-09-22T23:08:02.557734801Z"{,"hostname":"ip-10-150-1-74.us-west-2.compute.internal""msg",":pid":4291}
"{[0] Transition from New to WaitActive""msg":,""v":0,"name":"crucible","level":30[0] 47522d2c-b387-4d7d-a896-16a922aa39ba (704620be-2e8a-473a-8f41-8d5ed8675af2) Faulted Active Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.557783399Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test live_repair::repair_test::test_repair_dep_cleanup_some ... ,ok"
time":"2023-09-22T23:08:02.557790645Z"test live_repair::repair_test::test_repair_dep_cleanup_sk_repair ... ,"ok
hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":{4291}
{"msg":""msg":"[0] ddd0b289-599b-47c8-9e88-afd221589469 (9d011751-8e3e-40ec-9a26-e2c509116656) WaitActive New New ds_transition to WaitQuorum"[0] Transition from Faulted to LiveRepairReady",",v":"0v",:"name0,"name":"crucible","level":30":"crucible","level":30,"time":"2023-09-22T23:08:02.557901515Z",,""hostname"time":":"ip-10-150-1-74.us-west-2.compute.internal"2023-09-22T23:08:02.557897532Z,""pid":,"4291hostname":"}
ip-10-150-1-74.us-west-2.compute.internal","pid":4291{}
{"msg":""msg":"[0] 47522d2c-b387-4d7d-a896-16a922aa39ba (704620be-2e8a-473a-8f41-8d5ed8675af2) LiveRepairReady Active Active ds_transition to LiveRepair","v"[0] Transition from WaitActive to WaitQuorum":0,","namev":":"0crucible",","name"level":":crucible"30,"level":30,"time":","2023-09-22T23:08:02.557992933Z"time":","hostname":"2023-09-22T23:08:02.557995131Z","ip-10-150-1-74.us-west-2.compute.internalhostname"":","pid":4291ip-10-150-1-74.us-west-2.compute.internal","}pid"
{"msg":"[0] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":":2023-09-22T23:08:02.558073613Z"4291,"hostname":"}
ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"{Waiting for Close + ReOpen jobs","v":0,""name":"msg"crucible:"","level":30[0] ddd0b289-599b-47c8-9e88-afd221589469 (9d011751-8e3e-40ec-9a26-e2c509116656) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.558127041Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
,"time":"2023-09-22T23:08:02.558136732Z{","hostname":""msg":"ip-10-150-1-74.us-west-2.compute.internal","pidRE:0 repair extent with ids 1000,1001,1002,1003 deps:[]"":,4291"v":}0
,"name":"crucible","level":{30"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.558176542Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"}time":
{"msg":"RE:0 close id:1000 queued, notify DS","v":{0,"name":""crucible"msg":,""level":30Crucible stats registered with UUID: 6cc6b842-623b-4ef4-82a7-54fed8104c50","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.558224792Z",""hostname":","2023-09-22T23:08:02.558187001Ztime""ip-10-150-1-74.us-west-2.compute.internal,"hostname":"","ip-10-150-1-74.us-west-2.compute.internalpid"":,"4291pid":}
:"{2023-09-22T23:08:02.558232248Z",""msg"hostname"::""ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
RE:0 Wait for result from close command 1000:1"4291,"v":0,"{name"}:"
"crucible"msg":","level":30{Crucible 6cc6b842-623b-4ef4-82a7-54fed8104c50 has session id: 9828f64b-5ac8-46a1-a32f-54d5107bf3f1","v":0,""name":"msg"crucible":","level":30[1] ddd0b289-599b-47c8-9e88-afd221589469 (9d011751-8e3e-40ec-9a26-e2c509116656) Active New New ds_transition to WaitActive","v":,0","time"name:"":"crucible"2023-09-22T23:08:02.558296568Z",",level"":hostname"30:"ip-10-150-1-74.us-west-2.compute.internal,"","time"pid:"":42912023-09-22T23:08:02.558306885Z"}
,"time":"2023-09-22T23:08:02.558322383Z","hostname":{"ip-10-150-1-74.us-west-2.compute.internal"","msg"pid"::"4291}
Crucible stats registered with UUID: 4d2d5af5-94fb-4532-8477-38add7c548e4","v":0,"{name":"crucible",""level":msg"30:"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.558369873Z",,""hostname":"hostname":",ip-10-150-1-74.us-west-2.compute.internal"",ip-10-150-1-74.us-west-2.compute.internal""time"pid",::4291""}pid2023-09-22T23:08:02.558378202Z"
",:"4291hostname":"{}
ip-10-150-1-74.us-west-2.compute.internal"","msg"pid":{:"4291}
"Crucible 4d2d5af5-94fb-4532-8477-38add7c548e4 has session id: 46d4e137-4411-47b8-a10b-3e746949a248"msg":","v":0{,"name":"crucible",""[0] 6cc6b842-623b-4ef4-82a7-54fed8104c50 (b47c3c9d-3c29-4a37-8eb5-6188f9b2bee7) New New New ds_transition to WaitActive"levelmsg"":",:"30v":0,"name":"crucible"[1] ddd0b289-599b-47c8-9e88-afd221589469 (9d011751-8e3e-40ec-9a26-e2c509116656) Active WaitActive New ds_transition to WaitQuorum",","level"v"::300,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.558440566Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
,"time":"{,"2023-09-22T23:08:02.558449185Z"time":"",msg"2023-09-22T23:08:02.558453357Z""hostname",:":""hostname":"ip-10-150-1-74.us-west-2.compute.internal",ip-10-150-1-74.us-west-2.compute.internal"",pid[0] 4d2d5af5-94fb-4532-8477-38add7c548e4 (b46ad456-c0c5-43cb-8b84-fa6016b7a39b) New New New ds_transition to WaitActive"""pid,:"4291":v"4291:}0
},
{"name":"crucible","level"{"msg"::"30"msg":"[0] Transition from New to WaitActive","v":[1] Transition from WaitActive to WaitQuorum"0,,""namev""::"0crucible",","name"level"::"30crucible",","level"time:":30"2023-09-22T23:08:02.558508174Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
,"time":"2023-09-22T23:08:02.558522034Z"{,",hostname""":time"":msg"":ip-10-150-1-74.us-west-2.compute.internal"",2023-09-22T23:08:02.558526836Z""[0] Transition from New to WaitActive"pid",:,""4291hostname"v":}":
ip-10-150-1-74.us-west-2.compute.internal"0,","pidname"{:":"4291crucible","level":"30}msg"
:"{[0] 6cc6b842-623b-4ef4-82a7-54fed8104c50 (b47c3c9d-3c29-4a37-8eb5-6188f9b2bee7) WaitActive New New ds_transition to WaitQuorum",,""time"msg":"":v"":0,"2023-09-22T23:08:02.558571574Z"name":"[1] ddd0b289-599b-47c8-9e88-afd221589469 (9d011751-8e3e-40ec-9a26-e2c509116656) Active WaitQuorum New ds_transition to Active"crucible",,"",hostname"v":""level"::0ip-10-150-1-74.us-west-2.compute.internal"30,,""name"pid":":crucible"4291,"level"}:
30{","msg":time":""2023-09-22T23:08:02.558605809Z","hostname":"[0] 4d2d5af5-94fb-4532-8477-38add7c548e4 (b46ad456-c0c5-43cb-8b84-fa6016b7a39b) WaitActive New New ds_transition to WaitQuorum",ip-10-150-1-74.us-west-2.compute.internal""v":,"0pid",:"4291,name"":}time""
:crucible"","level":{2023-09-22T23:08:02.558616995Z30",""hostname"msg":":"ip-10-150-1-74.us-west-2.compute.internal"[0] Transition from WaitActive to WaitQuorum",",pid"":v":42910,"name"}:"
crucible",,""time"level:"{":302023-09-22T23:08:02.558649103Z",""hostname"msg"::""ip-10-150-1-74.us-west-2.compute.internal"[1] Transition from WaitQuorum to Active,""pid":,"4291v":0},
,""name"time:""{crucible":","level""msg"::302023-09-22T23:08:02.558673183Z"","hostname":"[0] Transition from WaitActive to WaitQuorum","v":ip-10-150-1-74.us-west-2.compute.internal0",","namepid""::"4291crucible","}
level":,"30time":"{2023-09-22T23:08:02.558698748Z",""hostname"msg":":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
[0] 6cc6b842-623b-4ef4-82a7-54fed8104c50 (b47c3c9d-3c29-4a37-8eb5-6188f9b2bee7) WaitQuorum New New ds_transition to Active",","time":v{"":0,"2023-09-22T23:08:02.558716408Zname""":,"msg"":hostname"":crucible"","level":ip-10-150-1-74.us-west-2.compute.internal"30,"[2] ddd0b289-599b-47c8-9e88-afd221589469 (9d011751-8e3e-40ec-9a26-e2c509116656) Active Active New ds_transition to WaitActive"pid":,"4291v":0},
",name":""time"crucible{:"","level2023-09-22T23:08:02.558749307Z""":msg",30":"hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291[0] 4d2d5af5-94fb-4532-8477-38add7c548e4 (b46ad456-c0c5-43cb-8b84-fa6016b7a39b) WaitQuorum New New ds_transition to Active"}
,"v":0,"name":"{crucible","level":30"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.558791323Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
,"{,"time":"time""msg2023-09-22T23:08:02.558774307Z":"":","hostname2023-09-22T23:08:02.558800315Z"[0] Transition from WaitQuorum to Active"",:","hostname"v:"":"0,"ip-10-150-1-74.us-west-2.compute.internalname":"ip-10-150-1-74.us-west-2.compute.internal""crucible,",""pid",:"4291level":pid}30
":4291}
{,"time":"2023-09-22T23:08:02.558843638Z","hostname"":"msg{":ip-10-150-1-74.us-west-2.compute.internal"","pid":4291"}
msg":"{[1] 6cc6b842-623b-4ef4-82a7-54fed8104c50 (b47c3c9d-3c29-4a37-8eb5-6188f9b2bee7) Active New New ds_transition to WaitActive""msg[2] Transition from New to WaitActive":,"""v",:"0v",:[1] 4d2d5af5-94fb-4532-8477-38add7c548e4 (b46ad456-c0c5-43cb-8b84-fa6016b7a39b) Active New New ds_transition to WaitActive""0name",:,"""cruciblename"v"::""0,,crucible""name","":level"":crucible"level",30":level":3030,"time":"2023-09-22T23:08:02.558896292Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":","[1] Transition from New to WaitActive"time","v,"::0"",time""2023-09-22T23:08:02.558895262Z"name":",:crucible"","hostnamelevel"":30"2023-09-22T23:08:02.558897073Z":","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":ip-10-150-1-74.us-west-2.compute.internal"4291,,""time":"pid"}2023-09-22T23:08:02.558927248Z"
,":hostname":"4291ip-10-150-1-74.us-west-2.compute.internal"},
"{pid":4291}
{"{msg":""msg":""msg"[1] Transition from New to WaitActive":","v":[1] 4d2d5af5-94fb-4532-8477-38add7c548e4 (b46ad456-c0c5-43cb-8b84-fa6016b7a39b) Active WaitActive New ds_transition to WaitQuorum"0,",v":"0,name""name":":"[2] ddd0b289-599b-47c8-9e88-afd221589469 (9d011751-8e3e-40ec-9a26-e2c509116656) Active Active WaitActive ds_transition to WaitQuorumcrucible"crucible"",","v,""level"::level030",":name30":"crucible","level":30,"time":"2023-09-22T23:08:02.558979251Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level,"":30time":"2023-09-22T23:08:02.558982039Z",","time":hostname"":,""time":"2023-09-22T23:08:02.558985357Z"ip-10-150-1-74.us-west-2.compute.internal"2023-09-22T23:08:02.558999189Z",,"",pid"hostname"hostname""::":"4291ip-10-150-1-74.us-west-2.compute.internalip-10-150-1-74.us-west-2.compute.internal"},
""pid":,"4291pid"}{:
"4291msg"{}:""msg":"
[1] 4d2d5af5-94fb-4532-8477-38add7c548e4 (b46ad456-c0c5-43cb-8b84-fa6016b7a39b) Active WaitQuorum New ds_transition to Active"[1] 6cc6b842-623b-4ef4-82a7-54fed8104c50 (b47c3c9d-3c29-4a37-8eb5-6188f9b2bee7) Active WaitActive New ds_transition to WaitQuorum","v",:0"{,"v":name":""0msgcrucible,""","name:level":"":30"[2] Transition from WaitActive to WaitQuorumcrucible"",","level"v:":,"30time":"0,"2023-09-22T23:08:02.559057093Z"name":,""hostname":"crucible","ip-10-150-1-74.us-west-2.compute.internal","level"pid"::429130}
{"msg":","[1] Transition from WaitQuorum to Active"time","v":0:","name":"crucible",2023-09-22T23:08:02.559065171Z""level":30,"hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":,4291","time":"time"}2023-09-22T23:08:02.559087442Z"
:","hostname":"2023-09-22T23:08:02.559075749Z"ip-10-150-1-74.us-west-2.compute.internal",",{pid":"4291}
"hostname"msg"::"{""msg":"[1] Transition from WaitActive to WaitQuorum"ip-10-150-1-74.us-west-2.compute.internal,"","[2] 4d2d5af5-94fb-4532-8477-38add7c548e4 (b46ad456-c0c5-43cb-8b84-fa6016b7a39b) Active Active New ds_transition to WaitActive"pid","v"v":0::,"42910name":",crucible""},
name{"msg":"[2] ddd0b289-599b-47c8-9e88-afd221589469 (9d011751-8e3e-40ec-9a26-e2c509116656) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,""time"::""crucible"2023-09-22T23:08:02.55915953Z",","level":hostname":30"ip-10-150-1-74.us-west-2.compute.internal","pid":4291"}
level":30{"msg":"[2] Transition from WaitQuorum to Active",","time":v,""":time":"02023-09-22T23:08:02.559182408Z"2023-09-22T23:08:02.559192565Z",,""name",:hostname":"""hostnamecrucible""ip-10-150-1-74.us-west-2.compute.internal",":,"level"":pid"30ip-10-150-1-74.us-west-2.compute.internal":4291,"}pid"
:4291{}
"msg":"[2] Transition from New to WaitActive","v":0,"name,":"{crucible"","leveltime"""::"30msg":"2023-09-22T23:08:02.559221817Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","[1] 6cc6b842-623b-4ef4-82a7-54fed8104c50 (b47c3c9d-3c29-4a37-8eb5-6188f9b2bee7) Active WaitQuorum New ds_transition to Activepid"":,,4291""time"}:"
v"2023-09-22T23:08:02.559244963Z:","0hostname":",{"ip-10-150-1-74.us-west-2.compute.internal"name",""pid":msg":"crucible:"4291"},
"level"ddd0b289-599b-47c8-9e88-afd221589469 is now active with session: 9d011751-8e3e-40ec-9a26-e2c509116656"{:,"30msg":""v":0,"name":"crucible[2] 4d2d5af5-94fb-4532-8477-38add7c548e4 (b46ad456-c0c5-43cb-8b84-fa6016b7a39b) Active Active WaitActive ds_transition to WaitQuorum"","v":0,,""name"level":":crucible"30,"level":30,"time":","2023-09-22T23:08:02.559285503Z"time":","2023-09-22T23:08:02.559297392Z"hostname,"":hostname":""ip-10-150-1-74.us-west-2.compute.internal","pid":ip-10-150-1-74.us-west-2.compute.internal4291,}"
,""{pid"time"":msg"::"4291"[2] Transition from WaitActive to WaitQuorum"},"
2023-09-22T23:08:02.559296874Z"v":0,",name"":"hostnamecrucible""{,":level":"30"msg":"ip-10-150-1-74.us-west-2.compute.internal","pid":[1] Transition from WaitQuorum to Active"4291,"v"}:
0,",time":""name"{2023-09-22T23:08:02.559338336Z":","crucible"hostname":"","msglevelip-10-150-1-74.us-west-2.compute.internal"",":"30pid"::4291"}
{"msg":"[0] ddd0b289-599b-47c8-9e88-afd221589469 (9d011751-8e3e-40ec-9a26-e2c509116656) Active Active Active ds_transition to Faulted","v":0[2] 4d2d5af5-94fb-4532-8477-38add7c548e4 (b46ad456-c0c5-43cb-8b84-fa6016b7a39b) Active Active WaitQuorum ds_transition to Active",","namev":":0","crucible"name":",","crucible"timelevel",""::30"level":302023-09-22T23:08:02.559362311Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid,""time"::"42912023-09-22T23:08:02.559386034Z",}"
hostname":","time"ip-10-150-1-74.us-west-2.compute.internal":","{pid2023-09-22T23:08:02.55938561Z"":4291},"
"msghostname"{":":msg":"""[2] Transition from WaitQuorum to Active","v":ip-10-150-1-74.us-west-2.compute.internal"0,",name":""crucible"pid,"[2] 6cc6b842-623b-4ef4-82a7-54fed8104c50 (b47c3c9d-3c29-4a37-8eb5-6188f9b2bee7) Active Active New ds_transition to WaitActive""level"::,4291"30v"}:
0,","time":name"":{"2023-09-22T23:08:02.559430161Z","crucible"hostname"",:msg"":level""":ip-10-150-1-74.us-west-2.compute.internal"30,"[0] Transition from Active to Faulted"pid":4291,"}
v":{0,""msg":"name":"crucible","level"4d2d5af5-94fb-4532-8477-38add7c548e4 is now active with session: b46ad456-c0c5-43cb-8b84-fa6016b7a39b":,"30v":0,"name":"crucible","level":,"30time":"2023-09-22T23:08:02.559449698Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal",","time":"pid":2023-09-22T23:08:02.55946625Z"4291,"hostname":"}
ip-10-150-1-74.us-west-2.compute.internal",,""pid":4291{time"}:
""{msg":""msg":"2023-09-22T23:08:02.55946281Z"[2] Transition from New to WaitActive",,""v[0] 4d2d5af5-94fb-4532-8477-38add7c548e4 (b46ad456-c0c5-43cb-8b84-fa6016b7a39b) Active Active Active ds_transition to Faultedhostname":"":,""v":00,",ip-10-150-1-74.us-west-2.compute.internal"namename":""crucible"":,","cruciblelevel":"30",pid"":level"4291:30}
,"time":"2023-09-22T23:08:02.559514406Z","hostname":"{ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
"msg"{:,"""time":msg":""[0] Transition from Active to Faulted","2023-09-22T23:08:02.559519119Z"v":0[0] ddd0b289-599b-47c8-9e88-afd221589469 (9d011751-8e3e-40ec-9a26-e2c509116656) Faulted Active Active ds_transition to LiveRepairReady",,,"""hostname"v"::name"":"0crucible",",ip-10-150-1-74.us-west-2.compute.internallevel":"30"name",:""pid":crucible"4291,"}level":,"30time":"
2023-09-22T23:08:02.559553402Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":{4291}
,""time"msg"{::""msg":""2023-09-22T23:08:02.559562215Z","hostname":"[0] 4d2d5af5-94fb-4532-8477-38add7c548e4 (b46ad456-c0c5-43cb-8b84-fa6016b7a39b) Faulted Active Active ds_transition to LiveRepairReady","v":0,"ip-10-150-1-74.us-west-2.compute.internal"name"[2] 6cc6b842-623b-4ef4-82a7-54fed8104c50 (b47c3c9d-3c29-4a37-8eb5-6188f9b2bee7) Active Active WaitActive ds_transition to WaitQuorum",:""crucible",pid"",":level4291":v"}30:
0,"name":","time":"{crucible"2023-09-22T23:08:02.559602681Z",,""hostname""level"msg"::":"30ip-10-150-1-74.us-west-2.compute.internal"[0] Transition from Faulted to LiveRepairReady","pid":,4291"}v"
:0{,""name"msg:":""crucible"[0] Transition from Faulted to LiveRepairReady",,","level"v":0:,""30name"time"::""crucible","level":302023-09-22T23:08:02.559618411Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":,"4291time":"}2023-09-22T23:08:02.559644024Z"
,",hostname":""{time"ip-10-150-1-74.us-west-2.compute.internal":,"""msg"pid"::2023-09-22T23:08:02.559640049Z"4291"},
"[2] Transition from WaitActive to WaitQuorum{hostname""":msg":",""v":0,ip-10-150-1-74.us-west-2.compute.internal"",[0] 4d2d5af5-94fb-4532-8477-38add7c548e4 (b46ad456-c0c5-43cb-8b84-fa6016b7a39b) LiveRepairReady Active Active ds_transition to LiveRepairname""":pid"","crucible":,v4291"":level0},
""name":":crucible","{level":3030"msg":","time":"2023-09-22T23:08:02.559700189Z","[0] ddd0b289-599b-47c8-9e88-afd221589469 (9d011751-8e3e-40ec-9a26-e2c509116656) LiveRepairReady Active Active ds_transition to LiveRepair"hostname":","v":ip-10-150-1-74.us-west-2.compute.internal","0pid":,4291"}name"
:,"{"crucibletime"":""msg":","2023-09-22T23:08:02.559701031Z"level"[0] Transition from LiveRepairReady to LiveRepair",,":v":300","namehostname"":":crucible"","level":30ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
,"time":"2023-09-22T23:08:02.559736044Z","hostname":",{"ip-10-150-1-74.us-west-2.compute.internal","time"pid"::4291"}
2023-09-22T23:08:02.559731271Z""{,"msg"hostname""::""msg":"Waiting for Close + ReOpen jobs"ip-10-150-1-74.us-west-2.compute.internal",",v":"0pid[2] 6cc6b842-623b-4ef4-82a7-54fed8104c50 (b47c3c9d-3c29-4a37-8eb5-6188f9b2bee7) Active Active WaitQuorum ds_transition to Active"",,":name4291":""crucible"}v"
,":level":300,"name":"{crucible","level":30,"time":"2023-09-22T23:08:02.559782311Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.559787881Z""msg",,":""time":"hostname":"2023-09-22T23:08:02.559801664Z"[0] Transition from LiveRepairReady to LiveRepair,""hostname":,"ip-10-150-1-74.us-west-2.compute.internal"",v""ip-10-150-1-74.us-west-2.compute.internalpid:"0",,":pid4291":"}name"4291
}
:"{{crucible"",msg":"""level"msg"::RE:0 close id:1000 queued, notify DS30"","v":0,"name":"[2] Transition from WaitQuorum to Active"crucible",,""level":30v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.559850269Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,}
"time{":""msg":"2023-09-22T23:08:02.5598442Z"RE:0 Wait for result from close command 1000:1",",v"":,hostname"0","time":name":""crucible":,""level":ip-10-150-1-74.us-west-2.compute.internal"2023-09-22T23:08:02.559856111Z"30,,""pid":hostname"4291:"}
,"ip-10-150-1-74.us-west-2.compute.internaltime":"","2023-09-22T23:08:02.559882266Z"pid,""hostname:{":"4291ip-10-150-1-74.us-west-2.compute.internal","}"pidmsg"":
4291:}"
{Waiting for Close + ReOpen jobs","v":"0msg",:""name":"crucible","level":306cc6b842-623b-4ef4-82a7-54fed8104c50 is now active with session: b47c3c9d-3c29-4a37-8eb5-6188f9b2bee7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.559922195Z","hostname":","time":"ip-10-150-1-74.us-west-2.compute.internal","pid":2023-09-22T23:08:02.559929135Z"4291,"}hostname":"
ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"{msg":""msg":"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]","v":0,"name":"crucible","[0] 6cc6b842-623b-4ef4-82a7-54fed8104c50 (b47c3c9d-3c29-4a37-8eb5-6188f9b2bee7) Active Active Active ds_transition to Faulted"level":,"30v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.559981726Z",",time":""hostname":"2023-09-22T23:08:02.559986603Z","ip-10-150-1-74.us-west-2.compute.internal"hostname",:""pid":4291}ip-10-150-1-74.us-west-2.compute.internal
","pid":4291{}
"msg":"RE:0 close id:1000 queued, notify DS"{,"v":0,""name"msg"::""crucible","[0] Transition from Active to Faulted"level":,30"v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.560035338Z","hostname":","time":"ip-10-150-1-74.us-west-2.compute.internal","2023-09-22T23:08:02.560040396Z"pid":,4291"hostname"}:
"ip-10-150-1-74.us-west-2.compute.internal","{pid":4291}"
msg":"RE:0 Wait for result from close command 1000:1"{,"v":0","msg":name"":"crucible","level":30[0] 6cc6b842-623b-4ef4-82a7-54fed8104c50 (b47c3c9d-3c29-4a37-8eb5-6188f9b2bee7) Faulted Active Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.560086627Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"}time"
:"2023-09-22T23:08:02.560095294Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.560140806Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 6cc6b842-623b-4ef4-82a7-54fed8104c50 (b47c3c9d-3c29-4a37-8eb5-6188f9b2bee7) LiveRepairReady Active Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.56018441Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.56022408Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for Close + ReOpen jobs","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.560261763Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.560300681Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 close id:1000 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.560339479Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from close command 1000:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:02.560379456Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:03.425446046Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"No Live Repair required at this time","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:03.425503829Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:03.426475731Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"No Live Repair required at this time","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:03.426509514Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:03.427372534Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"No Live Repair required at this time","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:03.427400648Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] ed581aed-4fb6-4c12-84d1-a857283bcda2 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:03.430809884Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"looper":"0","upstairs":1}
{"msg":"[0] Proc runs for 127.0.0.1:60909 in state Offline","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:03.430864903Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] client re-new 1 jobs since flush 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:03.430888606Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"negotiate packet HereIAm { version: 4, upstairs_id: ed581aed-4fb6-4c12-84d1-a857283bcda2, session_id: 99a7dfa5-d205-4d07-9135-9f9e70387e3a, gen: 1, read_only: false, encrypted: false, alternate_versions: [] }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:03.431050985Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"[0] upstairs guest_io_ready=TRUE, promote! session 99a7dfa5-d205-4d07-9135-9f9e70387e3a","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:03.431189197Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"negotiate packet PromoteToActive { upstairs_id: ed581aed-4fb6-4c12-84d1-a857283bcda2, session_id: 99a7dfa5-d205-4d07-9135-9f9e70387e3a, gen: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:03.431324611Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"negotiate packet RegionInfoPlease","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:03.431516376Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"[0] downstairs client at 127.0.0.1:60909 has UUID 761a4173-d6d6-4e0b-98cb-d77e4463419a","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:03.43165071Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 10, shift: 9 }, extent_count: 10, uuid: 761a4173-d6d6-4e0b-98cb-d77e4463419a, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:03.431683406Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"Returning client:0 UUID:761a4173-d6d6-4e0b-98cb-d77e4463419a matches","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:03.431704248Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] send last flush ID to this DS: 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:03.431723517Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"ed581aed-4fb6-4c12-84d1-a857283bcda2 Offline Active Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:03.43174306Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"negotiate packet LastFlush { last_flush_number: JobId(0) }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:03.431866099Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
waiting for ds1 message in test_replay_occurs
{"msg":"[0] Replied this last flush ID: 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:03.432044004Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] ed581aed-4fb6-4c12-84d1-a857283bcda2 (99a7dfa5-d205-4d07-9135-9f9e70387e3a) Offline Active Active ds_transition to Replay","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:03.432073342Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] Transition from Offline to Replay","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:03.432093119Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] ed581aed-4fb6-4c12-84d1-a857283bcda2 Transition from Replay to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:03.432118687Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] ed581aed-4fb6-4c12-84d1-a857283bcda2 (99a7dfa5-d205-4d07-9135-9f9e70387e3a) Replay Active Active ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:03.432141226Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] Transition from Replay to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:03.432160359Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:03.432178731Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] 127.0.0.1:60909 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:03.432198088Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"ed581aed-4fb6-4c12-84d1-a857283bcda2 Active Active Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:03.432217528Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:03.432236041Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:03.441665653Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"No Live Repair required at this time","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:03.441768563Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] DS Reports error Err(GenericError(\"bad\")) on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentFlushClose { dependencies: [], extent: 0, flush_number: 0, gen_number: 0, source_downstairs: ClientId(1), repair_downstairs: [ClientId(0)] }, state: ClientData([InProgress, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:03.556827282Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[0] Reports error GenericError(\"bad\") on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentFlushClose { dependencies: [], extent: 0, flush_number: 0, gen_number: 0, source_downstairs: ClientId(1), repair_downstairs: [ClientId(0)] }, state: ClientData([InProgress, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:03.556871668Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[0] client skip 2 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:03.556893165Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[0] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:03.556912693Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[0] 47522d2c-b387-4d7d-a896-16a922aa39ba (704620be-2e8a-473a-8f41-8d5ed8675af2) LiveRepair Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:03.556934565Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from LiveRepair to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:03.556952938Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now ACK the close job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:03.556976295Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for 3 jobs (currently 2)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:03.556995525Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Extent 0 close id:1000 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:03.557032561Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:03.557141784Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now ACK the close job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:03.55892295Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for 3 jobs (currently 2)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:03.558978095Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Repair for extent 0 s:1 d:[ClientId(0)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:03.559021583Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:03.559065189Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for 3 jobs (currently 2)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:03.560159328Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{{"msg":""No repair needed for extent 0"msg",:""v":0Waiting for 3 jobs (currently 2)","name":","crucible"v",:"level"0:,30"name":"crucible","level":30,"time":"2023-09-22T23:08:03.560214306Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs",}"
time":"2023-09-22T23:08:03.560219058Z"{,"hostname":""msg":"ip-10-150-1-74.us-west-2.compute.internalRE:0 Wait for result from repair command 1001:2"","v":,0","pidname":""crucible":,"4291level":30}
,"time":"2023-09-22T23:08:03.560263061Z","{hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":"4291msg"}
:"No repair needed for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:03.560289567Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:03.560332436Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] flow control start ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:03.649735642Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] flow control start ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:03.649769687Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[1] flow control start ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:03.649790121Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
waiting for ds1 message in test_replay_occurs
{"msg":"[0] flow control end ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:04.433698391Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:04.433752554Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"No Live Repair required at this time","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:04.433772968Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:04.558520272Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Extent 0 close id:1001 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:04.558578031Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:04.558619676Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now ACK the repair job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:04.560279684Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:04.560317513Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:04.560377271Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:04.561511161Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:04.561562274Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:04.561630568Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:04.561684934Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] flow control end ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:04.651486665Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[1] flow control end ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:04.651521984Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[2] flow control end ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:04.651542307Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
test dummy_downstairs_tests::protocol_test::test_replay_occurs ... ok
{"msg":"Crucible stats registered with UUID: f2efd0c8-30aa-476f-937f-c6a917e6b31a","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.435788686Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible f2efd0c8-30aa-476f-937f-c6a917e6b31a has session id: 9293bb25-dab9-452a-9395-23615764e473","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.435841552Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] f2efd0c8-30aa-476f-937f-c6a917e6b31a (96b0b6bd-173c-4e2c-90e7-f7302532f9ef) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.435864963Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.435883978Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] f2efd0c8-30aa-476f-937f-c6a917e6b31a (96b0b6bd-173c-4e2c-90e7-f7302532f9ef) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.435904725Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.435923009Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] f2efd0c8-30aa-476f-937f-c6a917e6b31a (96b0b6bd-173c-4e2c-90e7-f7302532f9ef) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.435943258Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.435969844Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] f2efd0c8-30aa-476f-937f-c6a917e6b31a (96b0b6bd-173c-4e2c-90e7-f7302532f9ef) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.435991742Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.436009889Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] f2efd0c8-30aa-476f-937f-c6a917e6b31a (96b0b6bd-173c-4e2c-90e7-f7302532f9ef) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.436030307Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.436048442Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] f2efd0c8-30aa-476f-937f-c6a917e6b31a (96b0b6bd-173c-4e2c-90e7-f7302532f9ef) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.436068672Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.436086806Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] f2efd0c8-30aa-476f-937f-c6a917e6b31a (96b0b6bd-173c-4e2c-90e7-f7302532f9ef) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.436106809Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.436124536Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] f2efd0c8-30aa-476f-937f-c6a917e6b31a (96b0b6bd-173c-4e2c-90e7-f7302532f9ef) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.436144811Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.436162861Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] f2efd0c8-30aa-476f-937f-c6a917e6b31a (96b0b6bd-173c-4e2c-90e7-f7302532f9ef) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.436182964Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.436200833Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"f2efd0c8-30aa-476f-937f-c6a917e6b31a is now active with session: 96b0b6bd-173c-4e2c-90e7-f7302532f9ef","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.436220222Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] f2efd0c8-30aa-476f-937f-c6a917e6b31a (96b0b6bd-173c-4e2c-90e7-f7302532f9ef) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.436240276Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.436258096Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] f2efd0c8-30aa-476f-937f-c6a917e6b31a (96b0b6bd-173c-4e2c-90e7-f7302532f9ef) Faulted Active Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.43627839Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.436296772Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] f2efd0c8-30aa-476f-937f-c6a917e6b31a (96b0b6bd-173c-4e2c-90e7-f7302532f9ef) LiveRepairReady Active Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.436322877Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.436342175Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for Close + ReOpen jobs","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.436360085Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.436378403Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 close id:1000 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.436396303Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from close command 1000:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.43641453Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] flow control start ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:05.519062727Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"Now move the NoOp job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.558834176Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now ACK the NoOp job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.558863616Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Finally, move the ReOpen job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.558882651Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now ACK the Reopen job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.558900579Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Extent 0 close id:1002 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:05.558928334Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from reopen command 1003:4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.558947386Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Extent 0 close id:1003 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:05.558965506Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Bailing with error","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:05.558983077Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
----------------------------------------------------------------
 Crucible gen:0 GIO:true work queues:  Upstairs:0  downstairs:4
GW_ID      ACK  DSID    TYPE  BLOCKS  DS:0  DS:1  DS:2  REPLAY
    1    Acked  1000  FClose       0   Err  Done  Done  false
    2    Acked  1001    NoOp       0  Skip  Done  Done  false
    3    Acked  1002    NoOp       0  Skip  Done  Done  false
    4    Acked  1003  Reopen       0  Skip  Done  Done  false
   STATES      DS:0   DS:1   DS:2   TOTAL
    New           0      0      0      0
    Sent          0      0      0      0
    Done          0      4      4      8
    Skipped       3      0      0      3
    Error         1      0      0      1
Last Flush: 0 0 0 
Downstairs last five completed:
Upstairs last five completed:      4    3    2    1
{"msg":"Crucible stats registered with UUID: a539c30f-2c8a-4e0b-a419-6c092883f4c8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.559638981Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible a539c30f-2c8a-4e0b-a419-6c092883f4c8 has session id: 5ae92c7b-6713-4df8-9f2b-2c31010701d8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.559681144Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] a539c30f-2c8a-4e0b-a419-6c092883f4c8 (2bb8288f-73e5-46ca-83e3-b652d8887f09) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.559705178Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.559724061Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] a539c30f-2c8a-4e0b-a419-6c092883f4c8 (2bb8288f-73e5-46ca-83e3-b652d8887f09) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.559744325Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.559762607Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] a539c30f-2c8a-4e0b-a419-6c092883f4c8 (2bb8288f-73e5-46ca-83e3-b652d8887f09) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.559782526Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.559800446Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] a539c30f-2c8a-4e0b-a419-6c092883f4c8 (2bb8288f-73e5-46ca-83e3-b652d8887f09) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.559820339Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.55983802Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] a539c30f-2c8a-4e0b-a419-6c092883f4c8 (2bb8288f-73e5-46ca-83e3-b652d8887f09) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.559858568Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.55987641Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] a539c30f-2c8a-4e0b-a419-6c092883f4c8 (2bb8288f-73e5-46ca-83e3-b652d8887f09) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.559896471Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.559914362Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] a539c30f-2c8a-4e0b-a419-6c092883f4c8 (2bb8288f-73e5-46ca-83e3-b652d8887f09) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.559934303Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.559952064Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] a539c30f-2c8a-4e0b-a419-6c092883f4c8 (2bb8288f-73e5-46ca-83e3-b652d8887f09) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.559972174Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.559990219Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] a539c30f-2c8a-4e0b-a419-6c092883f4c8 (2bb8288f-73e5-46ca-83e3-b652d8887f09) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.560010343Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.560032931Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"a539c30f-2c8a-4e0b-a419-6c092883f4c8 is now active with session: 2bb8288f-73e5-46ca-83e3-b652d8887f09","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.560053117Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] a539c30f-2c8a-4e0b-a419-6c092883f4c8 (2bb8288f-73e5-46ca-83e3-b652d8887f09) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.560073128Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.560090865Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] a539c30f-2c8a-4e0b-a419-6c092883f4c8 (2bb8288f-73e5-46ca-83e3-b652d8887f09) Faulted Active Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.560111061Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.560129089Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] a539c30f-2c8a-4e0b-a419-6c092883f4c8 (2bb8288f-73e5-46ca-83e3-b652d8887f09) LiveRepairReady Active Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.56014929Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.560167564Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for Close + ReOpen jobs","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.560185252Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.560203534Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 close id:1000 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.560221255Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from close command 1000:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.560239292Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now move the NoOp job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.561595209Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now ACK the NoOp job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.561625022Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Finally, move the ReOpen job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.561654463Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now ACK the repair job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.561678537Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from reopen command 1003:4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.561712896Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible stats registered with UUID: 1d851a9a-c4b3-459c-8d3d-fb8473f6f33d","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.562096305Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 1d851a9a-c4b3-459c-8d3d-fb8473f6f33d has session id: 9422dc73-f793-41ba-8b59-09b019134370","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.562128888Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 1d851a9a-c4b3-459c-8d3d-fb8473f6f33d (57ff38fd-1d30-4627-9201-6d66cd8c9a9f) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.562156262Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.562176022Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 1d851a9a-c4b3-459c-8d3d-fb8473f6f33d (57ff38fd-1d30-4627-9201-6d66cd8c9a9f) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.562196626Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.562214737Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 1d851a9a-c4b3-459c-8d3d-fb8473f6f33d (57ff38fd-1d30-4627-9201-6d66cd8c9a9f) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.562234701Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.562252661Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 1d851a9a-c4b3-459c-8d3d-fb8473f6f33d (57ff38fd-1d30-4627-9201-6d66cd8c9a9f) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.562272542Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.562290273Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 1d851a9a-c4b3-459c-8d3d-fb8473f6f33d (57ff38fd-1d30-4627-9201-6d66cd8c9a9f) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.562310656Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.56232867Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 1d851a9a-c4b3-459c-8d3d-fb8473f6f33d (57ff38fd-1d30-4627-9201-6d66cd8c9a9f) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.562348736Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.562366543Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 1d851a9a-c4b3-459c-8d3d-fb8473f6f33d (57ff38fd-1d30-4627-9201-6d66cd8c9a9f) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.562386648Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.562404588Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 1d851a9a-c4b3-459c-8d3d-fb8473f6f33d (57ff38fd-1d30-4627-9201-6d66cd8c9a9f) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.562424757Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.562442755Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 1d851a9a-c4b3-459c-8d3d-fb8473f6f33d (57ff38fd-1d30-4627-9201-6d66cd8c9a9f) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.56246286Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.562484663Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"1d851a9a-c4b3-459c-8d3d-fb8473f6f33d is now active with session: 57ff38fd-1d30-4627-9201-6d66cd8c9a9f","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.562505175Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 1d851a9a-c4b3-459c-8d3d-fb8473f6f33d (57ff38fd-1d30-4627-9201-6d66cd8c9a9f) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.56252531Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.562543112Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 1d851a9a-c4b3-459c-8d3d-fb8473f6f33d (57ff38fd-1d30-4627-9201-6d66cd8c9a9f) Active Faulted Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.562563297Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.562581404Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 1d851a9a-c4b3-459c-8d3d-fb8473f6f33d (57ff38fd-1d30-4627-9201-6d66cd8c9a9f) Active LiveRepairReady Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.562601779Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.56262005Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for Close + ReOpen jobs","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.562637835Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.562656053Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 close id:1000 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.562673858Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from close command 1000:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.562691936Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now move the NoOp job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.562801515Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] DS Reports error Err(GenericError(\"bad\")) on job 1002, DownstairsIO { ds_id: JobId(1002), guest_id: 3, work: ExtentLiveNoOp { dependencies: [JobId(1000), JobId(1001)] }, state: ClientData([InProgress, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:05.5628703Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"{downstairs"}
"msg":"{Now move the NoOp job forward",""v"msg"::"0,"[0] Reports error GenericError(name\""bad\":"crucible","level":30) on job 1002, DownstairsIO { ds_id: JobId(1002), guest_id: 3, work: ExtentLiveNoOp { dependencies: [JobId(1000), JobId(1001)] }, state: ClientData([InProgress, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:05.562918993Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291},
{"msg":"[0] DS Reports error Err(GenericError(\"bad\""time":"2023-09-22T23:08:05.562935467Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291)) on job 1003, DownstairsIO { ds_id: JobId(1003), guest_id: 4, work: ExtentLiveReopen { dependencies: [JobId(1000), JobId(1001), JobId(1002)], extent: 0 }, state: ClientData([InProgress, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","":"downstairs","}v"
:0,"name":"crucible"{,"level":50"msg":"[0] client skip 4 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.563042093Z",",time":""hostname2023-09-22T23:08:05.563048519Z"",":"hostname":"ip-10-150-1-74.us-west-2.compute.internal","ip-10-150-1-74.us-west-2.compute.internalpid"",:"4291pid",":":"4291downstairs"}
,"":"{"downstairs"msg":"}
[0] changed 1 jobs to fault skipped","v":0,"name":"crucible"{,"level":30"msg":"[0] Reports error GenericError(\"bad\","time":"2023-09-22T23:08:05.563090351Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[0] 6cc6b842-623b-4ef4-82a7-54fed8104c50 (b47c3c9d-3c29-4a37-8eb5-6188f9b2bee7) LiveRepair Active Active ds_transition to Faulted","v":0,"name) on job 1003, DownstairsIO { ds_id: JobId(1003), guest_id: 4, work: ExtentLiveReopen { dependencies: [JobId(1000), JobId(1001), JobId(1002)], extent: 0 }, state: ClientData([InProgress, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }"":",crucible"","v":level":030,"name":"crucible","level":50,"time":"2023-09-22T23:08:05.563118671Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":","[0] Transition from LiveRepair to Faultedtime"":,""v":0,"name":"crucible"2023-09-22T23:08:05.563122748Z",","level":30hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":","timedownstairs"":"}
2023-09-22T23:08:05.563142396Z","hostname":"{ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
"msg"{:""msg":"Extent 0 close id:1002 Failed: Error: bad","[0] client skip 4 in process jobs because fault"v":0,,""name"v:""crucible":,"0level",:"50name":"crucible","level":30,"time":"2023-09-22T23:08:05.563174379Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{,""msg":"time":"RE:0 Wait for result from reopen command 1003:4","v":2023-09-22T23:08:05.563177728Z"0,",name":""cruciblehostname"",":level":"30ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","}time":"
2023-09-22T23:08:05.563200623Z","hostname":"{ip-10-150-1-74.us-west-2.compute.internal","pid":4291"}
msg":"[0] changed 0 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.563226039Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[0] 4d2d5af5-94fb-4532-8477-38add7c548e4 (b46ad456-c0c5-43cb-8b84-fa6016b7a39b) LiveRepair Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.563266993Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from LiveRepair to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.563299186Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from reopen command 1003:4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.563335984Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Extent 0 close id:1003 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:05.563373651Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Bailing with error","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:05.56341323Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible stats registered with UUID: 46ff01ca-5774-41c5-a0d0-e4852575769b","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.563787731Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 46ff01ca-5774-41c5-a0d0-e4852575769b has session id: 7db0feac-d135-4774-a085-fa438156d01a","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.56382275Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 46ff01ca-5774-41c5-a0d0-e4852575769b (aeed4c9e-4d74-44ae-a346-e2cd02961908) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.563845225Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.563863967Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 46ff01ca-5774-41c5-a0d0-e4852575769b (aeed4c9e-4d74-44ae-a346-e2cd02961908) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.563884585Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.56390289Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 46ff01ca-5774-41c5-a0d0-e4852575769b (aeed4c9e-4d74-44ae-a346-e2cd02961908) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.563922984Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.563940976Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 46ff01ca-5774-41c5-a0d0-e4852575769b (aeed4c9e-4d74-44ae-a346-e2cd02961908) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.563961095Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.563979094Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 46ff01ca-5774-41c5-a0d0-e4852575769b (aeed4c9e-4d74-44ae-a346-e2cd02961908) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.563999237Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.564017452Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 46ff01ca-5774-41c5-a0d0-e4852575769b (aeed4c9e-4d74-44ae-a346-e2cd02961908) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.564037498Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.56405538Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 46ff01ca-5774-41c5-a0d0-e4852575769b (aeed4c9e-4d74-44ae-a346-e2cd02961908) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.56407528Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.564093071Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 46ff01ca-5774-41c5-a0d0-e4852575769b (aeed4c9e-4d74-44ae-a346-e2cd02961908) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.56411322Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.564131077Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 46ff01ca-5774-41c5-a0d0-e4852575769b (aeed4c9e-4d74-44ae-a346-e2cd02961908) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.564157748Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.564177145Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"46ff01ca-5774-41c5-a0d0-e4852575769b is now active with session: aeed4c9e-4d74-44ae-a346-e2cd02961908","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.56419668Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 46ff01ca-5774-41c5-a0d0-e4852575769b (aeed4c9e-4d74-44ae-a346-e2cd02961908) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.564216879Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.564234869Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 46ff01ca-5774-41c5-a0d0-e4852575769b (aeed4c9e-4d74-44ae-a346-e2cd02961908) Faulted Active Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.564255278Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.564273437Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 46ff01ca-5774-41c5-a0d0-e4852575769b (aeed4c9e-4d74-44ae-a346-e2cd02961908) LiveRepairReady Active Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.564293794Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.564312131Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for Close + ReOpen jobs","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.564329821Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.564348135Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 close id:1000 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.564366012Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from close command 1000:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.56438414Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] flow control start ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:05.603501839Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
test dummy_downstairs_tests::protocol_test::test_flow_control ... ok
{"msg":"Crucible stats registered with UUID: dd353b52-59c8-4b1c-b796-3ac33420495a","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.6762143Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible dd353b52-59c8-4b1c-b796-3ac33420495a has session id: b4f4e674-2022-4abb-84a7-f6e9d210c7c0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.676253412Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] dd353b52-59c8-4b1c-b796-3ac33420495a (6a5f3c94-1536-4e0f-b6e8-b72df1939203) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.676276776Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.676302573Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] dd353b52-59c8-4b1c-b796-3ac33420495a (6a5f3c94-1536-4e0f-b6e8-b72df1939203) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.676324749Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.676343229Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] dd353b52-59c8-4b1c-b796-3ac33420495a (6a5f3c94-1536-4e0f-b6e8-b72df1939203) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.676363516Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.676381797Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] dd353b52-59c8-4b1c-b796-3ac33420495a (6a5f3c94-1536-4e0f-b6e8-b72df1939203) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.676401876Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.676420018Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] dd353b52-59c8-4b1c-b796-3ac33420495a (6a5f3c94-1536-4e0f-b6e8-b72df1939203) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.67644056Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.676458939Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] dd353b52-59c8-4b1c-b796-3ac33420495a (6a5f3c94-1536-4e0f-b6e8-b72df1939203) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.676479013Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.67649684Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] dd353b52-59c8-4b1c-b796-3ac33420495a (6a5f3c94-1536-4e0f-b6e8-b72df1939203) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.676516917Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.676534647Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] dd353b52-59c8-4b1c-b796-3ac33420495a (6a5f3c94-1536-4e0f-b6e8-b72df1939203) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.676555031Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.676573017Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] dd353b52-59c8-4b1c-b796-3ac33420495a (6a5f3c94-1536-4e0f-b6e8-b72df1939203) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.676593236Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.676611342Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"dd353b52-59c8-4b1c-b796-3ac33420495a is now active with session: 6a5f3c94-1536-4e0f-b6e8-b72df1939203","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.676630903Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] dd353b52-59c8-4b1c-b796-3ac33420495a (6a5f3c94-1536-4e0f-b6e8-b72df1939203) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.676655606Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.67667444Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] dd353b52-59c8-4b1c-b796-3ac33420495a (6a5f3c94-1536-4e0f-b6e8-b72df1939203) Faulted Active Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.676695028Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.676713633Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] dd353b52-59c8-4b1c-b796-3ac33420495a (6a5f3c94-1536-4e0f-b6e8-b72df1939203) LiveRepairReady Active Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.676734142Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.676752536Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for Close + ReOpen jobs","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.676770365Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.676788687Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 close id:1000 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.676806594Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from close command 1000:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:05.676824939Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for 3 jobs (currently 2)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:06.437400278Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"No repair needed for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:06.437456249Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:06.437478616Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] DS Reports error Err(GenericError(\"bad\")) on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentFlushClose { dependencies: [], extent: 0, flush_number: 0, gen_number: 0, source_downstairs: ClientId(1), repair_downstairs: [ClientId(0)] }, state: ClientData([Done, InProgress, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:06.561199646Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] Reports error GenericError(\"bad\") on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentFlushClose { dependencies: [], extent: 0, flush_number: 0, gen_number: 0, source_downstairs: ClientId(1), repair_downstairs: [ClientId(0)] }, state: ClientData([Done, InProgress, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:06.561247322Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] client skip 2 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:06.561277503Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:06.5612984Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] a539c30f-2c8a-4e0b-a419-6c092883f4c8 (2bb8288f-73e5-46ca-83e3-b652d8887f09) LiveRepair Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:06.561321077Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:06.561339356Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now ACK the close job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:06.561356843Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for 3 jobs (currently 2)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:06.561374517Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Extent 0 close id:1000 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:06.561392531Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] client skip 2 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:06.561410672Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[0] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:06.561429729Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[0] a539c30f-2c8a-4e0b-a419-6c092883f4c8 (2bb8288f-73e5-46ca-83e3-b652d8887f09) LiveRepair Faulted Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:06.561450962Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from LiveRepair to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:06.56146916Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:06.561487698Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now ACK the close job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:06.563598577Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for 3 jobs (currently 2)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:06.563625517Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Repair for extent 0 s:2 d:[ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:06.563650858Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:06.563675184Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for 3 jobs (currently 2)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:06.564327749Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"No repair needed for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:06.564365635Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:06.564388128Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now ACK the close job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:06.67795702Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for 3 jobs (currently 2)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:06.677997769Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"No repair needed for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:06.678033963Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:06.678071576Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:07.437775454Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:07.43783366Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:07.562739823Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Extent 0 close id:1001 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:07.562798905Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:07.562842249Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now ACK the repair job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:07.564024332Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:07.564075664Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:07.564118595Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Finally, move the ReOpen job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:07.564264524Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now ACK the reopen job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:07.564316728Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Bailing with error","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:07.56435502Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible stats registered with UUID: 60f418b7-1c68-4219-b2c6-8690095ef46b","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:07.565006246Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 60f418b7-1c68-4219-b2c6-8690095ef46b has session id: ca7de67c-586a-48a2-86f2-35661d12ff50","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:07.565044033Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 60f418b7-1c68-4219-b2c6-8690095ef46b (b7455263-b441-43aa-ab5d-bc91879214ec) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:07.565067211Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:07.565086312Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 60f418b7-1c68-4219-b2c6-8690095ef46b (b7455263-b441-43aa-ab5d-bc91879214ec) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:07.565107135Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:07.565125592Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 60f418b7-1c68-4219-b2c6-8690095ef46b (b7455263-b441-43aa-ab5d-bc91879214ec) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:07.565155989Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:07.56517577Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 60f418b7-1c68-4219-b2c6-8690095ef46b (b7455263-b441-43aa-ab5d-bc91879214ec) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:07.565196351Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:07.565214218Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 60f418b7-1c68-4219-b2c6-8690095ef46b (b7455263-b441-43aa-ab5d-bc91879214ec) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:07.565234445Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:07.565252426Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 60f418b7-1c68-4219-b2c6-8690095ef46b (b7455263-b441-43aa-ab5d-bc91879214ec) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:07.565272634Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:07.56529053Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 60f418b7-1c68-4219-b2c6-8690095ef46b (b7455263-b441-43aa-ab5d-bc91879214ec) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:07.565310551Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:07.565328329Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 60f418b7-1c68-4219-b2c6-8690095ef46b (b7455263-b441-43aa-ab5d-bc91879214ec) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:07.565348439Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:07.565366446Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 60f418b7-1c68-4219-b2c6-8690095ef46b (b7455263-b441-43aa-ab5d-bc91879214ec) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:07.565386693Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:07.565404607Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"60f418b7-1c68-4219-b2c6-8690095ef46b is now active with session: b7455263-b441-43aa-ab5d-bc91879214ec","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:07.565424306Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 60f418b7-1c68-4219-b2c6-8690095ef46b (b7455263-b441-43aa-ab5d-bc91879214ec) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:07.565444278Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:07.565462031Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 60f418b7-1c68-4219-b2c6-8690095ef46b (b7455263-b441-43aa-ab5d-bc91879214ec) Faulted Active Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:07.565482465Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:07.565505395Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 60f418b7-1c68-4219-b2c6-8690095ef46b (b7455263-b441-43aa-ab5d-bc91879214ec) LiveRepairReady Active Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:07.56552715Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:07.565545456Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for Close + ReOpen jobs","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:07.565563349Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:07.565581615Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 close id:1000 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:07.565599425Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from close command 1000:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:07.56561756Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:07.56581135Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:07.56585969Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] DS Reports error Err(GenericError(\"bad\")) on job 1001, DownstairsIO { ds_id: JobId(1001), guest_id: 2, work: ExtentLiveNoOp { dependencies: [JobId(1000)] }, state: ClientData([InProgress, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:07.679371877Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[0] Reports error GenericError(\"bad\") on job 1001, DownstairsIO { ds_id: JobId(1001), guest_id: 2, work: ExtentLiveNoOp { dependencies: [JobId(1000)] }, state: ClientData([InProgress, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:07.679448806Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[0] client skip 3 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:07.679495507Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[0] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:07.679538787Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[0] dd353b52-59c8-4b1c-b796-3ac33420495a (6a5f3c94-1536-4e0f-b6e8-b72df1939203) LiveRepair Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:07.679583766Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from LiveRepair to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:07.679625153Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:07.679731889Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Extent 0 close id:1001 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:07.679762697Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:07.679782387Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now move the NoOp job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.564033982Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now ACK the NoOp job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.564115148Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Finally, move the ReOpen job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.564205075Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now ACK the Reopen job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.564247713Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Extent 0 close id:1002 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:08.564281043Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from reopen command 1003:4"----------------------------------------------------------------
,"v":0,"name":"crucible" Crucible gen:0 GIO:true work queues:  Upstairs:0  downstairs:4
,"level":30GW_ID      ACK  DSID    TYPE  BLOCKS  DS:0  DS:1  DS:2  REPLAY
    1    Acked  1000  FClose       0,"time":  Done"   Err  Done  false2023-09-22T23:08:08.564325154Z
","hostname    2    Acked  1001    NoOp       0":  Skip"  Skip  Done  false
ip-10-150-1-74.us-west-2.compute.internal","    3    Acked  1002    NoOp       0pid"  Skip:  Skip4291  Done  false
}
    4    Acked  1003  Reopen       0  Skip  Skip  Done{  false
"   STATES      DS:0   DS:1   DS:2   TOTAL
msg":    New        "   0      0   Extent 0 close id:1003 Failed: Error: bad   0   "   0
,"    Sent       v"   0   :   0   0   0   ,   0
"    Done       name   1   "   0   :"   4   crucible   5
"    Skipped    ,   3   "   3   level"   0   :   6
50    Error         0      1      0      1
Last Flush: 0 0 0 
Downstairs last five completed:,"
time":"2023-09-22T23:08:08.564383122Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{Upstairs last five completed:  "msg":"    4    3RE:0 Bailing with error    2"    1,
"v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:08.564416629Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible stats registered with UUID: 3d402dde-4b42-442d-92a4-920632ce6a71","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.564934126Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 3d402dde-4b42-442d-92a4-920632ce6a71 has session id: df6e2135-aea0-4363-80e7-c85916e908da","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.564973477Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 3d402dde-4b42-442d-92a4-920632ce6a71 (8170d26c-f851-4a07-85f6-cfb52781a824) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.564997154Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.565016445Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 3d402dde-4b42-442d-92a4-920632ce6a71 (8170d26c-f851-4a07-85f6-cfb52781a824) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.565037286Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.565055839Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 3d402dde-4b42-442d-92a4-920632ce6a71 (8170d26c-f851-4a07-85f6-cfb52781a824) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.565088386Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.565108151Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 3d402dde-4b42-442d-92a4-920632ce6a71 (8170d26c-f851-4a07-85f6-cfb52781a824) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.565128601Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.565146475Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 3d402dde-4b42-442d-92a4-920632ce6a71 (8170d26c-f851-4a07-85f6-cfb52781a824) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.565166835Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.565184795Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 3d402dde-4b42-442d-92a4-920632ce6a71 (8170d26c-f851-4a07-85f6-cfb52781a824) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.565204836Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.565222666Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 3d402dde-4b42-442d-92a4-920632ce6a71 (8170d26c-f851-4a07-85f6-cfb52781a824) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.565242478Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.565260193Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 3d402dde-4b42-442d-92a4-920632ce6a71 (8170d26c-f851-4a07-85f6-cfb52781a824) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.565280467Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.565298517Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 3d402dde-4b42-442d-92a4-920632ce6a71 (8170d26c-f851-4a07-85f6-cfb52781a824) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.565318686Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.565336546Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"3d402dde-4b42-442d-92a4-920632ce6a71 is now active with session: 8170d26c-f851-4a07-85f6-cfb52781a824","v":0,"name":"crucible","level":30{,"time":""2023-09-22T23:08:08.565355946Z",msg""hostname":":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291Now move the NoOp job forward"}
,"v"{:"0msg":","name":"crucible","[0] 3d402dde-4b42-442d-92a4-920632ce6a71 (8170d26c-f851-4a07-85f6-cfb52781a824) Active Active Active ds_transition to Faulted"level,"":v":030,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.565384438Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":",2023-09-22T23:08:08.565383032Z""time":","2023-09-22T23:08:08.565402135Z"hostname",":hostname":""ip-10-150-1-74.us-west-2.compute.internal","pid":4291ip-10-150-1-74.us-west-2.compute.internal"}
,"pid"{:4291"msg":"}
[0] 3d402dde-4b42-442d-92a4-920632ce6a71 (8170d26c-f851-4a07-85f6-cfb52781a824) Faulted Active Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":{30"msg":"Now ACK the NoOp job","v":0,"name":","timecrucible"":","level2023-09-22T23:08:08.565437451Z"",:"30hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.565450665Z","hostname":","time":"ip-10-150-1-74.us-west-2.compute.internal"2023-09-22T23:08:08.565461726Z",,""hostname":"pid":4291ip-10-150-1-74.us-west-2.compute.internal","}pid":
4291}
{{"msg"":"msg":"Finally, move the ReOpen job forward","[0] 3d402dde-4b42-442d-92a4-920632ce6a71 (8170d26c-f851-4a07-85f6-cfb52781a824) LiveRepairReady Active Active ds_transition to LiveRepair"v":,"0v",:"0,"name":name":""crucible","crucible"level":,"30level":30,"time":"2023-09-22T23:08:08.565499169Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
,"{time":""msg":"2023-09-22T23:08:08.565500374Z"[0] Transition from LiveRepairReady to LiveRepair",,""v":0hostname",:""name":"crucible","level":30ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
,"time":"{2023-09-22T23:08:08.565522731Z","hostname":""msg":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291Now ACK the repair job"}
,"v"{:0",msg":""name"Waiting for Close + ReOpen jobs":,""v":crucible"0,,""namelevel""::"30crucible","level":30,"time":"2023-09-22T23:08:08.565551352Z",",hostname":""time":ip-10-150-1-74.us-west-2.compute.internal"","pid":4291}
2023-09-22T23:08:08.565550391Z"{,""msghostname"":":"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]","v":ip-10-150-1-74.us-west-2.compute.internal"0,,""name":"pid"crucible":,"4291level":30}
{,"time":""msg":"2023-09-22T23:08:08.565580673Z","hostname":"RE:0 Wait for result from reopen command 1003:4"ip-10-150-1-74.us-west-2.compute.internal",",pid":"4291v"}:
0,"{name":""msg":"crucible","RE:0 close id:1000 queued, notify DS"level",":v":300,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.565607201Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal",",pid":"4291time"}:"
2023-09-22T23:08:08.565605255Z"{,""hostname"msg":":"RE:0 Wait for result from close command 1000:1","v":0,ip-10-150-1-74.us-west-2.compute.internal"",name":""crucible"pid",":level":429130}
,"time":"2023-09-22T23:08:08.565637361Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible stats registered with UUID: 125fefd1-90b0-4b49-b834-ad2ad49681ef","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.566069246Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 125fefd1-90b0-4b49-b834-ad2ad49681ef has session id: 19e8d39b-da45-43b4-8430-bfa6fd4ee3df","v":0,"name":"crucible","level":30{,"time":"2023-09-22T23:08:08.566100764Z","hostname":""ip-10-150-1-74.us-west-2.compute.internal","pid":msg4291"}:
"{Now move the NoOp job forward""msg":","v":0,"name"[0] 125fefd1-90b0-4b49-b834-ad2ad49681ef (f48d1731-e4d3-4e1d-bdf3-dfdfd3dd7a2a) New New New ds_transition to WaitActive:"","crucible"v":0,,""name":"level"crucible":,"level":3030,"time":"2023-09-22T23:08:08.566131884Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.566132377Z",","hostnametime":"":"2023-09-22T23:08:08.566150896Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal",ip-10-150-1-74.us-west-2.compute.internal"","pidpid""::42914291}
}
{"msg":"[0] 125fefd1-90b0-4b49-b834-ad2ad49681ef (f48d1731-e4d3-4e1d-bdf3-dfdfd3dd7a2a) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.566181933Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.566206552Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 125fefd1-90b0-4b49-b834-ad2ad49681ef (f48d1731-e4d3-4e1d-bdf3-dfdfd3dd7a2a) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.566228187Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.566246164Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 125fefd1-90b0-4b49-b834-ad2ad49681ef (f48d1731-e4d3-4e1d-bdf3-dfdfd3dd7a2a) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30{,"time":"2023-09-22T23:08:08.566266243Z","hostname":""msgip-10-150-1-74.us-west-2.compute.internal"",":pid":"4291}
[1] DS Reports error Err(GenericError({\""badmsg":"\"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.56628936Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
)) on job 1003, DownstairsIO { ds_id: JobId(1003), guest_id: 4, work: ExtentLiveReopen { dependencies: [JobId(1000), JobId(1001), JobId(1002)], extent: 0 }, state: ClientData([Done, InProgress, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }"{"msg":","v":0,"name":[1] 125fefd1-90b0-4b49-b834-ad2ad49681ef (f48d1731-e4d3-4e1d-bdf3-dfdfd3dd7a2a) Active WaitActive New ds_transition to WaitQuorum"","vcrucible"":0,","name":level"":crucible","50level":30,"time":"2023-09-22T23:08:08.566315204Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.566315459Z","hostname":","time":"ip-10-150-1-74.us-west-2.compute.internal"2023-09-22T23:08:08.566333484Z",,""pid"hostname":":4291ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"}":
"{downstairs""msg"}:"
[1] 125fefd1-90b0-4b49-b834-ad2ad49681ef (f48d1731-e4d3-4e1d-bdf3-dfdfd3dd7a2a) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30{"msg":","time":"[1] Reports error GenericError(\"2023-09-22T23:08:08.566362524Z"bad\","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30) on job 1003, DownstairsIO { ds_id: JobId(1003), guest_id: 4, work: ExtentLiveReopen { dependencies: [JobId(1000), JobId(1001), JobId(1002)], extent: 0 }, state: ClientData([Done, InProgress, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }",","v"time":":02023-09-22T23:08:08.566382492Z",,""hostname":name"":"crucible"ip-10-150-1-74.us-west-2.compute.internal",,""pid":level"4291:}
50{"msg":"[2] 125fefd1-90b0-4b49-b834-ad2ad49681ef (f48d1731-e4d3-4e1d-bdf3-dfdfd3dd7a2a) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.56641019Z",",hostname"":"time":"ip-10-150-1-74.us-west-2.compute.internal","pid":2023-09-22T23:08:08.566402919Z4291"}
,"{hostname":""msg":"[2] Transition from New to WaitActive"ip-10-150-1-74.us-west-2.compute.internal,""v":0,",name"":"pid"crucible":,"4291level":30,"":"downstairs"}
,"time":"2023-09-22T23:08:08.566436913Z","hostname":"{ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
"msg"{:""msg":"[1] client skip 4 in process jobs because fault","v":0[2] 125fefd1-90b0-4b49-b834-ad2ad49681ef (f48d1731-e4d3-4e1d-bdf3-dfdfd3dd7a2a) Active Active WaitActive ds_transition to WaitQuorum",,""vname""::0","namecrucible":""crucible",,""level":level30":30,"time":"2023-09-22T23:08:08.566467813Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
,"time"{:""msg":"2023-09-22T23:08:08.566469192Z","[2] Transition from WaitActive to WaitQuorum"hostname,"v""::"0,"name":"crucible","ip-10-150-1-74.us-west-2.compute.internal"level":,30"pid":4291,"":"downstairs"}
,"time":"2023-09-22T23:08:08.566497019Z","{hostname":"ip-10-150-1-74.us-west-2.compute.internal",""pid":msg"4291:"}
[1] changed 0 jobs to fault skipped",{"v":"0msg":","name":"crucible","level":[2] 125fefd1-90b0-4b49-b834-ad2ad49681ef (f48d1731-e4d3-4e1d-bdf3-dfdfd3dd7a2a) Active Active WaitQuorum ds_transition to Active"30,"v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.566526305Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":,"4291time"}:
"{"2023-09-22T23:08:08.566523851Zmsg":"",[2] Transition from WaitQuorum to Active"","vhostname""::0","name":"crucible","levelip-10-150-1-74.us-west-2.compute.internal":"30,"pid":4291,"":"downstairs"}
,"time":"2023-09-22T23:08:08.566554311Z","hostname":"{ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
"msg"{:""msg":"125fefd1-90b0-4b49-b834-ad2ad49681ef is now active with session: f48d1731-e4d3-4e1d-bdf3-dfdfd3dd7a2a","v":0,"name":"[1] 46ff01ca-5774-41c5-a0d0-e4852575769b (aeed4c9e-4d74-44ae-a346-e2cd02961908) LiveRepair Active Active ds_transition to Faulted"crucible",",level":"30v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.566581249Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 125fefd1-90b0-4b49-b834-ad2ad49681ef (f48d1731-e4d3-4e1d-bdf3-dfdfd3dd7a2a) Active Active Active ds_transition to Faulted","v":0,"name":"crucible",,""level":30time":"2023-09-22T23:08:08.566586315Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal",","time"pid":":42912023-09-22T23:08:08.56660357Z","}hostname":"
ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{{"msg"":"msg":"[2] Transition from Active to Faulted","v":0,[1] Transition from Active to Faulted""name":","cruciblev"":,"0level",":30name":"crucible","level":30{,"time":"2023-09-22T23:08:08.566635618Z","hostname":""ip-10-150-1-74.us-west-2.compute.internal","pid":msg4291"}:
",{"time":Waiting for 3 jobs (currently 2)"""msg":",2023-09-22T23:08:08.56663877Z"",v"":hostname":0[2] 125fefd1-90b0-4b49-b834-ad2ad49681ef (f48d1731-e4d3-4e1d-bdf3-dfdfd3dd7a2a) Active Active Faulted ds_transition to LiveRepairReady",,"""v":ip-10-150-1-74.us-west-2.compute.internalname"":,"0pid,":"4291name":}""
crucible"crucible",",{"level":level30"":msg":"30RE:0 Wait for result from reopen command 1003:4","v":0,"name":"crucible","level":,30"time":"2023-09-22T23:08:08.566678614Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":","time":"[2] Transition from Faulted to LiveRepairReady","2023-09-22T23:08:08.566685416Zv":"0,",hostname""name":":"crucible","ip-10-150-1-74.us-west-2.compute.internal,level""":time","30pid"::"4291}
2023-09-22T23:08:08.566681336Z"{,,"""hostnamemsgtime":""":2023-09-22T23:08:08.566708977Z":,"""hostname":"Extent 0 close id:1003 Failed: Error: bad","v":ip-10-150-1-74.us-west-2.compute.internal"0ip-10-150-1-74.us-west-2.compute.internal",",name",""pidpid":"":crucible"4291,":level4291}"
:50}
{"{msg":","time":""2023-09-22T23:08:08.566744483Z"[2] 125fefd1-90b0-4b49-b834-ad2ad49681ef (f48d1731-e4d3-4e1d-bdf3-dfdfd3dd7a2a) Active Active LiveRepairReady ds_transition to LiveRepair,"msg,""v:""":hostname":"0No repair needed for extent 0",ip-10-150-1-74.us-west-2.compute.internal",","name"v":""cruciblepid"::"4291,"}level":030
,"name"{:""cruciblemsg",",time":"":"2023-09-22T23:08:08.566772872Z"","level[0] client skip 4 in process jobs because fault"","hostname":"v":0:ip-10-150-1-74.us-west-2.compute.internal",,"30pid":"4291name":"}
crucible","{level":30"msg":"[2] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.566799186Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,,"""time":,":"time""downstairs":}"
2023-09-22T23:08:08.566804333Z"2023-09-22T23:08:08.56679358Z","{hostname","":hostname"msg":":""ip-10-150-1-74.us-west-2.compute.internal","[0] changed 0 jobs to fault skipped"pid"ip-10-150-1-74.us-west-2.compute.internal:,"4291",v":}0"
,"pidname{":""crucible":",4291msg":"",level"Waiting for Close + ReOpen jobs":":30"","downstairs"v":0,"}name"
:","crucibletime":"","level":2023-09-22T23:08:08.56685579Z{30","hostname":"ip-10-150-1-74.us-west-2.compute.internal",""pid":msg4291":,""":","downstairstime":"RE:0 Wait for result from repair command 1001:2""2023-09-22T23:08:08.566870132Z,}"
"v",":{hostname":""0msg":ip-10-150-1-74.us-west-2.compute.internal"",,""pid":name4291"[0] 46ff01ca-5774-41c5-a0d0-e4852575769b (aeed4c9e-4d74-44ae-a346-e2cd02961908) LiveRepair Faulted Active ds_transition to Faulted}":,""v":
crucible"0{,,""msg":""name":"levelRE:0 repair extent with ids 1000,1001,1002,1003 deps:[]crucible"","":level","30v"::030,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.566915318Z","hostname":","time":"ip-10-150-1-74.us-west-2.compute.internal","pid":2023-09-22T23:08:08.566917372Z4291","}hostname":"
ip-10-150-1-74.us-west-2.compute.internal",,""pid"time:"4291:{}""
msg":"{2023-09-22T23:08:08.566913829Z""[0] Transition from LiveRepair to Faultedmsg,""":"hostname",":v":0RE:0 close id:1000 queued, notify DS"",",name"ip-10-150-1-74.us-west-2.compute.internal"v":",0:""pid"crucible":,,"4291name":""crucible"level,"":}level":3030
,,""time":"time":"2023-09-22T23:08:08.566968757Z"2023-09-22T23:08:08.56696921Z,""hostname,"":"hostname":"ip-10-150-1-74.us-west-2.compute.internal","ip-10-150-1-74.us-west-2.compute.internal"pid":,4291"}pid":
4291}
{"{msg":""msg":"RE:0 Wait for result from close command 1000:1"RE:0 Bailing with error",,""v":v":00,,""namename":"":crucible"","crucible"level":40,"level":30,"time":","time":"2023-09-22T23:08:08.567008978Z","2023-09-22T23:08:08.567009867Z"hostname,"":"hostname":"ip-10-150-1-74.us-west-2.compute.internal","ip-10-150-1-74.us-west-2.compute.internal"pid":,"4291pid":}4291
}
{"msg":"Crucible stats registered with UUID: 2a9017ca-3786-4928-aa25-af146cb13064","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.567300857Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 2a9017ca-3786-4928-aa25-af146cb13064 has session id: af0da4c6-782e-4684-a546-1550f807b027","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.567330847Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 2a9017ca-3786-4928-aa25-af146cb13064 (41790776-3f0f-4ed5-8993-f2ed274b7319) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.567352897Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.567371695Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 2a9017ca-3786-4928-aa25-af146cb13064 (41790776-3f0f-4ed5-8993-f2ed274b7319) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.567392284Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.567410685Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 2a9017ca-3786-4928-aa25-af146cb13064 (41790776-3f0f-4ed5-8993-f2ed274b7319) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.567431113Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.567453196Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 2a9017ca-3786-4928-aa25-af146cb13064 (41790776-3f0f-4ed5-8993-f2ed274b7319) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.567473913Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.56749195Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 2a9017ca-3786-4928-aa25-af146cb13064 (41790776-3f0f-4ed5-8993-f2ed274b7319) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.567518078Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.567537535Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 2a9017ca-3786-4928-aa25-af146cb13064 (41790776-3f0f-4ed5-8993-f2ed274b7319) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.567557917Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.567576045Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 2a9017ca-3786-4928-aa25-af146cb13064 (41790776-3f0f-4ed5-8993-f2ed274b7319) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.567596155Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.56761408Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 2a9017ca-3786-4928-aa25-af146cb13064 (41790776-3f0f-4ed5-8993-f2ed274b7319) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.567634453Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.567652497Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 2a9017ca-3786-4928-aa25-af146cb13064 (41790776-3f0f-4ed5-8993-f2ed274b7319) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.567672679Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.567690591Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"2a9017ca-3786-4928-aa25-af146cb13064 is now active with session: 41790776-3f0f-4ed5-8993-f2ed274b7319","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.567710045Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 2a9017ca-3786-4928-aa25-af146cb13064 (41790776-3f0f-4ed5-8993-f2ed274b7319) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.56773018Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.567747955Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 2a9017ca-3786-4928-aa25-af146cb13064 (41790776-3f0f-4ed5-8993-f2ed274b7319) Faulted Active Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.567768251Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.567786375Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 2a9017ca-3786-4928-aa25-af146cb13064 (41790776-3f0f-4ed5-8993-f2ed274b7319) LiveRepairReady Active Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.567806727Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.56782489Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for Close + ReOpen jobs","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.567846991Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.56786636Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 close id:1000 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.567884292Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from close command 1000:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.567902484Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now move the NoOp job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.68075512Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now ACK the NoOp job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.680789424Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Finally, move the ReOpen job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.680809727Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now ACK the Reopen job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.680827805Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Extent 0 close id:1002 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:08.680861757Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from reopen command 1003:4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.680881719Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Extent 0 close id:1003 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:08.680900135Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Bailing with error","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:08.680918009Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible stats registered with UUID: 340c04f3-49b8-4ced-b263-4b0e2af7ead7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.681348405Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 340c04f3-49b8-4ced-b263-4b0e2af7ead7 has session id: d0ad791d-e99f-4e9b-a416-2c649bca391b","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.681381146Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 340c04f3-49b8-4ced-b263-4b0e2af7ead7 (85274e4a-b9a3-47aa-95cb-f70c1885651d) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.681403085Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.681422006Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 340c04f3-49b8-4ced-b263-4b0e2af7ead7 (85274e4a-b9a3-47aa-95cb-f70c1885651d) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.681442477Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.681460864Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 340c04f3-49b8-4ced-b263-4b0e2af7ead7 (85274e4a-b9a3-47aa-95cb-f70c1885651d) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.68148097Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.681499106Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 340c04f3-49b8-4ced-b263-4b0e2af7ead7 (85274e4a-b9a3-47aa-95cb-f70c1885651d) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.681526282Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.681545768Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 340c04f3-49b8-4ced-b263-4b0e2af7ead7 (85274e4a-b9a3-47aa-95cb-f70c1885651d) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.681566292Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.681584425Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 340c04f3-49b8-4ced-b263-4b0e2af7ead7 (85274e4a-b9a3-47aa-95cb-f70c1885651d) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.681604537Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.6816227Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 340c04f3-49b8-4ced-b263-4b0e2af7ead7 (85274e4a-b9a3-47aa-95cb-f70c1885651d) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.681643025Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.681660945Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 340c04f3-49b8-4ced-b263-4b0e2af7ead7 (85274e4a-b9a3-47aa-95cb-f70c1885651d) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.681681256Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.681699289Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 340c04f3-49b8-4ced-b263-4b0e2af7ead7 (85274e4a-b9a3-47aa-95cb-f70c1885651d) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.681719424Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.681737385Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"340c04f3-49b8-4ced-b263-4b0e2af7ead7 is now active with session: 85274e4a-b9a3-47aa-95cb-f70c1885651d","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.681756816Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 340c04f3-49b8-4ced-b263-4b0e2af7ead7 (85274e4a-b9a3-47aa-95cb-f70c1885651d) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.681777011Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.68179503Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 340c04f3-49b8-4ced-b263-4b0e2af7ead7 (85274e4a-b9a3-47aa-95cb-f70c1885651d) Faulted Active Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.681815509Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.681833624Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 340c04f3-49b8-4ced-b263-4b0e2af7ead7 (85274e4a-b9a3-47aa-95cb-f70c1885651d) LiveRepairReady Active Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.681858256Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.681877536Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for Close + ReOpen jobs","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.681895582Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.68191397Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 close id:1000 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.681932063Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from close command 1000:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:08.681950371Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] DS Reports error Err(GenericError(\"bad\")) on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentFlushClose { dependencies: [], extent: 0, flush_number: 0, gen_number: 0, source_downstairs: ClientId(1), repair_downstairs: [ClientId(0)] }, state: ClientData([Done, Done, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:09.566512607Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] Reports error GenericError(\"bad\") on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentFlushClose { dependencies: [], extent: 0, flush_number: 0, gen_number: 0, source_downstairs: ClientId(1), repair_downstairs: [ClientId(0)] }, state: ClientData([Done, Done, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:09.56660765Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] client skip 2 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:09.566632155Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:09.566651907Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] 3d402dde-4b42-442d-92a4-920632ce6a71 (8170d26c-f851-4a07-85f6-cfb52781a824) LiveRepair Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:09.566673857Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:09.566692217Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now ACK the close job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:09.566709902Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for 3 jobs (currently 2)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:09.566727898Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Extent 0 close id:1000 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:09.566746415Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] client skip 2 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:09.566764575Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[0] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:09.566792376Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[0] 3d402dde-4b42-442d-92a4-920632ce6a71 (8170d26c-f851-4a07-85f6-cfb52781a824) LiveRepair Active Faulted ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:09.566815276Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from LiveRepair to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:09.566833393Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:09.566886966Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:09.567068977Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:09.567123728Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now ACK the close job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:09.567745367Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for 3 jobs (currently 2)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:09.567773266Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Repair for extent 0 s:0 d:[ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:09.567874383Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:09.567902847Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for 3 jobs (currently 2)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:09.568823388Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"No repair needed for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:09.568849835Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:09.568871949Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now ACK the close job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:09.682838821Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for 3 jobs (currently 2)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:09.682877593Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"No repair needed for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:09.682897391Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:09.68291813Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
----------------------------------------------------------------
 Crucible gen:0 GIO:true work queues:  Upstairs:2  downstairs:4
GW_ID      ACK  DSID    TYPE  BLOCKS  DS:0  DS:1  DS:2  REPLAY
    1    Acked  1000  FClose       0  Done  Done  Done  false
    2    Acked  1001    NoOp       0  Done  Done  Done  false
    3 NotAcked  1002    NoOp       0   New   New   New  false
    4 NotAcked  1003  Reopen       0   New   New   New  false
   STATES      DS:0   DS:1   DS:2   TOTAL
    New           2      2      2      6
    Sent          0      0      0      0
    Done          2      2      2      6
    Skipped       0      0      0      0
    Error         0      0      0      0
Last Flush: 0 0 0 
Downstairs last five completed:
Upstairs last five completed:      2    1
{"msg":"Now move the NoOp job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:10.439168624Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Finally, move the ReOpen job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:10.439219915Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now ACK the reopen job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:10.439239325Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from reopen command 1003:4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:10.439257871Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"jobs are: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:10.439358434Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible stats registered with UUID: 471b8a98-0f08-4754-9135-7ad94157ce5e","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:10.439810702Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 471b8a98-0f08-4754-9135-7ad94157ce5e has session id: 6704f77c-c61f-4544-ad9e-8cb9ea762f74","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:10.439845955Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 471b8a98-0f08-4754-9135-7ad94157ce5e (c0eacee2-3501-40d4-8616-f8e3a6a14e2b) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:10.43986852Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:10.439887692Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 471b8a98-0f08-4754-9135-7ad94157ce5e (c0eacee2-3501-40d4-8616-f8e3a6a14e2b) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:10.439908042Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:10.439926288Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 471b8a98-0f08-4754-9135-7ad94157ce5e (c0eacee2-3501-40d4-8616-f8e3a6a14e2b) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:10.439946172Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:10.439964037Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 471b8a98-0f08-4754-9135-7ad94157ce5e (c0eacee2-3501-40d4-8616-f8e3a6a14e2b) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:10.439984033Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:10.44000185Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 471b8a98-0f08-4754-9135-7ad94157ce5e (c0eacee2-3501-40d4-8616-f8e3a6a14e2b) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:10.440022085Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:10.440039948Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 471b8a98-0f08-4754-9135-7ad94157ce5e (c0eacee2-3501-40d4-8616-f8e3a6a14e2b) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:10.440059842Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:10.440085405Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 471b8a98-0f08-4754-9135-7ad94157ce5e (c0eacee2-3501-40d4-8616-f8e3a6a14e2b) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:10.44010697Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:10.440124912Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 471b8a98-0f08-4754-9135-7ad94157ce5e (c0eacee2-3501-40d4-8616-f8e3a6a14e2b) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:10.440145182Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:10.440163228Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 471b8a98-0f08-4754-9135-7ad94157ce5e (c0eacee2-3501-40d4-8616-f8e3a6a14e2b) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:10.440183173Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:10.440200944Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"471b8a98-0f08-4754-9135-7ad94157ce5e is now active with session: c0eacee2-3501-40d4-8616-f8e3a6a14e2b","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:10.440220372Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 471b8a98-0f08-4754-9135-7ad94157ce5e (c0eacee2-3501-40d4-8616-f8e3a6a14e2b) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:10.440240266Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:10.440257985Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 471b8a98-0f08-4754-9135-7ad94157ce5e (c0eacee2-3501-40d4-8616-f8e3a6a14e2b) Active Faulted Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:10.440278147Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:10.440296183Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 471b8a98-0f08-4754-9135-7ad94157ce5e (c0eacee2-3501-40d4-8616-f8e3a6a14e2b) Active LiveRepairReady Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:10.440316486Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:10.440334513Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for Close + ReOpen jobs","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:10.440352206Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:10.440370492Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 close id:1000 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:10.44038817Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from close command 1000:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:10.440406384Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:10.567916055Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Extent 0 close id:1001 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:10.567948457Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:10.567967914Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now move the NoOp job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:10.568312457Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] DS Reports error Err(GenericError(\"bad\")) on job 1002, DownstairsIO { ds_id: JobId(1002), guest_id: 3, work: ExtentLiveNoOp { dependencies: [JobId(1000), JobId(1001)] }, state: ClientData([Done, InProgress, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:10.568351604Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] Reports error GenericError(\"bad\") on job 1002, DownstairsIO { ds_id: JobId(1002), guest_id: 3, work: ExtentLiveNoOp { dependencies: [JobId(1000), JobId(1001)] }, state: ClientData([Done, InProgress, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:10.56838284Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] client skip 4 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:10.568403007Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:10.568422044Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] 60f418b7-1c68-4219-b2c6-8690095ef46b (b7455263-b441-43aa-ab5d-bc91879214ec) LiveRepair Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:10.568443576Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:10.56846168Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Extent 0 close id:1002 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:10.568479756Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] client skip 4 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:10.568497875Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[0] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:10.568516779Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[0] 60f418b7-1c68-4219-b2c6-8690095ef46b (b7455263-b441-43aa-ab5d-bc91879214ec) LiveRepair Faulted Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:10.56853813Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from LiveRepair to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:10.568556294Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from reopen command 1003:4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:10.568574443Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now ACK the repair job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:10.569208041Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:10.569235232Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:10.569319974Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:10.570143732Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:10.57018633Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] DS Reports error Err(GenericError(\"bad\")) on job 1001, DownstairsIO { ds_id: JobId(1001), guest_id: 2, work: ExtentLiveNoOp { dependencies: [JobId(1000)] }, state: ClientData([Done, InProgress, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:10.684241983Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] Reports error GenericError(\"bad\") on job 1001, DownstairsIO { ds_id: JobId(1001), guest_id: 2, work: ExtentLiveNoOp { dependencies: [JobId(1000)] }, state: ClientData([Done, InProgress, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:10.684290665Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] client skip 3 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:10.684312759Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:10.684332093Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] 340c04f3-49b8-4ced-b263-4b0e2af7ead7 (85274e4a-b9a3-47aa-95cb-f70c1885651d) LiveRepair Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:10.684354214Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:10.6843728Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:10.684390963Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Extent 0 close id:1001 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:10.684408944Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] client skip 3 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:10.6844272Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[0] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:10.684446362Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[0] 340c04f3-49b8-4ced-b263-4b0e2af7ead7 (85274e4a-b9a3-47aa-95cb-f70c1885651d) LiveRepair Faulted Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:10.684467873Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from LiveRepair to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:10.684492861Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:10.684512513Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for 3 jobs (currently 2)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.441460433Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"No repair needed for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.441546419Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.441593448Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now move the NoOp job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.569291951Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now ACK the NoOp job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.569330141Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Finally, move the ReOpen job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.56935059Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now ACK the Reopen job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.569369145Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Extent 0 close id:1002 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.569387709Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from reopen command 1003:4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.569406342Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Extent 0 close id:1003 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.569424764Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Bailing with error","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:11.569442501Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
----------------------------------------------------------------
 Crucible gen:0 GIO:true work queues:  Upstairs:0  downstairs:4
GW_ID      ACK  DSID    TYPE  BLOCKS  DS:0  DS:1  DS:2  REPLAY
    1    Acked  1000  FClose       0  Done  Done   Err  false
    2    Acked  1001    NoOp       0  Skip  Done  Skip  false
    3    Acked  1002    NoOp       0  Skip  Done  Skip  false
    4    Acked  1003  Reopen       0  Skip  Done  Skip  false
   STATES      DS:0   DS:1   DS:2   TOTAL
    New           0      0      0      0
    Sent          0      0      0      0
    Done          1      4      0      5
    Skipped       3      0      3      6
    Error         0      0      1      1
Last Flush: 0 0 0 
Downstairs last five completed:
Upstairs last five completed:      4    3    2    1
{"msg":"Crucible stats registered with UUID: bb7a424d-9b30-4a13-96a7-a41c0f6628b1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.570106175Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible bb7a424d-9b30-4a13-96a7-a41c0f6628b1 has session id: 99f11c25-87f7-4d78-9e1c-2f89f88bf53d","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.570141542Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] bb7a424d-9b30-4a13-96a7-a41c0f6628b1 (5a93f758-ea9a-4a78-ad67-6f82f309f96e) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.570163931Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.570182337Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] bb7a424d-9b30-4a13-96a7-a41c0f6628b1 (5a93f758-ea9a-4a78-ad67-6f82f309f96e) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.570211946Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.570231674Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] bb7a424d-9b30-4a13-96a7-a41c0f6628b1 (5a93f758-ea9a-4a78-ad67-6f82f309f96e) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.570251668Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.570269453Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] bb7a424d-9b30-4a13-96a7-a41c0f6628b1 (5a93f758-ea9a-4a78-ad67-6f82f309f96e) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.570289118Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.570306792Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] bb7a424d-9b30-4a13-96a7-a41c0f6628b1 (5a93f758-ea9a-4a78-ad67-6f82f309f96e) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.570326818Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.570344686Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] bb7a424d-9b30-4a13-96a7-a41c0f6628b1 (5a93f758-ea9a-4a78-ad67-6f82f309f96e) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.570364609Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.570382257Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] bb7a424d-9b30-4a13-96a7-a41c0f6628b1 (5a93f758-ea9a-4a78-ad67-6f82f309f96e) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.570402297Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.570419934Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] bb7a424d-9b30-4a13-96a7-a41c0f6628b1 (5a93f758-ea9a-4a78-ad67-6f82f309f96e) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.570440272Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.570458034Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] bb7a424d-9b30-4a13-96a7-a41c0f6628b1 (5a93f758-ea9a-4a78-ad67-6f82f309f96e) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.570493817Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.570514887Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"bb7a424d-9b30-4a13-96a7-a41c0f6628b1 is now active with session: 5a93f758-ea9a-4a78-ad67-6f82f309f96e","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.570534507Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] bb7a424d-9b30-4a13-96a7-a41c0f6628b1 (5a93f758-ea9a-4a78-ad67-6f82f309f96e) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.570559821Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.570578664Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] bb7a424d-9b30-4a13-96a7-a41c0f6628b1 (5a93f758-ea9a-4a78-ad67-6f82f309f96e) Active Faulted Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.570598835Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.570616859Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] bb7a424d-9b30-4a13-96a7-a41c0f6628b1 (5a93f758-ea9a-4a78-ad67-6f82f309f96e) Active LiveRepairReady Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.570637054Z","hostname":"{ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{""msg":"msg":"[1] Transition from LiveRepairReady to LiveRepair","v":0,"nameNow move the NoOp job forward"":"crucible",",level":30"v":0,"name":"crucible","level",":time":"302023-09-22T23:08:11.570658866Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for Close + ReOpen jobs","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.570678103Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
,"{time":""msg":"2023-09-22T23:08:11.57066757Z"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]","v":,0","name"hostname:""crucible":,""level":30ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
,"time":"2023-09-22T23:08:11.570703077Z","hostname":"{ip-10-150-1-74.us-west-2.compute.internal","pid":4291"}msg"
:"{Now ACK the NoOp job""msg":","v":RE:0 close id:1000 queued, notify DS"0,",v"":0name",":name"":"cruciblecrucible"",",level":"30level":30,"time":"2023-09-22T23:08:11.5707386Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
,{""time"msg:"":"RE:0 Wait for result from close command 1000:12023-09-22T23:08:11.570739934Z"","v":,0","name":"hostnamecrucible"",":level"":30ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
,"time":"2023-09-22T23:08:11.570767051Z{","hostname":""msg":ip-10-150-1-74.us-west-2.compute.internal"","pid":4291}
Finally, move the ReOpen job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.570789712Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now ACK the repair job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.570822494Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from reopen command 1003:4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.570855301Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test live_repair::repair_test::test_repair_extent_do_repair_all ... ok
{"msg":"Now move the NoOp job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.571555575Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{{"msg":"Crucible stats registered with UUID: a8ba4201-0b23-4992-9a71-32e39bda2e5f"","msg"v:"":0,"name":"crucible"[2] DS Reports error Err(GenericError(,"\"level":bad30\",")) on job 1003, DownstairsIO { ds_id: JobId(1003), guest_id: 4, work: ExtentLiveReopen { dependencies: [JobId(1000), JobId(1001), JobId(1002)], extent: 0 }, state: ClientData([Done, Done, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }"time":"2023-09-22T23:08:11.571665256Z",","v"hostname":":0ip-10-150-1-74.us-west-2.compute.internal",,""pid":name"4291:"}crucible"
,"level":50{"msg":"Crucible a8ba4201-0b23-4992-9a71-32e39bda2e5f has session id: 3f4228c8-9c93-483f-af02-1dcfce7a1e85","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.571706091Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,}"
time":"{2023-09-22T23:08:11.571698706Z""msg":","hostname":"[0] a8ba4201-0b23-4992-9a71-32e39bda2e5f (68aef9ae-886d-4c85-a384-fc1ac4cdd4e7) New New New ds_transition to WaitActive"ip-10-150-1-74.us-west-2.compute.internal",",v"":0pid",":name":4291"crucible","level":,30"":"downstairs"}
,"time":"2023-09-22T23:08:11.571741391Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","{pid":4291}
"msg"{:""msg":"[2] Reports error GenericError(\"[0] Transition from New to WaitActive"bad\","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.571765371Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
) on job 1003, DownstairsIO { ds_id: JobId(1003), guest_id: 4, work: ExtentLiveReopen { dependencies: [JobId(1000), JobId(1001), JobId(1002)], extent: 0 }, state: ClientData([Done, Done, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }"{",msg":""v":0,"name":"[0] a8ba4201-0b23-4992-9a71-32e39bda2e5f (68aef9ae-886d-4c85-a384-fc1ac4cdd4e7) WaitActive New New ds_transition to WaitQuorumcrucible"",",v":0","levelname":"":crucible"50,"level":30,"time":"2023-09-22T23:08:11.571793075Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum",","v"time"::0","name":"crucible",2023-09-22T23:08:11.571792618Z""level":30,"hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,,""time":":""downstairs2023-09-22T23:08:11.571815443Z"","hostname}":"
ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{{"msg":""msg":"[0] a8ba4201-0b23-4992-9a71-32e39bda2e5f (68aef9ae-886d-4c85-a384-fc1ac4cdd4e7) WaitQuorum New New ds_transition to Active","v":0,"[2] client skip 4 in process jobs because fault"name":","cruciblev"":,"0level":,30"name":"crucible","level":30,"time":"2023-09-22T23:08:11.571848912Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.57185311Z","hostname":","time":"2023-09-22T23:08:11.57186721Z"ip-10-150-1-74.us-west-2.compute.internal,""hostname":","pid":ip-10-150-1-74.us-west-2.compute.internal4291","pid":4291,"}"
:"{downstairs""}msg"
:"{[1] a8ba4201-0b23-4992-9a71-32e39bda2e5f (68aef9ae-886d-4c85-a384-fc1ac4cdd4e7) Active New New ds_transition to WaitActive","v":0,"name":"crucible"","msg"level"::"30[2] changed 0 jobs to fault skipped","v":0,"name":"crucible",","level":time":"302023-09-22T23:08:11.571899656Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":","2023-09-22T23:08:11.571908238Ztime":""2023-09-22T23:08:11.571919426Z,"","hostname"hostname":":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291ip-10-150-1-74.us-west-2.compute.internal"}
,"pid"{:4291"msg":","":"downstairs"}
[1] a8ba4201-0b23-4992-9a71-32e39bda2e5f (68aef9ae-886d-4c85-a384-fc1ac4cdd4e7) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30{"msg":","time":"2023-09-22T23:08:11.571950002Z","hostname":"[2] 2a9017ca-3786-4928-aa25-af146cb13064 (41790776-3f0f-4ed5-8993-f2ed274b7319) LiveRepair Active Active ds_transition to Faulted"ip-10-150-1-74.us-west-2.compute.internal",",pid":"4291v"}:
0,{"name"":msg":""crucible"[1] Transition from WaitActive to WaitQuorum",,""v":level"0:,"name":"30crucible","level":30,"time":"2023-09-22T23:08:11.571978238Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":","time":"[1] a8ba4201-0b23-4992-9a71-32e39bda2e5f (68aef9ae-886d-4c85-a384-fc1ac4cdd4e7) Active WaitQuorum New ds_transition to Active","2023-09-22T23:08:11.571977588Z"v":0,",name":""crucible",hostname"":level"":30ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from Active to Faulted","v":0,"name":"crucible",,""level":time":"302023-09-22T23:08:11.572003813Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.572070113Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"}time"
:"2023-09-22T23:08:11.572087211Z","hostname":"{ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
"msg"{:""msg":"RE:0 Wait for result from reopen command 1003:4","v":0[2] a8ba4201-0b23-4992-9a71-32e39bda2e5f (68aef9ae-886d-4c85-a384-fc1ac4cdd4e7) Active Active New ds_transition to WaitActive",,""v":name"0:","namecrucible"":","crucible"level,""level"::3030,"time":"2023-09-22T23:08:11.572126268Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from New to WaitActive","v":0,,""name":"time"crucible":,""level":302023-09-22T23:08:11.572127107Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":,"4291time":"}2023-09-22T23:08:11.572148778Z
","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291{}
"{msg":""msg":"Extent 0 close id:1003 Failed: Error: bad","v":0[2] a8ba4201-0b23-4992-9a71-32e39bda2e5f (68aef9ae-886d-4c85-a384-fc1ac4cdd4e7) Active Active WaitActive ds_transition to WaitQuorum",,""v":name"0:,""name":"crucible"crucible",",level":"30level":50,"time":"2023-09-22T23:08:11.572183392Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v,":"0,"timename"":":crucible"","level":302023-09-22T23:08:11.572184872Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":,"4291time":"}2023-09-22T23:08:11.572205986Z
","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291{}
"msg"{:""msg":"[0] client skip 4 in process jobs because fault","v":0[2] a8ba4201-0b23-4992-9a71-32e39bda2e5f (68aef9ae-886d-4c85-a384-fc1ac4cdd4e7) Active Active WaitQuorum ds_transition to Active",,""v":name"0:,""name":"crucible"crucible",","level"level"::3030,"time":"2023-09-22T23:08:11.572240993Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active",",v":"0,"time"name":":"crucible","level":302023-09-22T23:08:11.572241737Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"time":","":"2023-09-22T23:08:11.57226238Z","downstairs"hostname":"}
ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{{"msg":""msg":"a8ba4201-0b23-4992-9a71-32e39bda2e5f is now active with session: 68aef9ae-886d-4c85-a384-fc1ac4cdd4e7",[0] changed 0 jobs to fault skipped""v":,"0v",":name":"0crucible",,""level":name30":"crucible","level":30,"time":"2023-09-22T23:08:11.572293756Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] a8ba4201-0b23-4992-9a71-32e39bda2e5f (68aef9ae-886d-4c85-a384-fc1ac4cdd4e7) Active Active Active ds_transition to Faulted","v":0,,""name":time"":crucible"","level":302023-09-22T23:08:11.572297341Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":,"4291time":","2023-09-22T23:08:11.572317785Z"",":hostname":""downstairs"ip-10-150-1-74.us-west-2.compute.internal","}pid":
4291}
{{"msg":"[1] Transition from Active to Faulted","v"":0msg",":name"":"crucible","level":30[0] 2a9017ca-3786-4928-aa25-af146cb13064 (41790776-3f0f-4ed5-8993-f2ed274b7319) LiveRepair Active Faulted ds_transition to Faulted","v":0,","time":"name":"2023-09-22T23:08:11.572348058Z"crucible",",hostname":""level":ip-10-150-1-74.us-west-2.compute.internal"30,"pid":4291}
{"msg":"[1] a8ba4201-0b23-4992-9a71-32e39bda2e5f (68aef9ae-886d-4c85-a384-fc1ac4cdd4e7) Active Faulted Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":","time":"2023-09-22T23:08:11.57236227Z"2023-09-22T23:08:11.572373044Z",","hostname":"hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291ip-10-150-1-74.us-west-2.compute.internal"}
,"pid"{:4291"msg":"}
[1] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30{"msg":"[0] Transition from LiveRepair to Faulted",",time":""v":2023-09-22T23:08:11.572398474Z"0,","hostname":name"":"crucible"ip-10-150-1-74.us-west-2.compute.internal",",pid":"4291level"}:
30{"msg":"[1] a8ba4201-0b23-4992-9a71-32e39bda2e5f (68aef9ae-886d-4c85-a384-fc1ac4cdd4e7) Active LiveRepairReady Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time,":""time2023-09-22T23:08:11.572431937Z"":","hostname":"2023-09-22T23:08:11.572423366Z"ip-10-150-1-74.us-west-2.compute.internal",,""pid":4291hostname":}"
ip-10-150-1-74.us-west-2.compute.internal",{"pid"":msg":4291"}[1] Transition from LiveRepairReady to LiveRepair
","v":0,"name":"crucible","level":30{"msg":"RE:0 Bailing with error","v":,"0time",:""name":"crucible","2023-09-22T23:08:11.572464377Z"level",:"40hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
,"time":"2023-09-22T23:08:11.572478638Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test live_repair::repair_test::test_repair_io_above_el_skipped ... ok
{"msg":"Crucible stats registered with UUID: e5daa207-6492-4466-92d0-cc614d6c56ce","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.573025101Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible e5daa207-6492-4466-92d0-cc614d6c56ce has session id: f6c82c27-cea7-4f69-8d6f-e4bb7ab6a3bb","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.573059144Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] e5daa207-6492-4466-92d0-cc614d6c56ce (8a6b2194-d5ad-4cea-9c66-5ded3e31a721) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.573081225Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.573099647Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] e5daa207-6492-4466-92d0-cc614d6c56ce (8a6b2194-d5ad-4cea-9c66-5ded3e31a721) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.573119854Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.573142268Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] e5daa207-6492-4466-92d0-cc614d6c56ce (8a6b2194-d5ad-4cea-9c66-5ded3e31a721) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.573178858Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.573211959Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] e5daa207-6492-4466-92d0-cc614d6c56ce (8a6b2194-d5ad-4cea-9c66-5ded3e31a721) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.573236178Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.573254224Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] e5daa207-6492-4466-92d0-cc614d6c56ce (8a6b2194-d5ad-4cea-9c66-5ded3e31a721) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.573274445Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.573292436Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] e5daa207-6492-4466-92d0-cc614d6c56ce (8a6b2194-d5ad-4cea-9c66-5ded3e31a721) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,{"time":"2023-09-22T23:08:11.573318003Z"","msg"hostname:"":"ip-10-150-1-74.us-west-2.compute.internal","pidCrucible stats registered with UUID: 3bb5f814-a110-4a59-9e32-8b8ba8617c89"":4291,"}v"
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.573393707Z","hostname":":ip-10-150-1-74.us-west-2.compute.internal","0pid":4291,"name}":
"crucible","level{":30"msg":"[2] e5daa207-6492-4466-92d0-cc614d6c56ce (8a6b2194-d5ad-4cea-9c66-5ded3e31a721) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":","2023-09-22T23:08:11.573422322Ztime"":","hostname"2023-09-22T23:08:11.573433033Z:"","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":ip-10-150-1-74.us-west-2.compute.internal4291","pid":}4291
}
{{"msg":""msg":"[2] Transition from New to WaitActive","v":0,"name":"Crucible 3bb5f814-a110-4a59-9e32-8b8ba8617c89 has session id: 1d016687-67b4-4780-a901-1e206fe29c5acrucible"",,""vlevel""::030,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.573466598Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] e5daa207-6492-4466-92d0-cc614d6c56ce (8a6b2194-d5ad-4cea-9c66-5ded3e31a721) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.573469942Z","hostname":",ip-10-150-1-74.us-west-2.compute.internal""time,"":"pid":42912023-09-22T23:08:11.573523504Z"}
,"hostname":"{ip-10-150-1-74.us-west-2.compute.internal","pid":"4291msg":"}
{"[0] 3bb5f814-a110-4a59-9e32-8b8ba8617c89 (f4d73180-9a3c-4adf-9c0e-cd91d883ab93) New New New ds_transition to WaitActivemsg"":","v":0,"[2] Transition from WaitActive to WaitQuorum"name":,""v"crucible:"0,,""levelname""::"30crucible","level":30,","time"time:"":"2023-09-22T23:08:11.57355599Z"2023-09-22T23:08:11.573557822Z",","hostname"hostname:"":"ip-10-150-1-74.us-west-2.compute.internal"ip-10-150-1-74.us-west-2.compute.internal,"","pid"pid:":42914291}}

{{"msg"":"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","[2] e5daa207-6492-4466-92d0-cc614d6c56ce (8a6b2194-d5ad-4cea-9c66-5ded3e31a721) Active Active WaitQuorum ds_transition to Active"level":,"30v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.573589434Z",","hostname"time:"":"2023-09-22T23:08:11.573593536Z"ip-10-150-1-74.us-west-2.compute.internal",,""pidhostname""::"4291}
ip-10-150-1-74.us-west-2.compute.internal","pid":{4291}
"msg":"{"msg":"[2] Transition from WaitQuorum to Active","v"[0] 3bb5f814-a110-4a59-9e32-8b8ba8617c89 (f4d73180-9a3c-4adf-9c0e-cd91d883ab93) WaitActive New New ds_transition to WaitQuorum:"0,,""vname""::0","crucible"name",:""level"crucible:"30,"level":30,"time,"":"time":"2023-09-22T23:08:11.573625676Z"2023-09-22T23:08:11.573626954Z,"","hostname":hostname"":"ip-10-150-1-74.us-west-2.compute.internal",ip-10-150-1-74.us-west-2.compute.internal""pid,"":pid4291":4291}
}
{"{msg":""msg":"[0] Transition from WaitActive to WaitQuorum","ve5daa207-6492-4466-92d0-cc614d6c56ce is now active with session: 8a6b2194-d5ad-4cea-9c66-5ded3e31a721"":0,","v":name"0:","cruciblename"":,""cruciblelevel"":,"30level":30,,""timetime""::""2023-09-22T23:08:11.573660552Z"2023-09-22T23:08:11.573659644Z",","hostname"hostname:"":"ip-10-150-1-74.us-west-2.compute.internal"ip-10-150-1-74.us-west-2.compute.internal",,""pidpid""::42914291}}

{"{msg":""msg":"[0] 3bb5f814-a110-4a59-9e32-8b8ba8617c89 (f4d73180-9a3c-4adf-9c0e-cd91d883ab93) WaitQuorum New New ds_transition to Active","v"[1] e5daa207-6492-4466-92d0-cc614d6c56ce (8a6b2194-d5ad-4cea-9c66-5ded3e31a721) Active Active Active ds_transition to Faulted:"0,,""v"name:"0:","cruciblename"":,""cruciblelevel"":,"30level":30,,""timetime""::""2023-09-22T23:08:11.573697916Z2023-09-22T23:08:11.573697048Z"",,""hostnamehostname""::""ip-10-150-1-74.us-west-2.compute.internalip-10-150-1-74.us-west-2.compute.internal"",,""pidpid""::42914291}}

{"{msg":""msg":"[1] Transition from Active to Faulted","v[0] Transition from WaitQuorum to Active"":0,","v":name0":,""namecrucible"":","cruciblelevel"":,"30level":30,,""timetime""::""2023-09-22T23:08:11.573731115Z2023-09-22T23:08:11.573730444Z"",,""hostnamehostname""::""ip-10-150-1-74.us-west-2.compute.internalip-10-150-1-74.us-west-2.compute.internal"",,""pidpid""::42914291}}

{{"msg"":"msg":"[1] 3bb5f814-a110-4a59-9e32-8b8ba8617c89 (f4d73180-9a3c-4adf-9c0e-cd91d883ab93) Active New New ds_transition to WaitActive","[1] e5daa207-6492-4466-92d0-cc614d6c56ce (8a6b2194-d5ad-4cea-9c66-5ded3e31a721) Active Faulted Active ds_transition to LiveRepairReadyv"":,"0v",:"0name,"":"name":crucible"",crucible"",level"":level"30:30,,""timetime""::""2023-09-22T23:08:11.573775536Z2023-09-22T23:08:11.573775326Z"",,""hostnamehostname""::""ip-10-150-1-74.us-west-2.compute.internalip-10-150-1-74.us-west-2.compute.internal"",,""pidpid""::42914291}}

{"{msg":""msg":"[1] Transition from Faulted to LiveRepairReady",[1] Transition from New to WaitActive""v",:"0v",:"0,name"":name"":"crucible"crucible,"","level":level"30:30,,""timetime""::""2023-09-22T23:08:11.573810728Z2023-09-22T23:08:11.573810405Z"",,""hostnamehostname""::""ip-10-150-1-74.us-west-2.compute.internalip-10-150-1-74.us-west-2.compute.internal"",,""pidpid""::42914291}}

{"{msg":""msg":"[1] 3bb5f814-a110-4a59-9e32-8b8ba8617c89 (f4d73180-9a3c-4adf-9c0e-cd91d883ab93) Active WaitActive New ds_transition to WaitQuorum","v":[1] e5daa207-6492-4466-92d0-cc614d6c56ce (8a6b2194-d5ad-4cea-9c66-5ded3e31a721) Active LiveRepairReady Active ds_transition to LiveRepair0",","namev""::"0,crucible""name,"":"level":crucible"30,"level":30,",time"":time"":"2023-09-22T23:08:11.573847805Z"2023-09-22T23:08:11.573849218Z",","hostname":hostname"":"ip-10-150-1-74.us-west-2.compute.internal"ip-10-150-1-74.us-west-2.compute.internal,"","pid"pid:":42914291}}

{"{msg":""msg":"[1] Transition from WaitActive to WaitQuorum","v":[1] Transition from LiveRepairReady to LiveRepair0",","namev""::"0,"crucible"name",:""level"crucible:"30,"level":30,"time,"":"time":"2023-09-22T23:08:11.573880365Z"2023-09-22T23:08:11.573881743Z,"","hostname":"hostname":"ip-10-150-1-74.us-west-2.compute.internal","ip-10-150-1-74.us-west-2.compute.internal"pid",:"4291pid":}4291
}
{"{msg":""msg":"Waiting for Close + ReOpen jobs","v":0,"name"[1] 3bb5f814-a110-4a59-9e32-8b8ba8617c89 (f4d73180-9a3c-4adf-9c0e-cd91d883ab93) Active WaitQuorum New ds_transition to Active:"","crucible"v",:"0level,"":30name":"crucible","level":30,"time":","time"2023-09-22T23:08:11.573913Z:"","hostname":2023-09-22T23:08:11.573915629Z"","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291ip-10-150-1-74.us-west-2.compute.internal",}"
pid":4291}{
"msg":"{"msg":"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]","v":0[1] Transition from WaitQuorum to Active,"",name"":v"":0crucible,"","name":level"":30crucible","level":30,"time":,""time":"2023-09-22T23:08:11.573946115Z","2023-09-22T23:08:11.5739478Z"hostname,"":"hostname":"ip-10-150-1-74.us-west-2.compute.internal","ip-10-150-1-74.us-west-2.compute.internal"pid,"":pid4291":4291}
}
{"{msg":""msg":"RE:0 close id:1000 queued, notify DS","v":0,"name":"crucible","[2] 3bb5f814-a110-4a59-9e32-8b8ba8617c89 (f4d73180-9a3c-4adf-9c0e-cd91d883ab93) Active Active New ds_transition to WaitActivelevel"":,30"v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.573977993Z",","hostname"time:"":"2023-09-22T23:08:11.573982233Z"ip-10-150-1-74.us-west-2.compute.internal",,""pidhostname""::"4291}
ip-10-150-1-74.us-west-2.compute.internal","pid":{4291}"
msg":"{"RE:0 Wait for result from close command 1000:1msg"":","v":0,"[2] Transition from New to WaitActive"name":,""v"crucible:"0,,""levelname""::30"crucible","level":30,"time":","time2023-09-22T23:08:11.574011315Z"":","hostname":"2023-09-22T23:08:11.574013602Z","hostname":ip-10-150-1-74.us-west-2.compute.internal"","pid":4291ip-10-150-1-74.us-west-2.compute.internal"},
"pid":4291}
{"msg":"[2] 3bb5f814-a110-4a59-9e32-8b8ba8617c89 (f4d73180-9a3c-4adf-9c0e-cd91d883ab93) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.57404831Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.574086643Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 3bb5f814-a110-4a59-9e32-8b8ba8617c89 (f4d73180-9a3c-4adf-9c0e-cd91d883ab93) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.574111046Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.574129117Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"3bb5f814-a110-4a59-9e32-8b8ba8617c89 is now active with session: f4d73180-9a3c-4adf-9c0e-cd91d883ab93","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.574148673Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 3bb5f814-a110-4a59-9e32-8b8ba8617c89 (f4d73180-9a3c-4adf-9c0e-cd91d883ab93) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.574168842Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.57418665Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 3bb5f814-a110-4a59-9e32-8b8ba8617c89 (f4d73180-9a3c-4adf-9c0e-cd91d883ab93) Active Faulted Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.574207099Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.574225218Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 3bb5f814-a110-4a59-9e32-8b8ba8617c89 (f4d73180-9a3c-4adf-9c0e-cd91d883ab93) Active LiveRepairReady Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.574245364Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.574263658Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Write to Extent 0:0:9 under repair","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:11.574281515Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Write to Extent 0:0:9 under repair","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:11.574299124Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test live_repair::repair_test::test_repair_io_at_el_sent ... ok
{"msg":"Crucible stats registered with UUID: 88bef107-1f4b-44d9-ac8b-a7773f1a6565","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.575029113Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 88bef107-1f4b-44d9-ac8b-a7773f1a6565 has session id: 3de173a3-0b31-42b5-b10d-3ba18bf4e7c1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.575068714Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 88bef107-1f4b-44d9-ac8b-a7773f1a6565 (9756f0fa-d081-4984-bbe5-479ed9dc2abd) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.575091971Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.575110731Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 88bef107-1f4b-44d9-ac8b-a7773f1a6565 (9756f0fa-d081-4984-bbe5-479ed9dc2abd) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.57513127Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.575149672Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 88bef107-1f4b-44d9-ac8b-a7773f1a6565 (9756f0fa-d081-4984-bbe5-479ed9dc2abd) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.575177557Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.575197761Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 88bef107-1f4b-44d9-ac8b-a7773f1a6565 (9756f0fa-d081-4984-bbe5-479ed9dc2abd) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.575218107Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.575235915Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 88bef107-1f4b-44d9-ac8b-a7773f1a6565 (9756f0fa-d081-4984-bbe5-479ed9dc2abd) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.575256193Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.575274229Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 88bef107-1f4b-44d9-ac8b-a7773f1a6565 (9756f0fa-d081-4984-bbe5-479ed9dc2abd) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.575294004Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.575311684Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 88bef107-1f4b-44d9-ac8b-a7773f1a6565 (9756f0fa-d081-4984-bbe5-479ed9dc2abd) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.575331624Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.575349368Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 88bef107-1f4b-44d9-ac8b-a7773f1a6565 (9756f0fa-d081-4984-bbe5-479ed9dc2abd) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.57536943Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.575387407Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 88bef107-1f4b-44d9-ac8b-a7773f1a6565 (9756f0fa-d081-4984-bbe5-479ed9dc2abd) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.57540755Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.575425346Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"88bef107-1f4b-44d9-ac8b-a7773f1a6565 is now active with session: 9756f0fa-d081-4984-bbe5-479ed9dc2abd","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.575444625Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 88bef107-1f4b-44d9-ac8b-a7773f1a6565 (9756f0fa-d081-4984-bbe5-479ed9dc2abd) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.575464471Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.575493277Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 88bef107-1f4b-44d9-ac8b-a7773f1a6565 (9756f0fa-d081-4984-bbe5-479ed9dc2abd) Active Faulted Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.575531172Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.57555209Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 88bef107-1f4b-44d9-ac8b-a7773f1a6565 (9756f0fa-d081-4984-bbe5-479ed9dc2abd) Active LiveRepairReady Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.575572713Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.575590962Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test live_repair::repair_test::test_repair_io_below_el_sent ... ok
{"msg":"Crucible stats registered with UUID: 40c33510-4dde-4fb8-bcf3-28cc28cde07e","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.576342584Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 40c33510-4dde-4fb8-bcf3-28cc28cde07e has session id: c5fd77f9-e431-4650-b803-1cbe789e5ab9","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.576376728Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 40c33510-4dde-4fb8-bcf3-28cc28cde07e (bae9366e-30e1-42e2-8fd8-3f0c49bf0433) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.576398237Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.576416888Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 40c33510-4dde-4fb8-bcf3-28cc28cde07e (bae9366e-30e1-42e2-8fd8-3f0c49bf0433) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.576437203Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.576455555Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 40c33510-4dde-4fb8-bcf3-28cc28cde07e (bae9366e-30e1-42e2-8fd8-3f0c49bf0433) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.576475764Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.576494399Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 40c33510-4dde-4fb8-bcf3-28cc28cde07e (bae9366e-30e1-42e2-8fd8-3f0c49bf0433) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.576514555Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.576532597Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 40c33510-4dde-4fb8-bcf3-28cc28cde07e (bae9366e-30e1-42e2-8fd8-3f0c49bf0433) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.576552866Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.576570946Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 40c33510-4dde-4fb8-bcf3-28cc28cde07e (bae9366e-30e1-42e2-8fd8-3f0c49bf0433) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.576591028Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.576615198Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 40c33510-4dde-4fb8-bcf3-28cc28cde07e (bae9366e-30e1-42e2-8fd8-3f0c49bf0433) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.576636995Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.576654936Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 40c33510-4dde-4fb8-bcf3-28cc28cde07e (bae9366e-30e1-42e2-8fd8-3f0c49bf0433) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.576675175Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.576693104Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 40c33510-4dde-4fb8-bcf3-28cc28cde07e (bae9366e-30e1-42e2-8fd8-3f0c49bf0433) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.576713093Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.576730985Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"40c33510-4dde-4fb8-bcf3-28cc28cde07e is now active with session: bae9366e-30e1-42e2-8fd8-3f0c49bf0433","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.576750303Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 40c33510-4dde-4fb8-bcf3-28cc28cde07e (bae9366e-30e1-42e2-8fd8-3f0c49bf0433) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.576770297Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.576788049Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 40c33510-4dde-4fb8-bcf3-28cc28cde07e (bae9366e-30e1-42e2-8fd8-3f0c49bf0433) Active Faulted Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.576808183Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.576826157Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 40c33510-4dde-4fb8-bcf3-28cc28cde07e (bae9366e-30e1-42e2-8fd8-3f0c49bf0433) Active LiveRepairReady Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.576846284Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.576864583Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test live_repair::repair_test::test_repair_io_no_el_skipped ... ok
{"msg":"Crucible stats registered with UUID: b7d2533c-27bb-4b70-ba98-81ede3647353","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.577509918Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible b7d2533c-27bb-4b70-ba98-81ede3647353 has session id: 2a94e344-a7a3-4742-bc06-4a867d1b3f91","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.577543893Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] b7d2533c-27bb-4b70-ba98-81ede3647353 (f97dd42b-6e52-41df-971f-7ff9a9ed49fa) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.577565667Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.577590759Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] b7d2533c-27bb-4b70-ba98-81ede3647353 (f97dd42b-6e52-41df-971f-7ff9a9ed49fa) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.577612499Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.577630651Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] b7d2533c-27bb-4b70-ba98-81ede3647353 (f97dd42b-6e52-41df-971f-7ff9a9ed49fa) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.577650717Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.577668658Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] b7d2533c-27bb-4b70-ba98-81ede3647353 (f97dd42b-6e52-41df-971f-7ff9a9ed49fa) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.577688643Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.577706483Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] b7d2533c-27bb-4b70-ba98-81ede3647353 (f97dd42b-6e52-41df-971f-7ff9a9ed49fa) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.577726512Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.577744642Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] b7d2533c-27bb-4b70-ba98-81ede3647353 (f97dd42b-6e52-41df-971f-7ff9a9ed49fa) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.577764749Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.57778273Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] b7d2533c-27bb-4b70-ba98-81ede3647353 (f97dd42b-6e52-41df-971f-7ff9a9ed49fa) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.577803164Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.577821117Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] b7d2533c-27bb-4b70-ba98-81ede3647353 (f97dd42b-6e52-41df-971f-7ff9a9ed49fa) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.57784137Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.577859403Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] b7d2533c-27bb-4b70-ba98-81ede3647353 (f97dd42b-6e52-41df-971f-7ff9a9ed49fa) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.577879481Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.577897441Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"b7d2533c-27bb-4b70-ba98-81ede3647353 is now active with session: f97dd42b-6e52-41df-971f-7ff9a9ed49fa","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.577916888Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] b7d2533c-27bb-4b70-ba98-81ede3647353 (f97dd42b-6e52-41df-971f-7ff9a9ed49fa) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.577941685Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.577960347Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] b7d2533c-27bb-4b70-ba98-81ede3647353 (f97dd42b-6e52-41df-971f-7ff9a9ed49fa) Active Faulted Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.577980595Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.577998604Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] b7d2533c-27bb-4b70-ba98-81ede3647353 (f97dd42b-6e52-41df-971f-7ff9a9ed49fa) Active LiveRepairReady Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.57801887Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.57803715Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Write to Extent 1:0:9 under repair","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:11.578054827Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Write to Extent 1:1:9 under repair","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:11.578072431Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Write to Extent 1:2:9 under repair","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:11.57809Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Write 2:0:9 past extent under repair 1","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:11.578108019Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Write 2:1:9 past extent under repair 1","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:11.578125815Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Write 2:2:9 past extent under repair 1","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:11.578143708Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"IO Write 1004 on eur 1 Added deps 2","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:11.578161495Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Create read repair deps for extent 2","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:11.578179251Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"IO Read  1005 extent 1 added deps 2","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:11.578196989Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Write to Extent 1:0:9 under repair","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:11.578214796Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Write to Extent 1:1:9 under repair","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:11.57823242Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Write to Extent 1:2:9 under repair","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:11.578250123Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Write 2:0:9 past extent under repair 1","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:11.578268082Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Write 2:1:9 past extent under repair 1","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:11.578285794Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Write 2:2:9 past extent under repair 1","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:11.578307991Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"IO Write 1006 on eur 1 Added deps 2","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:11.578326598Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test live_repair::repair_test::test_repair_io_span_el_sent ... ok
{"msg":"Crucible stats registered with UUID: ddbb46f6-ed67-479c-8e81-15b7886450cd","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.579026205Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible ddbb46f6-ed67-479c-8e81-15b7886450cd has session id: f8cd1053-bdd4-4f75-a78e-4f93dda8f9c3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.579064199Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] ddbb46f6-ed67-479c-8e81-15b7886450cd (1025ea98-7a45-49c5-9cc6-a08e2c6de343) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.579086838Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.579105511Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] ddbb46f6-ed67-479c-8e81-15b7886450cd (1025ea98-7a45-49c5-9cc6-a08e2c6de343) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.579126004Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.579144197Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] ddbb46f6-ed67-479c-8e81-15b7886450cd (1025ea98-7a45-49c5-9cc6-a08e2c6de343) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.57916449Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.579182763Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] ddbb46f6-ed67-479c-8e81-15b7886450cd (1025ea98-7a45-49c5-9cc6-a08e2c6de343) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.579202565Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.579220423Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] ddbb46f6-ed67-479c-8e81-15b7886450cd (1025ea98-7a45-49c5-9cc6-a08e2c6de343) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.579240805Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.57925866Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] ddbb46f6-ed67-479c-8e81-15b7886450cd (1025ea98-7a45-49c5-9cc6-a08e2c6de343) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.579278753Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.579296646Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] ddbb46f6-ed67-479c-8e81-15b7886450cd (1025ea98-7a45-49c5-9cc6-a08e2c6de343) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.579316511Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.579340991Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] ddbb46f6-ed67-479c-8e81-15b7886450cd (1025ea98-7a45-49c5-9cc6-a08e2c6de343) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.579362888Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.579380941Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] ddbb46f6-ed67-479c-8e81-15b7886450cd (1025ea98-7a45-49c5-9cc6-a08e2c6de343) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.57940089Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.579418769Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"ddbb46f6-ed67-479c-8e81-15b7886450cd is now active with session: 1025ea98-7a45-49c5-9cc6-a08e2c6de343","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.579438167Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] ddbb46f6-ed67-479c-8e81-15b7886450cd (1025ea98-7a45-49c5-9cc6-a08e2c6de343) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.579458194Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.579475891Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] ddbb46f6-ed67-479c-8e81-15b7886450cd (1025ea98-7a45-49c5-9cc6-a08e2c6de343) Active Faulted Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.579496084Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.579514154Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] ddbb46f6-ed67-479c-8e81-15b7886450cd (1025ea98-7a45-49c5-9cc6-a08e2c6de343) Active LiveRepairReady Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.57953436Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.579552364Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Create read repair deps for extent 2","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:11.579570064Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"IO Read  1004 extent 1 added deps 2","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:11.57958773Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test live_repair::repair_test::test_repair_read_span_el_sent ... ok
{"msg":"Crucible stats registered with UUID: ddbe0475-cb31-4366-8d95-8ff6d6cadb7d","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.580314994Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible ddbe0475-cb31-4366-8d95-8ff6d6cadb7d has session id: 51ea6dad-bef7-469c-9bae-157ef82d84ae","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.58034881Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] ddbe0475-cb31-4366-8d95-8ff6d6cadb7d (e680a265-0383-4b2d-b5c1-7fda88a3d8b1) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.580370605Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.580394756Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] ddbe0475-cb31-4366-8d95-8ff6d6cadb7d (e680a265-0383-4b2d-b5c1-7fda88a3d8b1) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.580416465Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.580434665Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] ddbe0475-cb31-4366-8d95-8ff6d6cadb7d (e680a265-0383-4b2d-b5c1-7fda88a3d8b1) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.580454796Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.580472755Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] ddbe0475-cb31-4366-8d95-8ff6d6cadb7d (e680a265-0383-4b2d-b5c1-7fda88a3d8b1) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.580492154Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.580510079Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] ddbe0475-cb31-4366-8d95-8ff6d6cadb7d (e680a265-0383-4b2d-b5c1-7fda88a3d8b1) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.580530561Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.580548528Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] ddbe0475-cb31-4366-8d95-8ff6d6cadb7d (e680a265-0383-4b2d-b5c1-7fda88a3d8b1) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.58056858Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.580586536Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] ddbe0475-cb31-4366-8d95-8ff6d6cadb7d (e680a265-0383-4b2d-b5c1-7fda88a3d8b1) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.580606424Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.580624213Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] ddbe0475-cb31-4366-8d95-8ff6d6cadb7d (e680a265-0383-4b2d-b5c1-7fda88a3d8b1) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.580644407Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.580662375Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] ddbe0475-cb31-4366-8d95-8ff6d6cadb7d (e680a265-0383-4b2d-b5c1-7fda88a3d8b1) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.580682363Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.580700159Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"ddbe0475-cb31-4366-8d95-8ff6d6cadb7d is now active with session: e680a265-0383-4b2d-b5c1-7fda88a3d8b1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.580719806Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] ddbe0475-cb31-4366-8d95-8ff6d6cadb7d (e680a265-0383-4b2d-b5c1-7fda88a3d8b1) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.580744366Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.580763024Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] ddbe0475-cb31-4366-8d95-8ff6d6cadb7d (e680a265-0383-4b2d-b5c1-7fda88a3d8b1) Active Faulted Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.580783498Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.580801687Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] ddbe0475-cb31-4366-8d95-8ff6d6cadb7d (e680a265-0383-4b2d-b5c1-7fda88a3d8b1) Active LiveRepairReady Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.580821938Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.580839977Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Write to Extent 1:0:9 under repair","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:11.580857651Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Write to Extent 1:1:9 under repair","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:11.580875488Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Write to Extent 1:2:9 under repair","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:11.580893118Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Write 2:0:9 past extent under repair 1","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:11.58091097Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Write 2:1:9 past extent under repair 1","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:11.580928962Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Write 2:2:9 past extent under repair 1","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:11.580946903Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"IO Write 1004 on eur 1 Added deps 2","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:11.580964846Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test live_repair::repair_test::test_repair_write_span_el_sent ... ok
{"msg":"Crucible stats registered with UUID: 4a1430a7-119a-493e-9505-fe3d80a07fe9","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.581591705Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 4a1430a7-119a-493e-9505-fe3d80a07fe9 has session id: 1df896f2-b218-460b-9ff9-6c436b028406","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.581625522Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 4a1430a7-119a-493e-9505-fe3d80a07fe9 (f37022e1-0807-4535-91d2-ef91755ded99) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.581647322Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.581665656Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 4a1430a7-119a-493e-9505-fe3d80a07fe9 (f37022e1-0807-4535-91d2-ef91755ded99) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.581685934Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.581710199Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 4a1430a7-119a-493e-9505-fe3d80a07fe9 (f37022e1-0807-4535-91d2-ef91755ded99) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.581731584Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.581749645Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 4a1430a7-119a-493e-9505-fe3d80a07fe9 (f37022e1-0807-4535-91d2-ef91755ded99) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.581769811Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.581787451Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 4a1430a7-119a-493e-9505-fe3d80a07fe9 (f37022e1-0807-4535-91d2-ef91755ded99) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.58180761Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.581825706Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 4a1430a7-119a-493e-9505-fe3d80a07fe9 (f37022e1-0807-4535-91d2-ef91755ded99) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.581845691Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.58186373Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 4a1430a7-119a-493e-9505-fe3d80a07fe9 (f37022e1-0807-4535-91d2-ef91755ded99) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.581883661Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.581901442Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 4a1430a7-119a-493e-9505-fe3d80a07fe9 (f37022e1-0807-4535-91d2-ef91755ded99) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.581921545Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.581939481Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 4a1430a7-119a-493e-9505-fe3d80a07fe9 (f37022e1-0807-4535-91d2-ef91755ded99) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.581959413Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.581977178Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"4a1430a7-119a-493e-9505-fe3d80a07fe9 is now active with session: f37022e1-0807-4535-91d2-ef91755ded99","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.581996853Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 4a1430a7-119a-493e-9505-fe3d80a07fe9 (f37022e1-0807-4535-91d2-ef91755ded99) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.582016963Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.582034742Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 4a1430a7-119a-493e-9505-fe3d80a07fe9 (f37022e1-0807-4535-91d2-ef91755ded99) Active Faulted Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.582059944Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.5820788Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 4a1430a7-119a-493e-9505-fe3d80a07fe9 (f37022e1-0807-4535-91d2-ef91755ded99) Active LiveRepairReady Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.582099186Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.582117379Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Write to Extent 0:0:9 under repair","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:11.582135093Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Write to Extent 0:1:9 under repair","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:11.582152776Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Write to Extent 0:2:9 under repair","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:11.58217052Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Write 1:0:9 past extent under repair 0","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:11.582188307Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Write 1:1:9 past extent under repair 0","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:11.582206039Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Write 1:2:9 past extent under repair 0","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:11.582223839Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Write 2:0:9 past extent under repair 0","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:11.582241584Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Write 2:1:9 past extent under repair 0","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:11.58225933Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Write 2:2:9 past extent under repair 0","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:11.582277103Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"IO Write 1008 on eur 0 Added deps 1","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:11.582294666Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"IO Write 1008 on eur 0 Added deps 2","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:11.582312369Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test live_repair::repair_test::test_repair_write_span_two_el_sent ... ok
{"msg":"Crucible stats registered with UUID: f977a52e-aaca-4b8d-a154-33554e95a9ae","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.583057761Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible f977a52e-aaca-4b8d-a154-33554e95a9ae has session id: 8e529dfd-872a-4973-a616-82c0bf4954f7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.583095484Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] f977a52e-aaca-4b8d-a154-33554e95a9ae (f4abcc1d-5f2e-4c67-ac26-73a45f2966c8) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.58311806Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.583136985Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] f977a52e-aaca-4b8d-a154-33554e95a9ae (f4abcc1d-5f2e-4c67-ac26-73a45f2966c8) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.583164122Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.583184181Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] f977a52e-aaca-4b8d-a154-33554e95a9ae (f4abcc1d-5f2e-4c67-ac26-73a45f2966c8) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.583204442Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.583222465Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] f977a52e-aaca-4b8d-a154-33554e95a9ae (f4abcc1d-5f2e-4c67-ac26-73a45f2966c8) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.583242815Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.583260896Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] f977a52e-aaca-4b8d-a154-33554e95a9ae (f4abcc1d-5f2e-4c67-ac26-73a45f2966c8) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.583281268Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.583299291Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] f977a52e-aaca-4b8d-a154-33554e95a9ae (f4abcc1d-5f2e-4c67-ac26-73a45f2966c8) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.583319215Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.583337111Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] f977a52e-aaca-4b8d-a154-33554e95a9ae (f4abcc1d-5f2e-4c67-ac26-73a45f2966c8) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.58335702Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.583374851Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] f977a52e-aaca-4b8d-a154-33554e95a9ae (f4abcc1d-5f2e-4c67-ac26-73a45f2966c8) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.583395004Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.583412834Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] f977a52e-aaca-4b8d-a154-33554e95a9ae (f4abcc1d-5f2e-4c67-ac26-73a45f2966c8) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.583432855Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.583450855Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"f977a52e-aaca-4b8d-a154-33554e95a9ae is now active with session: f4abcc1d-5f2e-4c67-ac26-73a45f2966c8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.583470303Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] f977a52e-aaca-4b8d-a154-33554e95a9ae (f4abcc1d-5f2e-4c67-ac26-73a45f2966c8) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.583494813Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.583513822Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] f977a52e-aaca-4b8d-a154-33554e95a9ae (f4abcc1d-5f2e-4c67-ac26-73a45f2966c8) Active Faulted Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.583534201Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.583552287Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] f977a52e-aaca-4b8d-a154-33554e95a9ae (f4abcc1d-5f2e-4c67-ac26-73a45f2966c8) Active LiveRepairReady Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.583572765Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.583590964Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test live_repair::repair_test::test_reserve_extent_repair_ids ... ok
Testing repair with s:0 r:[ClientId(1)]
Sep 22 23:08:11.584 DEBG Get repair info for 0 source, : downstairs
Sep 22 23:08:11.584 DEBG Get repair info for 1 bad, : downstairs
Sep 22 23:08:11.584 INFO Repair for extent 0 s:0 d:[ClientId(1)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(0), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(1)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
Testing repair with s:0 r:[ClientId(2)]
Sep 22 23:08:11.584 DEBG Get repair info for 0 source, : downstairs
Sep 22 23:08:11.584 DEBG Get repair info for 2 bad, : downstairs
Sep 22 23:08:11.584 INFO Repair for extent 0 s:0 d:[ClientId(2)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(0), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(2)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
Testing repair with s:1 r:[ClientId(0)]
Sep 22 23:08:11.584 DEBG Get repair info for 1 source, : downstairs
Sep 22 23:08:11.584 DEBG Get repair info for 0 bad, : downstairs
Sep 22 23:08:11.584 INFO Repair for extent 0 s:1 d:[ClientId(0)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(1), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(0)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
Testing repair with s:1 r:[ClientId(2)]
Sep 22 23:08:11.584 DEBG Get repair info for 1 source, : downstairs
Sep 22 23:08:11.584 DEBG Get repair info for 2 bad, : downstairs
Sep 22 23:08:11.584 INFO Repair for extent 0 s:1 d:[ClientId(2)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(1), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(2)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
Testing repair with s:2 r:[ClientId(0)]
Sep 22 23:08:11.584 DEBG Get repair info for 2 source, : downstairs
Sep 22 23:08:11.584 DEBG Get repair info for 0 bad, : downstairs
Sep 22 23:08:11.584 INFO Repair for extent 0 s:2 d:[ClientId(0)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(2), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(0)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
Testing repair with s:2 r:[ClientId(1)]
Sep 22 23:08:11.584 DEBG Get repair info for 2 source, : downstairs
Sep 22 23:08:11.584 DEBG Get repair info for 1 bad, : downstairs
Sep 22 23:08:11.584 INFO Repair for extent 0 s:2 d:[ClientId(1)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(2), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(1)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
test live_repair::repair_test::test_solver_dirty_needs_repair_one ... ok
Testing repair with s:0 r:[ClientId(1), ClientId(2)]
Sep 22 23:08:11.585 DEBG Get repair info for 0 source, : downstairs
Sep 22 23:08:11.585 DEBG Get repair info for 1 bad, : downstairs
Sep 22 23:08:11.585 DEBG Get repair info for 2 bad, : downstairs
Sep 22 23:08:11.585 INFO Repair for extent 0 s:0 d:[ClientId(1), ClientId(2)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(0), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
Testing repair with s:1 r:[ClientId(0), ClientId(2)]
Sep 22 23:08:11.585 DEBG Get repair info for 1 source, : downstairs
Sep 22 23:08:11.585 DEBG Get repair info for 0 bad, : downstairs
Sep 22 23:08:11.585 DEBG Get repair info for 2 bad, : downstairs
Sep 22 23:08:11.585 INFO Repair for extent 0 s:1 d:[ClientId(0), ClientId(2)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(1), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(0), ClientId(2)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
Testing repair with s:2 r:[ClientId(0), ClientId(1)]
Sep 22 23:08:11.585 DEBG Get repair info for 2 source, : downstairs
Sep 22 23:08:11.585 DEBG Get repair info for 0 bad, : downstairs
Sep 22 23:08:11.585 DEBG Get repair info for 1 bad, : downstairs
Sep 22 23:08:11.585 INFO Repair for extent 0 s:2 d:[ClientId(0), ClientId(1)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(2), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(0), ClientId(1)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
test live_repair::repair_test::test_solver_dirty_needs_repair_two ... ok
Testing repair with s:0 r:[ClientId(1)]
Sep 22 23:08:11.585 DEBG Get repair info for 0 source, : downstairs
Sep 22 23:08:11.585 DEBG Get repair info for 1 bad, : downstairs
Sep 22 23:08:11.585 INFO Repair for extent 0 s:0 d:[ClientId(1)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(0), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(1)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
Testing repair with s:0 r:[ClientId(2)]
Sep 22 23:08:11.585 DEBG Get repair info for 0 source, : downstairs
Sep 22 23:08:11.585 DEBG Get repair info for 2 bad, : downstairs
Sep 22 23:08:11.585 INFO Repair for extent 0 s:0 d:[ClientId(2)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(0), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(2)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
Testing repair with s:1 r:[ClientId(0)]
Sep 22 23:08:11.585 DEBG Get repair info for 1 source, : downstairs
Sep 22 23:08:11.585 DEBG Get repair info for 0 bad, : downstairs
Sep 22 23:08:11.585 INFO Repair for extent 0 s:1 d:[ClientId(0)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(1), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(0)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
Testing repair with s:1 r:[ClientId(2)]
Sep 22 23:08:11.585 DEBG Get repair info for 1 source, : downstairs
Sep 22 23:08:11.585 DEBG Get repair info for 2 bad, : downstairs
Sep 22 23:08:11.585 INFO Repair for extent 0 s:1 d:[ClientId(2)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(1), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(2)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
Testing repair with s:2 r:[ClientId(0)]
Sep 22 23:08:11.585 DEBG Get repair info for 2 source, : downstairs
Sep 22 23:08:11.585 DEBG Get repair info for 0 bad, : downstairs
Sep 22 23:08:11.585 INFO Repair for extent 0 s:2 d:[ClientId(0)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(2), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(0)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
Testing repair with s:2 r:[ClientId(1)]
Sep 22 23:08:11.585 DEBG Get repair info for 2 source, : downstairs
Sep 22 23:08:11.585 DEBG Get repair info for 1 bad, : downstairs
Sep 22 23:08:11.585 INFO Repair for extent 0 s:2 d:[ClientId(1)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(2), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(1)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
test live_repair::repair_test::test_solver_flush_higher_needs_repair_one ... ok
Testing repair with s:0 r:[ClientId(1), ClientId(2)]
Sep 22 23:08:11.586 DEBG Get repair info for 0 source, : downstairs
Sep 22 23:08:11.586 DEBG Get repair info for 1 bad, : downstairs
Sep 22 23:08:11.586 DEBG Get repair info for 2 bad, : downstairs
Sep 22 23:08:11.586 INFO Repair for extent 0 s:0 d:[ClientId(1), ClientId(2)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(0), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
Testing repair with s:1 r:[ClientId(0), ClientId(2)]
Sep 22 23:08:11.586 DEBG Get repair info for 1 source, : downstairs
Sep 22 23:08:11.586 DEBG Get repair info for 0 bad, : downstairs
Sep 22 23:08:11.586 DEBG Get repair info for 2 bad, : downstairs
Sep 22 23:08:11.586 INFO Repair for extent 0 s:1 d:[ClientId(0), ClientId(2)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(1), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(0), ClientId(2)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
Testing repair with s:2 r:[ClientId(0), ClientId(1)]
Sep 22 23:08:11.586 DEBG Get repair info for 2 source, : downstairs
Sep 22 23:08:11.586 DEBG Get repair info for 0 bad, : downstairs
Sep 22 23:08:11.586 DEBG Get repair info for 1 bad, : downstairs
Sep 22 23:08:11.586 INFO Repair for extent 0 s:2 d:[ClientId(0), ClientId(1)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(2), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(0), ClientId(1)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
test live_repair::repair_test::test_solver_flush_higher_needs_repair_two ... ok
Testing repair with s:0 r:[ClientId(1)]
Sep 22 23:08:11.587 DEBG Get repair info for 0 source, : downstairs
Sep 22 23:08:11.587 DEBG Get repair info for 1 bad, : downstairs
Sep 22 23:08:11.587 INFO Repair for extent 0 s:0 d:[ClientId(1)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(0), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(1)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
Testing repair with s:0 r:[ClientId(2)]
Sep 22 23:08:11.587 DEBG Get repair info for 0 source, : downstairs
Sep 22 23:08:11.587 DEBG Get repair info for 2 bad, : downstairs
Sep 22 23:08:11.587 INFO Repair for extent 0 s:0 d:[ClientId(2)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(0), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(2)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
Testing repair with s:1 r:[ClientId(0)]
Sep 22 23:08:11.587 DEBG Get repair info for 1 source, : downstairs
Sep 22 23:08:11.587 DEBG Get repair info for 0 bad, : downstairs
Sep 22 23:08:11.587 INFO Repair for extent 0 s:1 d:[ClientId(0)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(1), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(0)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
Testing repair with s:1 r:[ClientId(2)]
Sep 22 23:08:11.587 DEBG Get repair info for 1 source, : downstairs
Sep 22 23:08:11.587 DEBG Get repair info for 2 bad, : downstairs
Sep 22 23:08:11.587 INFO Repair for extent 0 s:1 d:[ClientId(2)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(1), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(2)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
Testing repair with s:2 r:[ClientId(0)]
Sep 22 23:08:11.587 DEBG Get repair info for 2 source, : downstairs
Sep 22 23:08:11.587 DEBG Get repair info for 0 bad, : downstairs
Sep 22 23:08:11.587 INFO Repair for extent 0 s:2 d:[ClientId(0)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(2), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(0)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
Testing repair with s:2 r:[ClientId(1)]
Sep 22 23:08:11.587 DEBG Get repair info for 2 source, : downstairs
Sep 22 23:08:11.587 DEBG Get repair info for 1 bad, : downstairs
Sep 22 23:08:11.587 INFO Repair for extent 0 s:2 d:[ClientId(1)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(2), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(1)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
test live_repair::repair_test::test_solver_flush_lower_needs_repair_one ... ok
Testing repair with s:0 r:[ClientId(1), ClientId(2)]
Sep 22 23:08:11.587 DEBG Get repair info for 0 source, : downstairs
Sep 22 23:08:11.587 DEBG Get repair info for 1 bad, : downstairs
Sep 22 23:08:11.587 DEBG Get repair info for 2 bad, : downstairs
Sep 22 23:08:11.587 INFO Repair for extent 0 s:0 d:[ClientId(1), ClientId(2)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(0), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
Testing repair with s:1 r:[ClientId(0), ClientId(2)]
Sep 22 23:08:11.587 DEBG Get repair info for 1 source, : downstairs
Sep 22 23:08:11.587 DEBG Get repair info for 0 bad, : downstairs
Sep 22 23:08:11.587 DEBG Get repair info for 2 bad, : downstairs
Sep 22 23:08:11.587 INFO Repair for extent 0 s:1 d:[ClientId(0), ClientId(2)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(1), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(0), ClientId(2)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
Testing repair with s:2 r:[ClientId(0), ClientId(1)]
Sep 22 23:08:11.587 DEBG Get repair info for 2 source, : downstairs
Sep 22 23:08:11.587 DEBG Get repair info for 0 bad, : downstairs
Sep 22 23:08:11.587 DEBG Get repair info for 1 bad, : downstairs
Sep 22 23:08:11.587 INFO Repair for extent 0 s:2 d:[ClientId(0), ClientId(1)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(2), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(0), ClientId(1)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
test live_repair::repair_test::test_solver_flush_lower_needs_repair_two ... ok
Testing repair with s:0 r:[ClientId(1)]
Sep 22 23:08:11.588 DEBG Get repair info for 0 source, : downstairs
Sep 22 23:08:11.588 DEBG Get repair info for 1 bad, : downstairs
Sep 22 23:08:11.588 INFO Repair for extent 0 s:0 d:[ClientId(1)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(0), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(1)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
Testing repair with s:0 r:[ClientId(2)]
Sep 22 23:08:11.588 DEBG Get repair info for 0 source, : downstairs
Sep 22 23:08:11.588 DEBG Get repair info for 2 bad, : downstairs
Sep 22 23:08:11.588 INFO Repair for extent 0 s:0 d:[ClientId(2)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(0), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(2)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
Testing repair with s:1 r:[ClientId(0)]
Sep 22 23:08:11.588 DEBG Get repair info for 1 source, : downstairs
Sep 22 23:08:11.588 DEBG Get repair info for 0 bad, : downstairs
Sep 22 23:08:11.588 INFO Repair for extent 0 s:1 d:[ClientId(0)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(1), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(0)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
Testing repair with s:1 r:[ClientId(2)]
Sep 22 23:08:11.588 DEBG Get repair info for 1 source, : downstairs
Sep 22 23:08:11.588 DEBG Get repair info for 2 bad, : downstairs
Sep 22 23:08:11.588 INFO Repair for extent 0 s:1 d:[ClientId(2)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(1), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(2)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
Testing repair with s:2 r:[ClientId(0)]
Sep 22 23:08:11.588 DEBG Get repair info for 2 source, : downstairs
Sep 22 23:08:11.588 DEBG Get repair info for 0 bad, : downstairs
Sep 22 23:08:11.588 INFO Repair for extent 0 s:2 d:[ClientId(0)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(2), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(0)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
Testing repair with s:2 r:[ClientId(1)]
Sep 22 23:08:11.588 DEBG Get repair info for 2 source, : downstairs
Sep 22 23:08:11.588 DEBG Get repair info for 1 bad, : downstairs
Sep 22 23:08:11.588 INFO Repair for extent 0 s:2 d:[ClientId(1)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(2), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(1)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
test live_repair::repair_test::test_solver_gen_higher_needs_repair_one ... ok
Testing repair with s:0 r:[ClientId(1), ClientId(2)]
Sep 22 23:08:11.589 DEBG Get repair info for 0 source, : downstairs
Sep 22 23:08:11.589 DEBG Get repair info for 1 bad, : downstairs
Sep 22 23:08:11.589 DEBG Get repair info for 2 bad, : downstairs
Sep 22 23:08:11.589 INFO Repair for extent 0 s:0 d:[ClientId(1), ClientId(2)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(0), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
Testing repair with s:1 r:[ClientId(0), ClientId(2)]
Sep 22 23:08:11.589 DEBG Get repair info for 1 source, : downstairs
Sep 22 23:08:11.589 DEBG Get repair info for 0 bad, : downstairs
Sep 22 23:08:11.589 DEBG Get repair info for 2 bad, : downstairs
Sep 22 23:08:11.589 INFO Repair for extent 0 s:1 d:[ClientId(0), ClientId(2)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(1), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(0), ClientId(2)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
Testing repair with s:2 r:[ClientId(0), ClientId(1)]
Sep 22 23:08:11.589 DEBG Get repair info for 2 source, : downstairs
Sep 22 23:08:11.589 DEBG Get repair info for 0 bad, : downstairs
Sep 22 23:08:11.589 DEBG Get repair info for 1 bad, : downstairs
Sep 22 23:08:11.589 INFO Repair for extent 0 s:2 d:[ClientId(0), ClientId(1)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(2), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(0), ClientId(1)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
test live_repair::repair_test::test_solver_gen_higher_needs_repair_two ... ok
Testing repair with s:0 r:[ClientId(1)]
Sep 22 23:08:11.589 DEBG Get repair info for 0 source, : downstairs
Sep 22 23:08:11.589 DEBG Get repair info for 1 bad, : downstairs
Sep 22 23:08:11.589 INFO Repair for extent 0 s:0 d:[ClientId(1)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(0), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(1)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
Testing repair with s:0 r:[ClientId(2)]
Sep 22 23:08:11.589 DEBG Get repair info for 0 source, : downstairs
Sep 22 23:08:11.589 DEBG Get repair info for 2 bad, : downstairs
Sep 22 23:08:11.589 INFO Repair for extent 0 s:0 d:[ClientId(2)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(0), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(2)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
Testing repair with s:1 r:[ClientId(0)]
Sep 22 23:08:11.589 DEBG Get repair info for 1 source, : downstairs
Sep 22 23:08:11.589 DEBG Get repair info for 0 bad, : downstairs
Sep 22 23:08:11.589 INFO Repair for extent 0 s:1 d:[ClientId(0)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(1), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(0)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
Testing repair with s:1 r:[ClientId(2)]
Sep 22 23:08:11.589 DEBG Get repair info for 1 source, : downstairs
Sep 22 23:08:11.589 DEBG Get repair info for 2 bad, : downstairs
Sep 22 23:08:11.589 INFO Repair for extent 0 s:1 d:[ClientId(2)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(1), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(2)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
Testing repair with s:2 r:[ClientId(0)]
Sep 22 23:08:11.589 DEBG Get repair info for 2 source, : downstairs
Sep 22 23:08:11.589 DEBG Get repair info for 0 bad, : downstairs
Sep 22 23:08:11.589 INFO Repair for extent 0 s:2 d:[ClientId(0)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(2), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(0)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
Testing repair with s:2 r:[ClientId(1)]
Sep 22 23:08:11.589 DEBG Get repair info for 2 source, : downstairs
Sep 22 23:08:11.589 DEBG Get repair info for 1 bad, : downstairs
Sep 22 23:08:11.589 INFO Repair for extent 0 s:2 d:[ClientId(1)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(2), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(1)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
test live_repair::repair_test::test_solver_gen_lower_needs_repair_one ... ok
Testing repair with s:0 r:[ClientId(1), ClientId(2)]
Sep 22 23:08:11.590 DEBG Get repair info for 0 source, : downstairs
Sep 22 23:08:11.590 DEBG Get repair info for 1 bad, : downstairs
Sep 22 23:08:11.590 DEBG Get repair info for 2 bad, : downstairs
Sep 22 23:08:11.590 INFO Repair for extent 0 s:0 d:[ClientId(1), ClientId(2)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(0), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
Testing repair with s:1 r:[ClientId(0), ClientId(2)]
Sep 22 23:08:11.590 DEBG Get repair info for 1 source, : downstairs
Sep 22 23:08:11.590 DEBG Get repair info for 0 bad, : downstairs
Sep 22 23:08:11.590 DEBG Get repair info for 2 bad, : downstairs
Sep 22 23:08:11.590 INFO Repair for extent 0 s:1 d:[ClientId(0), ClientId(2)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(1), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(0), ClientId(2)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
Testing repair with s:2 r:[ClientId(0), ClientId(1)]
Sep 22 23:08:11.590 DEBG Get repair info for 2 source, : downstairs
Sep 22 23:08:11.590 DEBG Get repair info for 0 bad, : downstairs
Sep 22 23:08:11.590 DEBG Get repair info for 1 bad, : downstairs
Sep 22 23:08:11.590 INFO Repair for extent 0 s:2 d:[ClientId(0), ClientId(1)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(2), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(0), ClientId(1)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
test live_repair::repair_test::test_solver_gen_lower_needs_repair_two ... ok
Sep 22 23:08:11.590 DEBG Get repair info for 0 source, : downstairs
Sep 22 23:08:11.590 DEBG Get repair info for 1 bad, : downstairs
Sep 22 23:08:11.590 DEBG Get repair info for 2 bad, : downstairs
Sep 22 23:08:11.590 INFO No repair needed for extent 0, : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveNoOp { dependencies: [] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
Passed for source 0
Sep 22 23:08:11.590 DEBG Get repair info for 1 source, : downstairs
Sep 22 23:08:11.590 DEBG Get repair info for 0 bad, : downstairs
Sep 22 23:08:11.591 DEBG Get repair info for 2 bad, : downstairs
Sep 22 23:08:11.591 INFO No repair needed for extent 0, : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveNoOp { dependencies: [] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
Passed for source 1
Sep 22 23:08:11.591 DEBG Get repair info for 2 source, : downstairs
Sep 22 23:08:11.591 DEBG Get repair info for 0 bad, : downstairs
Sep 22 23:08:11.591 DEBG Get repair info for 1 bad, : downstairs
Sep 22 23:08:11.591 INFO No repair needed for extent 0, : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveNoOp { dependencies: [] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
Passed for source 2
test live_repair::repair_test::test_solver_no_work ... ok
{"msg":"Crucible stats registered with UUID: 343c66cc-8bda-4c3b-8e86-1cbe92d25fe5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.591714666Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 343c66cc-8bda-4c3b-8e86-1cbe92d25fe5 has session id: 2e11232a-0bde-48dd-b700-3773d8ee797a","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.591751848Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 343c66cc-8bda-4c3b-8e86-1cbe92d25fe5 (7e42f015-9c9b-4236-a152-064c8ad97d22) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.591774283Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.591793172Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 343c66cc-8bda-4c3b-8e86-1cbe92d25fe5 (7e42f015-9c9b-4236-a152-064c8ad97d22) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.591813786Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.591832144Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 343c66cc-8bda-4c3b-8e86-1cbe92d25fe5 (7e42f015-9c9b-4236-a152-064c8ad97d22) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.591852306Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.59187047Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 343c66cc-8bda-4c3b-8e86-1cbe92d25fe5 (7e42f015-9c9b-4236-a152-064c8ad97d22) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.591890749Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.59190877Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 343c66cc-8bda-4c3b-8e86-1cbe92d25fe5 (7e42f015-9c9b-4236-a152-064c8ad97d22) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.591929126Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.591947033Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 343c66cc-8bda-4c3b-8e86-1cbe92d25fe5 (7e42f015-9c9b-4236-a152-064c8ad97d22) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.591967171Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.591985162Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 343c66cc-8bda-4c3b-8e86-1cbe92d25fe5 (7e42f015-9c9b-4236-a152-064c8ad97d22) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.592005341Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.592023191Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 343c66cc-8bda-4c3b-8e86-1cbe92d25fe5 (7e42f015-9c9b-4236-a152-064c8ad97d22) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.592043275Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.592068008Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 343c66cc-8bda-4c3b-8e86-1cbe92d25fe5 (7e42f015-9c9b-4236-a152-064c8ad97d22) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.592089692Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.592107717Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"343c66cc-8bda-4c3b-8e86-1cbe92d25fe5 is now active with session: 7e42f015-9c9b-4236-a152-064c8ad97d22","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.592127276Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 343c66cc-8bda-4c3b-8e86-1cbe92d25fe5 (7e42f015-9c9b-4236-a152-064c8ad97d22) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.592147249Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.592165123Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 343c66cc-8bda-4c3b-8e86-1cbe92d25fe5 (7e42f015-9c9b-4236-a152-064c8ad97d22) Active Faulted Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.592185521Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.592203666Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 343c66cc-8bda-4c3b-8e86-1cbe92d25fe5 (7e42f015-9c9b-4236-a152-064c8ad97d22) Active LiveRepairReady Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.592224003Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.592242265Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 343c66cc-8bda-4c3b-8e86-1cbe92d25fe5 (7e42f015-9c9b-4236-a152-064c8ad97d22) Active LiveRepair Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.59226252Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from LiveRepair to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.5922804Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 343c66cc-8bda-4c3b-8e86-1cbe92d25fe5 (7e42f015-9c9b-4236-a152-064c8ad97d22) Active Faulted Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.592300842Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.592318864Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 343c66cc-8bda-4c3b-8e86-1cbe92d25fe5 (7e42f015-9c9b-4236-a152-064c8ad97d22) Active LiveRepairReady Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.592339105Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.592357277Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Write to Extent 0:2:9 under repair","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:11.592375112Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Write 1:0:9 past extent under repair 0","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:11.592393211Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"IO Write 1005 on eur 0 Added deps 1","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:11.592415212Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Create read repair deps for extent 2","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:11.592433919Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"IO Read  1010 extent 0 added deps 2","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:11.59245185Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test live_repair::repair_test::test_spicy_live_repair ... ok
test mend::test::reconcile_dirty_length_bad - should panic ... ok
{"msg":"Extents 2 dirty","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.593707692Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"First source client ID for extent 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.593741845Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"dirty"}
{"msg":"extent:2  gens: 7 7 7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.593762325Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"dirty"}
{"msg":"extent:2  flush: 2 2 3 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.593782452Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"dirty"}
{"msg":"max_flush now has: [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.593801491Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"dirty"}
{"msg":"find dest for source 2 for extent at index 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.593820893Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"dirty"}
{"msg":"source 2, add dest 0 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.593839776Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"dirty"}
{"msg":"source 2, add dest 1 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.593858327Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"dirty"}
test mend::test::reconcile_dirty_mismatch_c0 ... ok
{"msg":"Extents 2 dirty","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.594466799Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"First source client ID for extent 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.594501523Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"dirty"}
{"msg":"extent:2  gens: 7 7 7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.59452232Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"dirty"}
{"msg":"extent:2  flush: 2 2 2 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.594542527Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"dirty"}
{"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.594562535Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"dirty"}
{"msg":"extent:2  dirty: false true false","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.594581935Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"dirty"}
{"msg":"find dest for source 1 for extent at index 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.594601322Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"dirty"}
{"msg":"source 1, add dest 0 source flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.594620122Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"dirty"}
{"msg":"source 1, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.594638889Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"dirty"}
test mend::test::reconcile_dirty_mismatch_c1 ... ok
{"msg":"Extents 1 dirty","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.595236074Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"First source client ID for extent 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.595268249Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"dirty"}
{"msg":"extent:1  gens: 8 8 7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.595288573Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"dirty"}
{"msg":"extent:1  flush: 1 1 1 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.595308251Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"dirty"}
{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.595327576Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"dirty"}
{"msg":"extent:1  dirty: false false true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.59534634Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"dirty"}
{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.595365904Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"dirty"}
{"msg":"find dest for source 0 for extent at index 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.595384984Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"dirty"}
{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.59540355Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"dirty"}
test mend::test::reconcile_dirty_mismatch_c2 ... ok
{"msg":"Extents 0 dirty","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.595912977Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"First source client ID for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.595945164Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"dirty"}
{"msg":"extent:0  gens: 9 9 9","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.595965214Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"dirty"}
{"msg":"extent:0  flush: 2 2 2 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.595985242Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"dirty"}
{"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.596004858Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"dirty"}
{"msg":"extent:0  dirty: true true true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.596023713Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"dirty"}
{"msg":"find dest for source 0 for extent at index 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.596042818Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"dirty"}
{"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.596061584Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"dirty"}
{"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.596080236Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"dirty"}
{"msg":"Extents 3 dirty","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.596099237Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"First source client ID for extent 3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.596117184Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"dirty"}
{"msg":"extent:3  gens: 7 7 7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.596142426Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"dirty"}
{"msg":"extent:3  flush: 1 1 1 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.596163353Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"dirty"}
{"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.596183079Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"dirty"}
{"msg":"extent:3  dirty: true true true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.596201863Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"dirty"}
{"msg":"find dest for source 0 for extent at index 3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.596221006Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"dirty"}
{"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.5962397Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"dirty"}
{"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.596258745Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"dirty"}
ef.dest [
    ClientId(
        1,
    ),
    ClientId(
        2,
    ),
]
test mend::test::reconcile_dirty_true ... ok
{"msg":"Extent 1 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.596923237Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"First source client ID for extent 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.596957332Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:1  gens: 1 1 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.596978701Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:1  flush: 2 1 1 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.596999115Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"max_flush now has: [ClientId(0)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.597018332Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"find dest for source 0 for extent at index 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.59703786Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"source 0, add dest 1 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.597057028Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"source 0, add dest 2 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.597076045Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"Extent 2 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.597096196Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"First source client ID for extent 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.597114447Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:2  gens: 1 1 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.597133347Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:2  flush: 3 1 1 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.597153485Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"max_flush now has: [ClientId(0)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.597179105Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"find dest for source 0 for extent at index 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.597200049Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"source 0, add dest 1 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.597219077Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"source 0, add dest 2 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.597237974Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"Extent 3 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.597257661Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"First source client ID for extent 3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.59727565Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:3  gens: 1 1 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.597294654Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:3  flush: 1 2 1 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.597314605Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"max_flush now has: [ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.597333531Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"find dest for source 1 for extent at index 3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.597357213Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"source 1, add dest 0 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.597376328Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"source 1, add dest 2 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.597395345Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"Extent 4 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.597414776Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"First source client ID for extent 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.597432855Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:4  gens: 1 1 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.597451705Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:4  flush: 2 2 1 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.597471528Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.597491125Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:4  dirty: false false false","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.59751013Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.597530079Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"find dest for source 0 for extent at index 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.597553919Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"source 0, add dest 2 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.597573842Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"Extent 5 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.597593605Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"First source client ID for extent 5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.597611437Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:5  gens: 1 1 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.597630495Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:5  flush: 3 2 1 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.59765041Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"max_flush now has: [ClientId(0)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.597669606Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"find dest for source 0 for extent at index 5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.597689036Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"source 0, add dest 1 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.597707941Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"source 0, add dest 2 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.597727017Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"Extent 6 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.597746505Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"First source client ID for extent 6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.597764405Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:6  gens: 1 1 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.597783264Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:6  flush: 1 3 1 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.597803344Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"max_flush now has: [ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.597822518Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"find dest for source 1 for extent at index 6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.597841698Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"source 1, add dest 0 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.597860647Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"source 1, add dest 2 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.597879598Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"Extent 7 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.597898948Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"First source client ID for extent 7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.597916941Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:7  gens: 1 1 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.597939744Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:7  flush: 2 3 1 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.597960739Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"max_flush now has: [ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.597979943Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"find dest for source 1 for extent at index 7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.597999385Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"source 1, add dest 0 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.598018304Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"source 1, add dest 2 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.598037119Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"Extent 8 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.59805645Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"First source client ID for extent 8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.598074487Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:8  gens: 1 1 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.59809318Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:8  flush: 3 3 1 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.598113129Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.598132686Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:8  dirty: false false false","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.598151798Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.598171493Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"find dest for source 0 for extent at index 8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.598190897Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"source 0, add dest 2 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.59820975Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
test mend::test::reconcile_flush_a ... ok
{"msg":"Extent 0 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.59887392Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"First source client ID for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.598905524Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:0  gens: 1 1 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.598925927Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:0  flush: 1 1 2 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.598946149Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"max_flush now has: [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.598971496Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"find dest for source 2 for extent at index 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.598992486Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"source 2, add dest 0 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.599011616Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"source 2, add dest 1 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.599030531Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"Extent 1 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.599050375Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"First source client ID for extent 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.599068376Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:1  gens: 1 1 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.599087311Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:1  flush: 2 1 2 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.5991073Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"max_flush now has: [ClientId(0), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.59912735Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:1  dirty: false false false","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.599146687Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"No maxes found, left with: [ClientId(0), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.599166214Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"find dest for source 0 for extent at index 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.599185575Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"source 0, add dest 1 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.599204432Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"Extent 2 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.59922383Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"First source client ID for extent 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.599241702Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:2  gens: 1 1 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.599260694Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:2  flush: 3 1 2 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.599280696Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"max_flush now has: [ClientId(0)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.59929987Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"find dest for source 0 for extent at index 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.599319303Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"source 0, add dest 1 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.599338134Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"source 0, add dest 2 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.599361453Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"Extent 3 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.59938182Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"First source client ID for extent 3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.599399824Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:3  gens: 1 1 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.599418691Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:3  flush: 1 2 2 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.59943882Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"max_flush now has: [ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.599458283Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:3  dirty: false false false","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.599477284Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"No maxes found, left with: [ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.599497144Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"find dest for source 1 for extent at index 3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.599516667Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"source 1, add dest 0 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.599535578Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"Extent 5 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.599554952Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"First source client ID for extent 5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.599572928Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:5  gens: 1 1 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.599591832Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:5  flush: 3 2 2 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.599612012Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"max_flush now has: [ClientId(0)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.599631104Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"find dest for source 0 for extent at index 5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.599650562Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"source 0, add dest 1 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.599669562Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"source 0, add dest 2 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.599688544Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"Extent 6 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.599708058Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"First source client ID for extent 6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.599726001Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:6  gens: 1 1 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.599748895Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:6  flush: 1 3 2 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.599769666Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"max_flush now has: [ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.599788846Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"find dest for source 1 for extent at index 6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.599808214Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"source 1, add dest 0 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.599827293Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"source 1, add dest 2 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.599846217Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"Extent 7 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.599865734Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"First source client ID for extent 7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.59988378Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:7  gens: 1 1 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.59990271Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:7  flush: 2 3 2 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.599922816Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"max_flush now has: [ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.599941935Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"find dest for source 1 for extent at index 7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.599961423Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"source 1, add dest 0 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.599980465Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"source 1, add dest 2 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.599999327Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"Extent 8 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.600018865Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"First source client ID for extent 8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.600036815Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:8  gens: 1 1 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.600055873Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:8  flush: 3 3 2 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.600075894Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.600095375Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:8  dirty: false false false","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.600118515Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.600138966Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"find dest for source 0 for extent at index 8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.600158489Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"source 0, add dest 2 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.600177446Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
test mend::test::reconcile_flush_b ... ok
{"msg":"Extent 0 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.600716964Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"First source client ID for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.600749286Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:0  gens: 1 1 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.600769695Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:0  flush: 1 1 3 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.60078979Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"max_flush now has: [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.600809394Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"find dest for source 2 for extent at index 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.600828694Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"source 2, add dest 0 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.600847677Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"source 2, add dest 1 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.60086659Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"Extent 1 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.600886706Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"First source client ID for extent 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.60090482Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:1  gens: 1 1 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.600923833Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:1  flush: 2 1 3 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.600943918Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"max_flush now has: [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.600963252Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"find dest for source 2 for extent at index 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.600982744Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"source 2, add dest 0 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.601001653Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"source 2, add dest 1 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.601020672Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"Extent 2 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.601046192Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"First source client ID for extent 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.601065472Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:2  gens: 1 1 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.601084391Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:2  flush: 3 1 3 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.601104372Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"max_flush now has: [ClientId(0), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.601123896Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:2  dirty: false false false","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.601142852Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"No maxes found, left with: [ClientId(0), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.601162476Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"find dest for source 0 for extent at index 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.601181765Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"source 0, add dest 1 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.601200659Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"Extent 3 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.60122011Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"First source client ID for extent 3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.601238024Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:3  gens: 1 1 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.601256875Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:3  flush: 1 2 3 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.601276774Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"max_flush now has: [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.601296021Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"find dest for source 2 for extent at index 3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.601315225Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"source 2, add dest 0 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.60133411Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"source 2, add dest 1 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.60135282Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"Extent 4 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.601371856Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"First source client ID for extent 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.60138967Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:4  gens: 1 1 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.601412613Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:4  flush: 2 2 3 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.601433461Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"max_flush now has: [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.601452675Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"find dest for source 2 for extent at index 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.601487937Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"source 2, add dest 0 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.601510673Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"source 2, add dest 1 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.601529984Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"Extent 5 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.601549758Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"First source client ID for extent 5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.601567952Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:5  gens: 1 1 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.601586931Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:5  flush: 3 2 3 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.60160709Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"max_flush now has: [ClientId(0), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.601626578Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:5  dirty: false false false","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.601645569Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"No maxes found, left with: [ClientId(0), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.601665117Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"find dest for source 0 for extent at index 5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.601684431Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"source 0, add dest 1 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.601703302Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"Extent 6 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.601722757Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"First source client ID for extent 6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.601740731Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:6  gens: 1 1 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.601759819Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:6  flush: 1 3 3 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.601779657Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"max_flush now has: [ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.601799118Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:6  dirty: false false false","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.601822709Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"No maxes found, left with: [ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.601843511Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"find dest for source 1 for extent at index 6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.601863196Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"source 1, add dest 0 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.601882222Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"Extent 7 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.601901797Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"First source client ID for extent 7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.601919829Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:7  gens: 1 1 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.601938961Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:7  flush: 2 3 3 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.601958978Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"max_flush now has: [ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.601978465Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:7  dirty: false false false","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.601997474Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"No maxes found, left with: [ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.602016988Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"find dest for source 1 for extent at index 7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.60203627Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"source 1, add dest 0 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.602055303Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
test mend::test::reconcile_flush_c ... ok
test mend::test::reconcile_flush_length_bad - should panic ... ok
{"msg":"Extent 0 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.603139936Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"First source client ID for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.603171884Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:0  gens: 9 9 9","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.603192352Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:0  flush: 1 2 2 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.603212679Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"max_flush now has: [ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.603232117Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:0  dirty: false false false","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.60325709Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"No maxes found, left with: [ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.603277991Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"find dest for source 1 for extent at index 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.603297491Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"source 1, add dest 0 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.603316324Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
test mend::test::reconcile_flush_mismatch_c0 ... ok
{"msg":"Extent 0 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.603836683Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"First source client ID for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.603871595Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:0  gens: 9 9 9","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.603892879Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:0  flush: 1 2 3 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.603913073Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"max_flush now has: [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.603932417Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"find dest for source 2 for extent at index 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.603951964Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"source 2, add dest 0 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.603971264Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"source 2, add dest 1 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.603990454Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"Extent 1 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.604010522Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"First source client ID for extent 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.604028719Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:1  gens: 8 8 8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.604047545Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:1  flush: 2 1 3 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.60406755Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"max_flush now has: [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.60408694Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"find dest for source 2 for extent at index 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.604106238Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"source 2, add dest 0 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.604125194Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"source 2, add dest 1 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.604144104Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"Extent 2 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.604170152Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"First source client ID for extent 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.604189726Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:2  gens: 7 7 7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.604208802Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:2  flush: 3 2 3 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.604228938Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"max_flush now has: [ClientId(0), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.604248496Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:2  dirty: false false false","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.60426782Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"No maxes found, left with: [ClientId(0), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.604287605Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"find dest for source 0 for extent at index 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.604307188Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"source 0, add dest 1 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.604326271Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"Extent 3 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.604345759Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"First source client ID for extent 3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.604363783Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:3  gens: 7 7 7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.604382825Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:3  flush: 3 2 1 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.604402843Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"max_flush now has: [ClientId(0)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.604422173Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"find dest for source 0 for extent at index 3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.604441523Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"source 0, add dest 1 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.604460517Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"source 0, add dest 2 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.60447958Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"Extent 4 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.604499313Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"First source client ID for extent 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.604517194Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:4  gens: 6 6 6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.604540245Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:4  flush: 1 3 3 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.604560967Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"max_flush now has: [ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.604580234Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:4  dirty: false false false","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.604599415Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"No maxes found, left with: [ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.604619095Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"find dest for source 1 for extent at index 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.604638615Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"source 1, add dest 0 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.604657667Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"Extent 5 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.604677156Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"First source client ID for extent 5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.604695038Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:5  gens: 5 5 5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.604714095Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:5  flush: 2 3 2 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.604734189Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"max_flush now has: [ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.604753319Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"find dest for source 1 for extent at index 5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.604772719Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"source 1, add dest 0 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.604791721Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"source 1, add dest 2 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.60481066Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
test mend::test::reconcile_flush_mismatch_c1 ... ok
{"msg":"Extent 0 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.605415599Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"First source client ID for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.605447987Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:0  gens: 9 9 9","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.605468463Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:0  flush: 1 1 2 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.605488722Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"max_flush now has: [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.605508086Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"find dest for source 2 for extent at index 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.605533385Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"source 2, add dest 0 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.60555394Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"source 2, add dest 1 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.605573066Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"Extent 3 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.605592712Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"First source client ID for extent 3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.605610918Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:3  gens: 7 7 7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.60563019Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:3  flush: 1 1 3 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.605650308Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"max_flush now has: [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.605669587Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"find dest for source 2 for extent at index 3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.605688878Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"source 2, add dest 0 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.605707673Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"source 2, add dest 1 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.605726553Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
test mend::test::reconcile_flush_mismatch_c2 ... ok
{"msg":"generation number mismatch 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.606242678Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"First source client ID for extent 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.606274619Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"extent:1  gens: 2 1 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.606294605Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"find dest for source 0 for extent at index 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.606314128Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"source 0, add dest 1 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.606333146Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.606352026Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"generation number mismatch 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.606371719Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"First source client ID for extent 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.606389798Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"extent:2  gens: 3 1 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.606414547Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"find dest for source 0 for extent at index 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.606435102Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"source 0, add dest 1 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.606453808Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.606487634Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"generation number mismatch 3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.606510545Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"First source client ID for extent 3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.606528673Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"extent:3  gens: 1 2 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.606547638Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"find dest for source 1 for extent at index 3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.606567034Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"source 1, add dest 0 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.606585688Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"source 1, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.606604277Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"generation number mismatch 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.606623319Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"First source client ID for extent 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.606641183Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"extent:4  gens: 2 2 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.606660009Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"extent:4  flush: 1 1 1 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.606679468Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.606698637Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"extent:4  dirty: false false false","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.60671754Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.606737099Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"find dest for source 0 for extent at index 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.606756197Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.606774823Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"generation number mismatch 5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.60679382Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"First source client ID for extent 5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.606811613Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"extent:5  gens: 3 2 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.606835161Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"find dest for source 0 for extent at index 5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.606855424Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"source 0, add dest 1 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.60687413Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.606892836Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"generation number mismatch 6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.606911853Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"First source client ID for extent 6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.60692968Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"extent:6  gens: 1 3 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.60694847Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"find dest for source 1 for extent at index 6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.606967661Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"source 1, add dest 0 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.606986345Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"source 1, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.607005079Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"generation number mismatch 7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.607024078Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"First source client ID for extent 7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.607041829Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"extent:7  gens: 2 3 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.607060827Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"find dest for source 1 for extent at index 7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.607080244Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"source 1, add dest 0 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.607098962Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"source 1, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.607117767Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"generation number mismatch 8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.607136774Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"First source client ID for extent 8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.607154503Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"extent:8  gens: 3 3 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.607173323Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"extent:8  flush: 1 1 1 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.607192957Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.607216261Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"extent:8  dirty: false false false","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.60723609Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.607255753Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"find dest for source 0 for extent at index 8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.607274996Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.60729362Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
test mend::test::reconcile_gen_a ... ok
{"msg":"generation number mismatch 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.607869292Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"First source client ID for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.60790371Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"extent:0  gens: 1 1 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.607924914Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"find dest for source 2 for extent at index 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.607944916Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"source 2, add dest 0 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.607964009Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"source 2, add dest 1 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.607983071Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"generation number mismatch 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.608003015Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"First source client ID for extent 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.608021147Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"extent:1  gens: 2 1 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.608039999Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"extent:1  flush: 1 1 1 scs: [ClientId(0), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.608059715Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"max_flush now has: [ClientId(0), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.60807913Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"extent:1  dirty: false false false","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.608098395Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"No maxes found, left with: [ClientId(0), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.608118128Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"find dest for source 0 for extent at index 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.608137593Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"source 0, add dest 1 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.608156392Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"generation number mismatch 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.608182634Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"First source client ID for extent 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.608202348Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"extent:2  gens: 3 1 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.608221312Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"find dest for source 0 for extent at index 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.608240584Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"source 0, add dest 1 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.608259423Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.608278288Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"generation number mismatch 3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.608297551Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"First source client ID for extent 3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.608315543Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"extent:3  gens: 1 2 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.608334485Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"extent:3  flush: 1 1 1 scs: [ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.60835416Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"max_flush now has: [ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.608373526Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"extent:3  dirty: false false false","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.60839269Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"No maxes found, left with: [ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.608412344Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"find dest for source 1 for extent at index 3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.608431648Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"source 1, add dest 0 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.608450466Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"generation number mismatch 5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.608469568Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"First source client ID for extent 5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.608487787Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"extent:5  gens: 3 2 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.608506688Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"find dest for source 0 for extent at index 5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.608525935Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"source 0, add dest 1 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.60854472Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.608568347Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"generation number mismatch 6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.60858856Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"First source client ID for extent 6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.60860683Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"extent:6  gens: 1 3 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.60862571Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"find dest for source 1 for extent at index 6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.608645211Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"source 1, add dest 0 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.608664192Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"source 1, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.608683154Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"generation number mismatch 7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.608702519Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"First source client ID for extent 7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.608720708Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"extent:7  gens: 2 3 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.608739562Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"find dest for source 1 for extent at index 7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.608759276Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"source 1, add dest 0 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.608778166Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"source 1, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.608797117Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"generation number mismatch 8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.6088164Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"First source client ID for extent 8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.608834537Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"extent:8  gens: 3 3 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.608853614Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"extent:8  flush: 1 1 1 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.608873483Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.60889288Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"extent:8  dirty: false false false","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.60891192Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.60893149Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"find dest for source 0 for extent at index 8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.608954807Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.608974693Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
test mend::test::reconcile_gen_b ... ok
{"msg":"generation number mismatch 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.609504368Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"First source client ID for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.609536532Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"extent:0  gens: 1 1 3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.609556602Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"find dest for source 2 for extent at index 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.609576227Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"source 2, add dest 0 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.60959487Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"source 2, add dest 1 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.609613879Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"generation number mismatch 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.609633602Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"First source client ID for extent 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.60965175Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"extent:1  gens: 2 1 3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.609670642Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"find dest for source 2 for extent at index 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.609690106Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"source 2, add dest 0 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.609708923Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"source 2, add dest 1 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.609727802Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"generation number mismatch 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.609746794Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"First source client ID for extent 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.609764686Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"extent:2  gens: 3 1 3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.609783566Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"extent:2  flush: 1 1 1 scs: [ClientId(0), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.609803213Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"max_flush now has: [ClientId(0), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.609822598Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"extent:2  dirty: false false false","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.609841598Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"No maxes found, left with: [ClientId(0), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.609867122Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"find dest for source 0 for extent at index 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.609888071Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"source 0, add dest 1 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.609907076Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"generation number mismatch 3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.609926148Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"First source client ID for extent 3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.60994416Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"extent:3  gens: 1 2 3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.609962973Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"find dest for source 2 for extent at index 3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.60998218Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"source 2, add dest 0 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.610000965Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"source 2, add dest 1 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.610019696Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"generation number mismatch 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.610038719Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"First source client ID for extent 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.610056559Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"extent:4  gens: 2 2 3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.610075608Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"find dest for source 2 for extent at index 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.610095069Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"source 2, add dest 0 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.610113717Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"source 2, add dest 1 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.610132274Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"generation number mismatch 5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.610151221Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"First source client ID for extent 5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.610169051Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"extent:5  gens: 3 2 3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.610187628Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"extent:5  flush: 1 1 1 scs: [ClientId(0), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.610207205Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"max_flush now has: [ClientId(0), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.610226398Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"extent:5  dirty: false false false","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.610245175Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"No maxes found, left with: [ClientId(0), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.610269556Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"find dest for source 0 for extent at index 5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.610289782Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"source 0, add dest 1 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.610308689Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"generation number mismatch 6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.610328156Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"First source client ID for extent 6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.610346155Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"extent:6  gens: 1 3 3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.610364968Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"extent:6  flush: 1 1 1 scs: [ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.610384553Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"max_flush now has: [ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.610404069Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"extent:6  dirty: false false false","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.610422976Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"No maxes found, left with: [ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.610442421Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"find dest for source 1 for extent at index 6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.6104615Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"source 1, add dest 0 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.610497134Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"generation number mismatch 7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.610518674Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"First source client ID for extent 7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.610536822Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"extent:7  gens: 2 3 3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.610555655Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"extent:7  flush: 1 1 1 scs: [ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.610575175Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"max_flush now has: [ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.610594595Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"extent:7  dirty: false false false","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.610613617Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"No maxes found, left with: [ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.610633017Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"find dest for source 1 for extent at index 7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.610656559Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"source 1, add dest 0 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.610676437Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
test mend::test::reconcile_gen_c ... ok
test mend::test::reconcile_gen_length_bad - should panic ... ok
{"msg":"generation number mismatch 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.61177273Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"First source client ID for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.611804458Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"extent:0  gens: 9 8 8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.611824695Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"find dest for source 0 for extent at index 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.611844303Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"source 0, add dest 1 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.611863226Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.611881946Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
test mend::test::reconcile_generation_mismatch_c0 ... ok
{"msg":"generation number mismatch 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.612422718Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"First source client ID for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.612457713Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"extent:0  gens: 9 8 9","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.612478937Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"extent:0  flush: 2 2 2 scs: [ClientId(0), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.612499222Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"max_flush now has: [ClientId(0), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.612519203Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"extent:0  dirty: false false false","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.612538574Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"No maxes found, left with: [ClientId(0), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.612558528Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"find dest for source 0 for extent at index 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.612578249Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"source 0, add dest 1 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.612597239Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
my ef is: ExtentFix { source: ClientId(0), dest: [ClientId(1)] }
test mend::test::reconcile_generation_mismatch_c1 ... ok
{"msg":"generation number mismatch 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.613167372Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"First source client ID for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.613198767Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"extent:0  gens: 7 8 8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.613226139Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"extent:0  flush: 2 2 2 scs: [ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.613247568Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"max_flush now has: [ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.613267099Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"extent:0  dirty: false false false","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.613286184Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"No maxes found, left with: [ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.613305815Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"find dest for source 1 for extent at index 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.613325321Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"source 1, add dest 0 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.613344116Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"generation number mismatch 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.61336373Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"First source client ID for extent 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.613382064Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"extent:1  gens: 8 9 10","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.61340099Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"find dest for source 2 for extent at index 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.613420333Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"source 2, add dest 0 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.613439048Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"source 2, add dest 1 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.613457822Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"generation number mismatch 3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.613476929Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"First source client ID for extent 3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.613494987Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"extent:3  gens: 5 4 3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.613514326Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"find dest for source 0 for extent at index 3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.613533534Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"source 0, add dest 1 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.613552105Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.613570856Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
test mend::test::reconcile_generation_mismatch_c2 ... ok
test mend::test::reconcile_length_mismatch - should panic ... ok
{"msg":"Extent 0 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.614651982Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"First source client ID for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.61468496Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:0  gens: 9 9 8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.614705651Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:0  flush: 1 2 3 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.614725552Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"max_flush now has: [ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.614744966Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"find dest for source 1 for extent at index 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.614764707Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"source 1, add dest 0 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.614783932Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"source 1, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.614803019Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"Extent 1 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.614823215Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"First source client ID for extent 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.61484154Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:1  gens: 7 8 8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.614860725Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:1  flush: 1 1 2 scs: [ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.61488056Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"max_flush now has: [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.614899969Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"find dest for source 2 for extent at index 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.614919738Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"source 2, add dest 0 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.614938712Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"source 2, add dest 1 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.614957935Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"Extent 2 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.614977431Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"First source client ID for extent 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.614995621Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:2  gens: 7 9 7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.61501479Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"find dest for source 1 for extent at index 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.615034258Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"source 1, add dest 0 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.615058782Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"source 1, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.615079133Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"Extent 3 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.615098976Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"First source client ID for extent 3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.615117167Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"extent:3  gens: 7 8 9","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.615136344Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"find dest for source 2 for extent at index 3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.615156002Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"source 2, add dest 0 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.615175091Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
{"msg":"source 2, add dest 1 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.61519412Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"flush_mismatch"}
test mend::test::reconcile_multiple_source ... ok
test mend::test::reconcile_one ... ok
{"msg":"Extents 0 dirty","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.616077862Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"First source client ID for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.616110484Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"dirty"}
{"msg":"extent:0  gens: 9 9 9","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.616130635Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"dirty"}
{"msg":"extent:0  flush: 2 2 3 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.616150589Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"dirty"}
{"msg":"max_flush now has: [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.616169573Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"dirty"}
{"msg":"find dest for source 2 for extent at index 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.616188723Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"dirty"}
{"msg":"source 2, add dest 0 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.616207554Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"dirty"}
{"msg":"source 2, add dest 1 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.61622622Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"dirty"}
{"msg":"Extents 2 dirty","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.616245585Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"First source client ID for extent 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.616263483Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"dirty"}
{"msg":"extent:2  gens: 7 7 8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.61628207Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"dirty"}
{"msg":"find dest for source 2 for extent at index 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.616301296Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"dirty"}
{"msg":"source 2, add dest 0 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.616319901Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"dirty"}
{"msg":"source 2, add dest 1 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.616344351Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"dirty"}
{"msg":"Extents 3 dirty","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.616364502Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"First source client ID for extent 3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.616382458Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"dirty"}
{"msg":"extent:3  gens: 7 7 7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.616401067Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"dirty"}
{"msg":"extent:3  flush: 1 1 1 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.616420966Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"dirty"}
{"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.616440743Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"dirty"}
{"msg":"extent:3  dirty: true true true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.61645953Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"dirty"}
{"msg":"find dest for source 0 for extent at index 3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.616487686Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"dirty"}
{"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.616513938Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"dirty"}
{"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.616533596Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"dirty"}
{"msg":"generation number mismatch 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.616552973Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"First source client ID for extent 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.616570982Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"extent:1  gens: 8 7 8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.61658991Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"extent:1  flush: 1 1 1 scs: [ClientId(0), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.616609713Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"max_flush now has: [ClientId(0), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.616629054Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"extent:1  dirty: false false false","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.616648097Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"No maxes found, left with: [ClientId(0), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.616667753Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"find dest for source 0 for extent at index 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.616687075Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
{"msg":"source 0, add dest 1 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.616705864Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"mrl":"gen_mismatch"}
test mend::test::reconcile_one_of_each ... ok
test mend::test::reconcile_to_repair ... ok
{"msg":"Crucible stats registered with UUID: 39c08d56-d632-40e8-a607-2d3ab8106ac1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.617656408Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 39c08d56-d632-40e8-a607-2d3ab8106ac1 has session id: e10d0efe-35f3-4ac4-9215-8d9ca8ed9712","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.617690401Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"39c08d56-d632-40e8-a607-2d3ab8106ac1 is now active with session: f5b05529-5456-441f-a40c-bb1539235cdd","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.617711144Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Decryption failed even though integrity hash matched!","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:11.618399624Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"Decryption failed with correct hash","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.618426932Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[0] Reports error DecryptionError on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Read { dependencies: [], requests: [ReadRequest { eid: 0, offset: Block { value: 7, shift: 9 } }] }, state: ClientData([InProgress, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.618457779Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
test test::up_test::bad_decryption_means_panic ... ok
{"msg":"No match for integrity hash","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.619354356Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"Expected: 0x2710 != Computed: 0xf0c5cd5c81177cbb","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.619387509Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[0] Reports error HashMismatch on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Read { dependencies: [], requests: [ReadRequest { eid: 0, offset: Block { value: 7, shift: 9 } }] }, state: ClientData([InProgress, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.619418336Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
test test::up_test::bad_hash_on_encrypted_read_panic ... ok
{"msg":"Crucible stats registered with UUID: 5b8a6a51-7c13-4127-bc35-9bb8891d5eb0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.620007067Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 5b8a6a51-7c13-4127-bc35-9bb8891d5eb0 has session id: be8e0e0b-31a3-43d6-8c20-f3f5d22372cb","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.620040738Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"5b8a6a51-7c13-4127-bc35-9bb8891d5eb0 is now active with session: 1bb8800d-f95c-49b5-9cb3-d4ffa956aad3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.620061794Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"No match computed hash:0x78fc2d7d9eaf9bbf","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.620084497Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"No match          hash:0x2710","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.620103885Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"Data from hash:","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.620122483Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[0]:1","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.620147187Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1]:1","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.620167083Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2]:1","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.620185293Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[3]:1","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.620203577Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[4]:1","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.620221666Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[5]:1","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.620239581Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[0] Reports error HashMismatch on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Read { dependencies: [], requests: [ReadRequest { eid: 0, offset: Block { value: 7, shift: 9 } }] }, state: ClientData([InProgress, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.620268874Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
test test::up_test::bad_read_hash_means_panic ... ok
{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.620839377Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: 2a83ae12-f4fc-42f6-b107-998ee5c4b845","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.62087354Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"00000000-0000-0000-0000-000000000000 is now active with session: e6c4a2df-5e4e-4055-83af-21f523de50ed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.620894556Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"00000000-0000-0000-0000-000000000000 set deactivating.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.620913117Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] check deactivate YES","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.621005446Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 00000000-0000-0000-0000-000000000000 (e6c4a2df-5e4e-4055-83af-21f523de50ed) Active Active Active ds_transition to Deactivated","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.621033698Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from Active to Deactivated","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.621052673Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] check deactivate YES","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.621070459Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 00000000-0000-0000-0000-000000000000 (e6c4a2df-5e4e-4055-83af-21f523de50ed) Deactivated Active Active ds_transition to Deactivated","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.621090875Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from Active to Deactivated","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.621108876Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] deactivate job 1001 not New flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.621126763Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"deactivate transition checking...","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.621150103Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"deactivate_transition Deactivated NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.621169242Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"deactivate_transition Active NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.621187107Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"deactivate_transition Deactivated NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.621204927Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] check deactivate YES","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.621222306Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 00000000-0000-0000-0000-000000000000 (e6c4a2df-5e4e-4055-83af-21f523de50ed) Deactivated Active Deactivated ds_transition to Deactivated","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.621242648Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Active to Deactivated","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.621260719Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 00000000-0000-0000-0000-000000000000 Gone missing, transition from Deactivated to New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.62128051Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 00000000-0000-0000-0000-000000000000 Gone missing, transition from Deactivated to New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.621299938Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 00000000-0000-0000-0000-000000000000 Gone missing, transition from Deactivated to New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.621319088Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"deactivate transition checking...","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.621336967Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"deactivate_transition New Maybe ","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.621354913Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"deactivate_transition New Maybe ","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.621372783Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"deactivate_transition New Maybe ","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.621390308Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"All DS in the proper state! -> INIT","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.621408203Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::deactivate_after_work_completed_write ... ok
{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.622048808Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: b689aa0f-2697-4d8e-bc0f-51129a721e77","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.622082419Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"00000000-0000-0000-0000-000000000000 is now active with session: e6090615-6d96-4f4c-9110-b1e9f5922673","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.622103041Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"00000000-0000-0000-0000-000000000000 set deactivating.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.622121599Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] check deactivate YES","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.62216032Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 00000000-0000-0000-0000-000000000000 (e6090615-6d96-4f4c-9110-b1e9f5922673) Active Active Active ds_transition to Deactivated","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.622183291Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from Active to Deactivated","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.622201444Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] check deactivate YES","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.622219307Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 00000000-0000-0000-0000-000000000000 (e6090615-6d96-4f4c-9110-b1e9f5922673) Deactivated Active Active ds_transition to Deactivated","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.622239795Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from Active to Deactivated","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.622257758Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] deactivate job 1001 not New flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.622275728Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"deactivate transition checking...","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.622293467Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"deactivate_transition Deactivated NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.622311229Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"deactivate_transition Active NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.622328995Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"deactivate_transition Deactivated NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.622346821Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] check deactivate YES","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.62236435Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 00000000-0000-0000-0000-000000000000 (e6090615-6d96-4f4c-9110-b1e9f5922673) Deactivated Active Deactivated ds_transition to Deactivated","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.622384862Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Active to Deactivated","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.622402766Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 00000000-0000-0000-0000-000000000000 Gone missing, transition from Deactivated to New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.622422461Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 00000000-0000-0000-0000-000000000000 Gone missing, transition from Deactivated to New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.622441869Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 00000000-0000-0000-0000-000000000000 Gone missing, transition from Deactivated to New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.622461049Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"deactivate transition checking...","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.622488838Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"deactivate_transition New Maybe ","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.622512905Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"deactivate_transition New Maybe ","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.622535699Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"deactivate_transition New Maybe ","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.622554513Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"All DS in the proper state! -> INIT","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.622572328Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::deactivate_after_work_completed_write_unwritten ... ok
{"msg":"Crucible stats registered with UUID: 7c40f313-4b9b-431a-8bc5-4e217b2b2806","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.623160077Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 7c40f313-4b9b-431a-8bc5-4e217b2b2806 has session id: b12049c9-7e3d-41db-864c-b847f7405e20","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.623193436Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"7c40f313-4b9b-431a-8bc5-4e217b2b2806 is now active with session: 4a9e9c9c-ea59-404d-ac9a-e3b3c137be22","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.623214239Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::deactivate_ds_not_when_active ... ok
{"msg":"Crucible stats registered with UUID: 47268ea8-52a4-4142-bcaa-302a9423a8fe","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.623768152Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 47268ea8-52a4-4142-bcaa-302a9423a8fe has session id: 8470d040-f972-41fb-b673-d422f8314eef","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.623801767Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::deactivate_ds_not_when_initializing ... ok
{"msg":"Crucible stats registered with UUID: 73975c6f-8deb-48ac-a65e-8e75d0cd386f","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.624349946Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 73975c6f-8deb-48ac-a65e-8e75d0cd386f has session id: 7ae18f99-3dac-4ea1-ba26-72f927fe3b3c","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.624383313Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"73975c6f-8deb-48ac-a65e-8e75d0cd386f is now active with session: 86331d61-a1ad-4418-a2e9-7028dcf66f35","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.624404235Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"73975c6f-8deb-48ac-a65e-8e75d0cd386f set deactivating.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.624422534Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::deactivate_not_when_active ... ok
{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.624966956Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: 36e7bc94-548c-425f-a8b5-a631300e85f2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.625000171Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"00000000-0000-0000-0000-000000000000 is now active with session: 54f115b4-a287-44a4-b8c4-398744c6c761","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.625020815Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"00000000-0000-0000-0000-000000000000 set deactivating.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.625039215Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] deactivate job 1001 not New flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.625076352Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] deactivate job 1001 not New flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.625095472Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] deactivate job 1001 not New flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.625119313Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"deactivate transition checking...","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.625138476Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"deactivate_transition Active NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.625156132Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"deactivate_transition Active NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.625173932Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"deactivate_transition Active NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.625191631Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::deactivate_not_without_flush_write ... ok
{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.625739547Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: c2254082-863e-4bff-9654-370c18a50ef8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.62577282Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"00000000-0000-0000-0000-000000000000 is now active with session: 0bc0db6e-7184-4297-a860-75fd98a494bc","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.625793604Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"00000000-0000-0000-0000-000000000000 set deactivating.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.625812251Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] deactivate job 1001 not New flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.625847099Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] deactivate job 1001 not New flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.62586638Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] deactivate job 1001 not New flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.62588442Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"deactivate transition checking...","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.625902366Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"deactivate_transition Active NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.62592013Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"deactivate_transition Active NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.625937873Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"deactivate_transition Active NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.625955469Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::deactivate_not_without_flush_write_unwritten ... ok
{"msg":"Crucible stats registered with UUID: e272e714-f2cd-4519-8e52-011bc9f14ce1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.626505641Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible e272e714-f2cd-4519-8e52-011bc9f14ce1 has session id: bfdaad5f-47fe-4862-838c-e1f630e0e373","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.626538954Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"e272e714-f2cd-4519-8e52-011bc9f14ce1 is now active with session: 6614485b-fa35-4ce7-9cd1-b06d76215aaa","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.626559525Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"e272e714-f2cd-4519-8e52-011bc9f14ce1 set deactivating.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.626583813Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] deactivate, no work so YES","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.626603001Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] e272e714-f2cd-4519-8e52-011bc9f14ce1 (6614485b-fa35-4ce7-9cd1-b06d76215aaa) Active Active Active ds_transition to Deactivated","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.626623299Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from Active to Deactivated","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.626641619Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] deactivate, no work so YES","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.626659381Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] e272e714-f2cd-4519-8e52-011bc9f14ce1 (6614485b-fa35-4ce7-9cd1-b06d76215aaa) Deactivated Active Active ds_transition to Deactivated","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.626679519Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Active to Deactivated","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.626697462Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] deactivate, no work so YES","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.626715172Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] e272e714-f2cd-4519-8e52-011bc9f14ce1 (6614485b-fa35-4ce7-9cd1-b06d76215aaa) Deactivated Deactivated Active ds_transition to Deactivated","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.626735564Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from Active to Deactivated","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.626753641Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] e272e714-f2cd-4519-8e52-011bc9f14ce1 Gone missing, transition from Deactivated to New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.626773131Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] e272e714-f2cd-4519-8e52-011bc9f14ce1 Gone missing, transition from Deactivated to New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.626792241Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] e272e714-f2cd-4519-8e52-011bc9f14ce1 Gone missing, transition from Deactivated to New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.626811519Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"deactivate transition checking...","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.626829217Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"deactivate_transition New Maybe ","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.626846941Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"deactivate_transition New Maybe ","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.626864609Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"deactivate_transition New Maybe ","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.626882253Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"All DS in the proper state! -> INIT","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.626900137Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::deactivate_when_empty ... ok
{"msg":"Crucible stats registered with UUID: e9df70e4-2285-4c18-959f-b6883a32cd53","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.627455943Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible e9df70e4-2285-4c18-959f-b6883a32cd53 has session id: 8afbce38-5825-4804-b8f5-0c8f23207fef","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.627505983Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] e9df70e4-2285-4c18-959f-b6883a32cd53 (4d20ac4f-cf0d-423a-bf8d-e1296ac427a8) New New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.627531171Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::downstairs_bad_transition_wq - should panic ... ok
{"msg":"Crucible stats registered with UUID: 76d2a4b1-8b63-4bd7-a38a-9aa2230ee510","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.628305616Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 76d2a4b1-8b63-4bd7-a38a-9aa2230ee510 has session id: f06bf8bb-0546-4a85-82b8-a56305cb9ec1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.628338564Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 76d2a4b1-8b63-4bd7-a38a-9aa2230ee510 (3fa475c2-a6fe-4f75-8518-36f337feca6a) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.628360026Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.628378288Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 76d2a4b1-8b63-4bd7-a38a-9aa2230ee510 (3fa475c2-a6fe-4f75-8518-36f337feca6a) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.628398625Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.628416617Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 76d2a4b1-8b63-4bd7-a38a-9aa2230ee510 (3fa475c2-a6fe-4f75-8518-36f337feca6a) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.628436693Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.628454539Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 76d2a4b1-8b63-4bd7-a38a-9aa2230ee510 (3fa475c2-a6fe-4f75-8518-36f337feca6a) Active New New ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.628480274Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.628507521Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::downstairs_transition_active_faulted ... ok
{"msg":"Crucible stats registered with UUID: 345ad8eb-d2f8-45ed-ac5c-022696dd3420","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.629058175Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 345ad8eb-d2f8-45ed-ac5c-022696dd3420 has session id: bb4c7da7-615d-461b-805a-7472074e8ca2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.629091205Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 345ad8eb-d2f8-45ed-ac5c-022696dd3420 (0906270a-8d2a-40c7-98ee-734d146362aa) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.629112771Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.629131028Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 345ad8eb-d2f8-45ed-ac5c-022696dd3420 (0906270a-8d2a-40c7-98ee-734d146362aa) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.629151202Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.629174818Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 345ad8eb-d2f8-45ed-ac5c-022696dd3420 (0906270a-8d2a-40c7-98ee-734d146362aa) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.629196373Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.629214574Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 345ad8eb-d2f8-45ed-ac5c-022696dd3420 (0906270a-8d2a-40c7-98ee-734d146362aa) Active New New ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.62923469Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.629252946Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::downstairs_transition_active_to_faulted ... ok
{"msg":"Crucible stats registered with UUID: 7a88b5eb-5f23-4308-81cb-e06f5a336ac5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.629812711Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 7a88b5eb-5f23-4308-81cb-e06f5a336ac5 has session id: 6f0d1fb7-fa9c-498f-b6eb-8bd9ae7fccb5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.629846464Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 7a88b5eb-5f23-4308-81cb-e06f5a336ac5 (a2d56bbb-f5a7-4244-b1fb-1a80a00fcc28) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.629868005Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.629886441Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 7a88b5eb-5f23-4308-81cb-e06f5a336ac5 (a2d56bbb-f5a7-4244-b1fb-1a80a00fcc28) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.629906611Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.629924622Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 7a88b5eb-5f23-4308-81cb-e06f5a336ac5 (a2d56bbb-f5a7-4244-b1fb-1a80a00fcc28) WaitQuorum New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.6299448Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::downstairs_transition_backwards - should panic ... ok
{"msg":"Crucible stats registered with UUID: 7a05383d-e1df-40f2-9c5a-a11413c20abc","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.630539868Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 7a05383d-e1df-40f2-9c5a-a11413c20abc has session id: b81aebcc-ae6c-4a8c-81fe-e5ceb3a0e0e9","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.630573441Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 7a05383d-e1df-40f2-9c5a-a11413c20abc (f2055bc4-e802-429a-b8db-9cc1be84dde0) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.630595065Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.630613464Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 7a05383d-e1df-40f2-9c5a-a11413c20abc (f2055bc4-e802-429a-b8db-9cc1be84dde0) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.630641513Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.63066095Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 7a05383d-e1df-40f2-9c5a-a11413c20abc (f2055bc4-e802-429a-b8db-9cc1be84dde0) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.630681188Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.630698987Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 7a05383d-e1df-40f2-9c5a-a11413c20abc (f2055bc4-e802-429a-b8db-9cc1be84dde0) Active New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.630718947Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::downstairs_transition_bad_active - should panic ... ok
{"msg":"Crucible stats registered with UUID: 6e224231-e7bb-41fb-ab05-7d5803f086a9","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.631306749Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 6e224231-e7bb-41fb-ab05-7d5803f086a9 has session id: 0ec9a497-79c3-4ad0-b142-bc2335c3becc","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.631339817Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 6e224231-e7bb-41fb-ab05-7d5803f086a9 (e6ef2e6a-c2b1-430a-b04d-3f7ad93d8fb5) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.631361204Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.631379608Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 6e224231-e7bb-41fb-ab05-7d5803f086a9 (e6ef2e6a-c2b1-430a-b04d-3f7ad93d8fb5) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.631399825Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.631418101Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 6e224231-e7bb-41fb-ab05-7d5803f086a9 (e6ef2e6a-c2b1-430a-b04d-3f7ad93d8fb5) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.631438044Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.631456021Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 6e224231-e7bb-41fb-ab05-7d5803f086a9 (e6ef2e6a-c2b1-430a-b04d-3f7ad93d8fb5) Active New New ds_transition to Offline","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.631482468Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from Active to Offline","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.631510256Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 6e224231-e7bb-41fb-ab05-7d5803f086a9 (e6ef2e6a-c2b1-430a-b04d-3f7ad93d8fb5) Offline New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.631531575Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::downstairs_transition_bad_offline - should panic ... ok
{"msg":"Crucible stats registered with UUID: 749ed771-276f-481e-9643-b0214e228df1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.6321209Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 749ed771-276f-481e-9643-b0214e228df1 has session id: 77b322fe-2418-4558-9366-1709743b4d80","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.632159306Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 749ed771-276f-481e-9643-b0214e228df1 (6d741d92-52c3-4732-913b-3f6c75095aab) New New New ds_transition to Replay","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.632181859Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::downstairs_transition_bad_replay - should panic ... ok
{"msg":"Crucible stats registered with UUID: 1450174a-c9b8-44d8-9ece-9f3ed352a136","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.632772315Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 1450174a-c9b8-44d8-9ece-9f3ed352a136 has session id: 98a6c46c-bb1f-4a68-817f-9f70df6eb538","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.63280539Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 1450174a-c9b8-44d8-9ece-9f3ed352a136 (f53bc8ad-c575-4a97-9221-26650f3c9328) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.63282715Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.632845456Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 1450174a-c9b8-44d8-9ece-9f3ed352a136 (f53bc8ad-c575-4a97-9221-26650f3c9328) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.632865426Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.63288338Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 1450174a-c9b8-44d8-9ece-9f3ed352a136 (f53bc8ad-c575-4a97-9221-26650f3c9328) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.632903285Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.63292119Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"1450174a-c9b8-44d8-9ece-9f3ed352a136 is now active with session: f53bc8ad-c575-4a97-9221-26650f3c9328","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.632940595Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 1450174a-c9b8-44d8-9ece-9f3ed352a136 (f53bc8ad-c575-4a97-9221-26650f3c9328) Active New New ds_transition to Deactivated","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.632960542Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from Active to Deactivated","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.632978438Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 1450174a-c9b8-44d8-9ece-9f3ed352a136 (f53bc8ad-c575-4a97-9221-26650f3c9328) Deactivated New New ds_transition to New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.632998373Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from Deactivated to New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.63301619Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::downstairs_transition_deactivate_new ... ok
{"msg":"Crucible stats registered with UUID: 71fc23d9-91e0-4c81-b9af-810c36135e15","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.633561553Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 71fc23d9-91e0-4c81-b9af-810c36135e15 has session id: e9c5041a-5f8b-49b8-9ae5-f7e975a39549","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.6335954Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 71fc23d9-91e0-4c81-b9af-810c36135e15 (90a19801-74d1-4aed-b5cb-6a0df88d651c) New New New ds_transition to Deactivated","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.633622958Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::downstairs_transition_deactivate_not_new - should panic ... ok
{"msg":"Crucible stats registered with UUID: e51d3849-32fd-4d3f-8571-9d95ec08c424","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.634217908Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible e51d3849-32fd-4d3f-8571-9d95ec08c424 has session id: 6cdcf234-ce3b-4bd5-91f6-a184b9341c83","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.634250959Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] e51d3849-32fd-4d3f-8571-9d95ec08c424 (76bb2160-b19c-440e-ae54-4301d50f5e0e) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.634272523Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.634291134Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] e51d3849-32fd-4d3f-8571-9d95ec08c424 (76bb2160-b19c-440e-ae54-4301d50f5e0e) WaitActive New New ds_transition to Deactivated","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.634311274Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::downstairs_transition_deactivate_not_wa - should panic ... ok
{"msg":"Crucible stats registered with UUID: 19160081-922d-4e17-878b-bbb0b9008597","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.634870743Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 19160081-922d-4e17-878b-bbb0b9008597 has session id: c79bf584-6377-4cf1-b71b-e553b46e6ec8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.634904107Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 19160081-922d-4e17-878b-bbb0b9008597 (b1dd3b46-aac7-4b32-b9e4-5b904af67683) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.634925711Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.63494416Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 19160081-922d-4e17-878b-bbb0b9008597 (b1dd3b46-aac7-4b32-b9e4-5b904af67683) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.634964395Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.634982381Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 19160081-922d-4e17-878b-bbb0b9008597 (b1dd3b46-aac7-4b32-b9e4-5b904af67683) WaitQuorum New New ds_transition to Deactivated","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.635002642Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::downstairs_transition_deactivate_not_wq - should panic ... ok
{"msg":"Crucible stats registered with UUID: 31c3dd37-38a2-4236-92ff-a8dfac26c022","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.635589532Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 31c3dd37-38a2-4236-92ff-a8dfac26c022 has session id: 97897687-7855-45af-8b4b-00a9c45fbb66","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.635623013Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 31c3dd37-38a2-4236-92ff-a8dfac26c022 (fe9c2b0a-c6f2-4ec3-a7b7-e3fd494cc14e) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.635650429Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.635670344Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 31c3dd37-38a2-4236-92ff-a8dfac26c022 (fe9c2b0a-c6f2-4ec3-a7b7-e3fd494cc14e) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.635690588Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.635708692Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 31c3dd37-38a2-4236-92ff-a8dfac26c022 (fe9c2b0a-c6f2-4ec3-a7b7-e3fd494cc14e) WaitQuorum New New ds_transition to Deactivated","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.635728922Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::downstairs_transition_disconnect_no_active - should panic ... ok
{"msg":"Crucible stats registered with UUID: 86ba2ad7-5cb3-44f1-a126-2cb4b4d215f5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.636322028Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 86ba2ad7-5cb3-44f1-a126-2cb4b4d215f5 has session id: 577bacc8-951e-4686-be55-6c769efb4f5d","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.636355308Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 86ba2ad7-5cb3-44f1-a126-2cb4b4d215f5 (f180244a-cb04-4921-a34a-9654b24a63c8) New New New ds_transition to Offline","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.636376644Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::downstairs_transition_no_new_to_offline - should panic ... ok
{"msg":"Crucible stats registered with UUID: 0fe70785-506d-4660-ab80-bbb7de2b0b2a","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.636968509Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 0fe70785-506d-4660-ab80-bbb7de2b0b2a has session id: 0166ba75-9a2e-4969-8f6d-c295ff66a1b3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.637001807Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 0fe70785-506d-4660-ab80-bbb7de2b0b2a (74f5fac3-72d8-45bf-8e50-f8a9502b7636) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.63702304Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.637041626Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 0fe70785-506d-4660-ab80-bbb7de2b0b2a (74f5fac3-72d8-45bf-8e50-f8a9502b7636) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.637061875Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.637079898Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 0fe70785-506d-4660-ab80-bbb7de2b0b2a (74f5fac3-72d8-45bf-8e50-f8a9502b7636) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.637099986Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.637118112Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::downstairs_transition_normal ... ok
{"msg":"Crucible stats registered with UUID: 0db93807-24cb-479d-a7e5-cfaccfb35c64","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.63766033Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 0db93807-24cb-479d-a7e5-cfaccfb35c64 has session id: 1771c71d-8f3f-4715-a2ae-e2bb61bdae3c","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.637699078Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 0db93807-24cb-479d-a7e5-cfaccfb35c64 (3fadfdd8-680f-47fd-a927-4dc587f6e2db) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.637816048Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.637841862Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 0db93807-24cb-479d-a7e5-cfaccfb35c64 (3fadfdd8-680f-47fd-a927-4dc587f6e2db) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.637862728Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.637880847Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 0db93807-24cb-479d-a7e5-cfaccfb35c64 (3fadfdd8-680f-47fd-a927-4dc587f6e2db) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.637900901Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.637918799Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 0db93807-24cb-479d-a7e5-cfaccfb35c64 (3fadfdd8-680f-47fd-a927-4dc587f6e2db) Active New New ds_transition to Offline","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.637938554Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from Active to Offline","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.637956367Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 0db93807-24cb-479d-a7e5-cfaccfb35c64 (3fadfdd8-680f-47fd-a927-4dc587f6e2db) Offline New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.637976302Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::downstairs_transition_offline_no_active - should panic ... ok
{"msg":"Crucible stats registered with UUID: ffcb4455-79ad-4fa6-bc30-85dd4266da80","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.6385669Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible ffcb4455-79ad-4fa6-bc30-85dd4266da80 has session id: 2d180d76-07d3-4b3c-a8d2-bb9d00ba48fe","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.638600362Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] ffcb4455-79ad-4fa6-bc30-85dd4266da80 (e72a4722-8393-4970-9058-71709c18d6d4) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.638621913Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.638640368Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] ffcb4455-79ad-4fa6-bc30-85dd4266da80 (e72a4722-8393-4970-9058-71709c18d6d4) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.638660639Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.638678765Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"ffcb4455-79ad-4fa6-bc30-85dd4266da80 is now active with session: e72a4722-8393-4970-9058-71709c18d6d4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.638698441Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] ffcb4455-79ad-4fa6-bc30-85dd4266da80 (e72a4722-8393-4970-9058-71709c18d6d4) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.6387244Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.638743849Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] ffcb4455-79ad-4fa6-bc30-85dd4266da80 (e72a4722-8393-4970-9058-71709c18d6d4) Active New New ds_transition to Offline","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.6387641Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from Active to Offline","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.638782071Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] ffcb4455-79ad-4fa6-bc30-85dd4266da80 (e72a4722-8393-4970-9058-71709c18d6d4) Offline New New ds_transition to Replay","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.638801958Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from Offline to Replay","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.638820009Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::downstairs_transition_replay ... ok
{"msg":"Crucible stats registered with UUID: 89776979-2662-40aa-a9f1-044189059988","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.639365849Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 89776979-2662-40aa-a9f1-044189059988 has session id: 21c08393-c5da-4e80-b02e-bf50a22c6912","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.639399564Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 89776979-2662-40aa-a9f1-044189059988 (4c6078b9-6175-4cb8-b8ef-d98478e11fec) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.639421107Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.639439649Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 89776979-2662-40aa-a9f1-044189059988 (4c6078b9-6175-4cb8-b8ef-d98478e11fec) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.639459987Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.639487972Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 89776979-2662-40aa-a9f1-044189059988 (4c6078b9-6175-4cb8-b8ef-d98478e11fec) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.639514711Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.639533469Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 89776979-2662-40aa-a9f1-044189059988 (4c6078b9-6175-4cb8-b8ef-d98478e11fec) Active New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.639553411Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::downstairs_transition_same_active - should panic ... ok
{"msg":"Crucible stats registered with UUID: 5d2fceb8-8ac5-46e1-a268-e3d9138ee48b","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.640139108Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 5d2fceb8-8ac5-46e1-a268-e3d9138ee48b has session id: 748a65f6-2812-4f9c-bc13-a06ce04e9490","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.640177955Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 5d2fceb8-8ac5-46e1-a268-e3d9138ee48b (0a4120a9-dea3-4dca-8555-e9791a197bbd) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.640200916Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.640219418Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 5d2fceb8-8ac5-46e1-a268-e3d9138ee48b (0a4120a9-dea3-4dca-8555-e9791a197bbd) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.640239731Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.640257926Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 5d2fceb8-8ac5-46e1-a268-e3d9138ee48b (0a4120a9-dea3-4dca-8555-e9791a197bbd) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.640278001Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.640296014Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 5d2fceb8-8ac5-46e1-a268-e3d9138ee48b (0a4120a9-dea3-4dca-8555-e9791a197bbd) Active New New ds_transition to Offline","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.640316111Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from Active to Offline","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.640334004Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 5d2fceb8-8ac5-46e1-a268-e3d9138ee48b (0a4120a9-dea3-4dca-8555-e9791a197bbd) Offline New New ds_transition to Offline","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.640354005Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::downstairs_transition_same_offline - should panic ... ok
{"msg":"Crucible stats registered with UUID: ea2dab8a-1336-463d-98a4-e05d78c9b678","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.640993407Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible ea2dab8a-1336-463d-98a4-e05d78c9b678 has session id: afdf035a-a12f-491c-b567-fdf52af90228","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.641027279Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] ea2dab8a-1336-463d-98a4-e05d78c9b678 (9c7cab95-9236-449e-8dd8-96c0eba91581) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.641048752Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.641067231Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] ea2dab8a-1336-463d-98a4-e05d78c9b678 (9c7cab95-9236-449e-8dd8-96c0eba91581) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.641087589Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::downstairs_transition_same_wa - should panic ... ok
{"msg":"Crucible stats registered with UUID: 1f7f81c6-a72d-421f-a514-8f633a876aea","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.641674994Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 1f7f81c6-a72d-421f-a514-8f633a876aea has session id: 5f366355-35aa-4591-a6de-88df03551d17","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.64170841Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 1f7f81c6-a72d-421f-a514-8f633a876aea (4a786c80-d276-4d84-a740-e98a20ae047b) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.641735336Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.641754881Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 1f7f81c6-a72d-421f-a514-8f633a876aea (4a786c80-d276-4d84-a740-e98a20ae047b) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.641775059Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.641793135Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 1f7f81c6-a72d-421f-a514-8f633a876aea (4a786c80-d276-4d84-a740-e98a20ae047b) WaitQuorum New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.641813374Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::downstairs_transition_same_wq - should panic ... ok
{"msg":"Crucible stats registered with UUID: be7ed2d4-01b8-4997-8b73-86b6eec8763a","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.642410943Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible be7ed2d4-01b8-4997-8b73-86b6eec8763a has session id: 5919cf68-5c7d-43c5-9f19-693bfc5ce6bd","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.642444357Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] be7ed2d4-01b8-4997-8b73-86b6eec8763a (5985a313-72b4-4515-b4d2-a311c9846e1d) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.642466072Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.64249719Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] be7ed2d4-01b8-4997-8b73-86b6eec8763a (5985a313-72b4-4515-b4d2-a311c9846e1d) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.642520504Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.642538814Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] be7ed2d4-01b8-4997-8b73-86b6eec8763a (5985a313-72b4-4515-b4d2-a311c9846e1d) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.642559098Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.642576832Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] be7ed2d4-01b8-4997-8b73-86b6eec8763a (5985a313-72b4-4515-b4d2-a311c9846e1d) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.642596804Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.64261448Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] be7ed2d4-01b8-4997-8b73-86b6eec8763a (5985a313-72b4-4515-b4d2-a311c9846e1d) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.642634764Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.642652696Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] be7ed2d4-01b8-4997-8b73-86b6eec8763a (5985a313-72b4-4515-b4d2-a311c9846e1d) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.642678611Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.642697936Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] be7ed2d4-01b8-4997-8b73-86b6eec8763a (5985a313-72b4-4515-b4d2-a311c9846e1d) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.642718101Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.642736042Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] be7ed2d4-01b8-4997-8b73-86b6eec8763a (5985a313-72b4-4515-b4d2-a311c9846e1d) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.642756369Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.642774409Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] be7ed2d4-01b8-4997-8b73-86b6eec8763a (5985a313-72b4-4515-b4d2-a311c9846e1d) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.642794527Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.642812432Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"be7ed2d4-01b8-4997-8b73-86b6eec8763a is now active with session: 5985a313-72b4-4515-b4d2-a311c9846e1d","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.64283229Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] be7ed2d4-01b8-4997-8b73-86b6eec8763a (5985a313-72b4-4515-b4d2-a311c9846e1d) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.642852573Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.642870413Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::faulted_downstairs_skips_but_still_does_work ... ok
{"msg":"Crucible stats registered with UUID: a07fe4f6-3a09-461f-b7b7-4cb79508ffc4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.643430236Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible a07fe4f6-3a09-461f-b7b7-4cb79508ffc4 has session id: f6ce5f3c-9be1-44a4-b83b-54f18e22fd78","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.64346364Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] a07fe4f6-3a09-461f-b7b7-4cb79508ffc4 (731572d1-814f-4180-b2cf-7e69c0608a59) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.643496934Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.643518813Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] a07fe4f6-3a09-461f-b7b7-4cb79508ffc4 (731572d1-814f-4180-b2cf-7e69c0608a59) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.643539165Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.643557239Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] a07fe4f6-3a09-461f-b7b7-4cb79508ffc4 (731572d1-814f-4180-b2cf-7e69c0608a59) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.643583024Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.643602159Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] a07fe4f6-3a09-461f-b7b7-4cb79508ffc4 (731572d1-814f-4180-b2cf-7e69c0608a59) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.643622438Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.643640265Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] a07fe4f6-3a09-461f-b7b7-4cb79508ffc4 (731572d1-814f-4180-b2cf-7e69c0608a59) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.643660594Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.64367864Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] a07fe4f6-3a09-461f-b7b7-4cb79508ffc4 (731572d1-814f-4180-b2cf-7e69c0608a59) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.643698844Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.643716599Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] a07fe4f6-3a09-461f-b7b7-4cb79508ffc4 (731572d1-814f-4180-b2cf-7e69c0608a59) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.643736764Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.643754553Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] a07fe4f6-3a09-461f-b7b7-4cb79508ffc4 (731572d1-814f-4180-b2cf-7e69c0608a59) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.643774744Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.64379286Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] a07fe4f6-3a09-461f-b7b7-4cb79508ffc4 (731572d1-814f-4180-b2cf-7e69c0608a59) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.643812982Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.643830716Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"a07fe4f6-3a09-461f-b7b7-4cb79508ffc4 is now active with session: 731572d1-814f-4180-b2cf-7e69c0608a59","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.64384997Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] a07fe4f6-3a09-461f-b7b7-4cb79508ffc4 (731572d1-814f-4180-b2cf-7e69c0608a59) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.643870038Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.643888058Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::faulted_downstairs_skips_work ... ok
{"msg":"Crucible stats registered with UUID: a46292f6-8b2f-47df-8c30-c1bec8094915","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.644447484Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible a46292f6-8b2f-47df-8c30-c1bec8094915 has session id: 6031d7a3-0344-41e9-abba-455bb1501038","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.644488747Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"a46292f6-8b2f-47df-8c30-c1bec8094915 is now active with session: 379ac299-7319-4aac-9b9e-66085dbccaf1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.644515337Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] a46292f6-8b2f-47df-8c30-c1bec8094915 (379ac299-7319-4aac-9b9e-66085dbccaf1) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.644536159Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.644554614Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] a46292f6-8b2f-47df-8c30-c1bec8094915 (379ac299-7319-4aac-9b9e-66085dbccaf1) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.644574696Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.644592865Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] a46292f6-8b2f-47df-8c30-c1bec8094915 (379ac299-7319-4aac-9b9e-66085dbccaf1) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.644612879Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.644630999Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] a46292f6-8b2f-47df-8c30-c1bec8094915 (379ac299-7319-4aac-9b9e-66085dbccaf1) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.644651155Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.644669049Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] a46292f6-8b2f-47df-8c30-c1bec8094915 (379ac299-7319-4aac-9b9e-66085dbccaf1) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.644689332Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.644707455Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] a46292f6-8b2f-47df-8c30-c1bec8094915 (379ac299-7319-4aac-9b9e-66085dbccaf1) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.644727505Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.644745355Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] a46292f6-8b2f-47df-8c30-c1bec8094915 (379ac299-7319-4aac-9b9e-66085dbccaf1) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.644765395Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.644783388Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] a46292f6-8b2f-47df-8c30-c1bec8094915 (379ac299-7319-4aac-9b9e-66085dbccaf1) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.6448035Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.644827349Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] a46292f6-8b2f-47df-8c30-c1bec8094915 (379ac299-7319-4aac-9b9e-66085dbccaf1) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.644848895Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.644866722Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] a46292f6-8b2f-47df-8c30-c1bec8094915 (379ac299-7319-4aac-9b9e-66085dbccaf1) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.644886563Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.644904339Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] a46292f6-8b2f-47df-8c30-c1bec8094915 (379ac299-7319-4aac-9b9e-66085dbccaf1) Active Faulted Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.644924491Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.644942262Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::flush_io_double_skip ... ok
{"msg":"Crucible stats registered with UUID: 034b3592-4704-4869-95dd-da40923e5770","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.645500575Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 034b3592-4704-4869-95dd-da40923e5770 has session id: 002d4345-ef8c-4ec7-b825-1089e35be56a","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.645533968Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"034b3592-4704-4869-95dd-da40923e5770 is now active with session: beb18472-6f47-4f48-bd85-e0b787ecd825","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.645554821Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 034b3592-4704-4869-95dd-da40923e5770 (beb18472-6f47-4f48-bd85-e0b787ecd825) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.645575013Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.645593256Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 034b3592-4704-4869-95dd-da40923e5770 (beb18472-6f47-4f48-bd85-e0b787ecd825) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.64561347Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.645631826Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 034b3592-4704-4869-95dd-da40923e5770 (beb18472-6f47-4f48-bd85-e0b787ecd825) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.645651909Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.645669979Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 034b3592-4704-4869-95dd-da40923e5770 (beb18472-6f47-4f48-bd85-e0b787ecd825) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.645690052Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.645713744Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 034b3592-4704-4869-95dd-da40923e5770 (beb18472-6f47-4f48-bd85-e0b787ecd825) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.64573539Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.645753678Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 034b3592-4704-4869-95dd-da40923e5770 (beb18472-6f47-4f48-bd85-e0b787ecd825) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.645773625Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.645791483Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 034b3592-4704-4869-95dd-da40923e5770 (beb18472-6f47-4f48-bd85-e0b787ecd825) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.645811679Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.645829471Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 034b3592-4704-4869-95dd-da40923e5770 (beb18472-6f47-4f48-bd85-e0b787ecd825) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.645849642Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.645867687Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 034b3592-4704-4869-95dd-da40923e5770 (beb18472-6f47-4f48-bd85-e0b787ecd825) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.645887902Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.645905867Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 034b3592-4704-4869-95dd-da40923e5770 (beb18472-6f47-4f48-bd85-e0b787ecd825) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.645926Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.645944122Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] DS Reports error Err(GenericError(\"bad\")) on job 1010, DownstairsIO { ds_id: JobId(1010), guest_id: 19, work: Flush { dependencies: [], flush_number: 22, gen_number: 11, snapshot_details: None, extent_limit: None }, state: ClientData([Skipped, InProgress, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.64597209Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] Reports error GenericError(\"bad\") on job 1010, DownstairsIO { ds_id: JobId(1010), guest_id: 19, work: Flush { dependencies: [], flush_number: 22, gen_number: 11, snapshot_details: None, extent_limit: None }, state: ClientData([Skipped, InProgress, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.646001145Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
test test::up_test::flush_io_fail_and_skip ... ok
{"msg":"Crucible stats registered with UUID: 1e270f52-861c-4e7c-a049-3129084987a3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.646561008Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 1e270f52-861c-4e7c-a049-3129084987a3 has session id: 60af427f-6e62-41ee-9f96-43ccc0600e56","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.646594848Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"1e270f52-861c-4e7c-a049-3129084987a3 is now active with session: f8936756-3f9b-4036-be33-913f93aa3488","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.64661578Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 1e270f52-861c-4e7c-a049-3129084987a3 (f8936756-3f9b-4036-be33-913f93aa3488) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.646636111Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.646654377Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 1e270f52-861c-4e7c-a049-3129084987a3 (f8936756-3f9b-4036-be33-913f93aa3488) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.64667444Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.646692473Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 1e270f52-861c-4e7c-a049-3129084987a3 (f8936756-3f9b-4036-be33-913f93aa3488) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.646712537Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.646730389Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 1e270f52-861c-4e7c-a049-3129084987a3 (f8936756-3f9b-4036-be33-913f93aa3488) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.646750122Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.646767847Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 1e270f52-861c-4e7c-a049-3129084987a3 (f8936756-3f9b-4036-be33-913f93aa3488) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.646787941Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.646805699Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 1e270f52-861c-4e7c-a049-3129084987a3 (f8936756-3f9b-4036-be33-913f93aa3488) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.646825634Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.646843458Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 1e270f52-861c-4e7c-a049-3129084987a3 (f8936756-3f9b-4036-be33-913f93aa3488) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.646863588Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.646881225Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 1e270f52-861c-4e7c-a049-3129084987a3 (f8936756-3f9b-4036-be33-913f93aa3488) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.646906827Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.646926179Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 1e270f52-861c-4e7c-a049-3129084987a3 (f8936756-3f9b-4036-be33-913f93aa3488) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.646946266Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.646964022Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 1e270f52-861c-4e7c-a049-3129084987a3 (f8936756-3f9b-4036-be33-913f93aa3488) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.646983841Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.647001506Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::flush_io_single_skip ... ok
{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.647564872Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: dff94e1e-c8dd-487d-aa88-79807384cfec","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.647598185Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::not_right_block_size - should panic ... ok
{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.648286346Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: 93afa2a1-f575-4ce6-80eb-79dbe509fde7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.648319647Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::off_to_extent_bridge ... ok
{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.651092356Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: c819c307-acff-4270-a283-272e7a71e185","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.65112584Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::off_to_extent_length_almost_too_big ... ok
{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.651788399Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: 516222bf-6c67-41ec-a836-a85a4490ab11","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.651822096Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::off_to_extent_length_and_offset_almost_too_big ... ok
{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.652371565Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: 10bfea0a-096f-4f83-84be-e28e35f1ad33","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.65240498Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::off_to_extent_length_and_offset_too_big - should panic ... ok
{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.653015116Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: fdf734ce-2166-470c-b2b1-0bdfec50aa0d","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.653048462Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::off_to_extent_length_too_big - should panic ... ok
{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.653669841Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: d028b1d7-85d1-4006-b18f-ffdc6cc650b5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.65370371Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::off_to_extent_length_zero ... ok
{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.654250989Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: fd318d37-3755-49e4-9e8b-7942d7e5d089","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.654284812Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::off_to_extent_one_block ... ok
{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.655244276Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: b2f32480-30f4-4c73-8f7c-29ee92b1df26","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.655277679Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::off_to_extent_two_blocks ... ok
{"msg":"Crucible stats registered with UUID: 141c4e5d-2218-4f3c-b3e8-d77c3da711d5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.656275679Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 141c4e5d-2218-4f3c-b3e8-d77c3da711d5 has session id: 484404fb-896d-4caa-aa02-dd6788eb1bb9","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.656309206Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 141c4e5d-2218-4f3c-b3e8-d77c3da711d5 (86bb4911-337c-49d8-ab07-a5d33d70b336) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.656330829Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.656349311Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 141c4e5d-2218-4f3c-b3e8-d77c3da711d5 (86bb4911-337c-49d8-ab07-a5d33d70b336) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.656369531Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.656387575Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 141c4e5d-2218-4f3c-b3e8-d77c3da711d5 (86bb4911-337c-49d8-ab07-a5d33d70b336) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.656407496Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.656425459Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 141c4e5d-2218-4f3c-b3e8-d77c3da711d5 (86bb4911-337c-49d8-ab07-a5d33d70b336) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.656451435Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.65647547Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 141c4e5d-2218-4f3c-b3e8-d77c3da711d5 (86bb4911-337c-49d8-ab07-a5d33d70b336) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.656506807Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.656526334Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 141c4e5d-2218-4f3c-b3e8-d77c3da711d5 (86bb4911-337c-49d8-ab07-a5d33d70b336) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.656546489Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.656564481Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 141c4e5d-2218-4f3c-b3e8-d77c3da711d5 (86bb4911-337c-49d8-ab07-a5d33d70b336) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.656584601Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.656602373Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 141c4e5d-2218-4f3c-b3e8-d77c3da711d5 (86bb4911-337c-49d8-ab07-a5d33d70b336) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.656622436Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.656640441Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 141c4e5d-2218-4f3c-b3e8-d77c3da711d5 (86bb4911-337c-49d8-ab07-a5d33d70b336) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.656660506Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.656678355Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"141c4e5d-2218-4f3c-b3e8-d77c3da711d5 is now active with session: 86bb4911-337c-49d8-ab07-a5d33d70b336","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.656697719Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] DS Reports error Err(GenericError(\"bad\")) on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Write { dependencies: [], writes: [Write { eid: 0, offset: Block { value: 7, shift: 9 }, data: b\"\\x01\", block_context: BlockContext { hash: 0, encryption_context: None } }] }, state: ClientData([InProgress, InProgress, InProgress]), ack_status: AckReady, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.656730677Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[0] Reports error GenericError(\"bad\") on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Write { dependencies: [], writes: [Write { eid: 0, offset: Block { value: 7, shift: 9 }, data: b\"\\x01\", block_context: BlockContext { hash: 0, encryption_context: None } }] }, state: ClientData([InProgress, InProgress, InProgress]), ack_status: AckReady, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.656764881Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[0] client skip 1 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.656790199Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[0] changed 0 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.65681035Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[0] 141c4e5d-2218-4f3c-b3e8-d77c3da711d5 (86bb4911-337c-49d8-ab07-a5d33d70b336) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.656831612Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.656849547Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] DS Reports error Err(GenericError(\"bad\")) on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Write { dependencies: [], writes: [Write { eid: 0, offset: Block { value: 7, shift: 9 }, data: b\"\\x01\", block_context: BlockContext { hash: 0, encryption_context: None } }] }, state: ClientData([Error(GenericError(\"bad\")), InProgress, InProgress]), ack_status: AckReady, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.65688229Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] Reports error GenericError(\"bad\") on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Write { dependencies: [], writes: [Write { eid: 0, offset: Block { value: 7, shift: 9 }, data: b\"\\x01\", block_context: BlockContext { hash: 0, encryption_context: None } }] }, state: ClientData([Error(GenericError(\"bad\")), InProgress, InProgress]), ack_status: AckReady, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.656916034Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] client skip 1 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.656935611Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] changed 0 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.656954434Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] 141c4e5d-2218-4f3c-b3e8-d77c3da711d5 (86bb4911-337c-49d8-ab07-a5d33d70b336) Faulted Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.656976557Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.656994852Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::read_after_two_write_fail_is_alright ... ok
{"msg":"Crucible stats registered with UUID: b1b74c4b-9f65-4d10-b833-07ab3e622bae","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.6575549Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible b1b74c4b-9f65-4d10-b833-07ab3e622bae has session id: 52258f1b-1814-4e9f-9cd6-7c52d8d8cfa3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.657588526Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] b1b74c4b-9f65-4d10-b833-07ab3e622bae (e5df52ad-f85c-4cf3-b7ea-b0c869edb935) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.657615429Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.657635471Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] b1b74c4b-9f65-4d10-b833-07ab3e622bae (e5df52ad-f85c-4cf3-b7ea-b0c869edb935) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.657656086Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.657674188Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] b1b74c4b-9f65-4d10-b833-07ab3e622bae (e5df52ad-f85c-4cf3-b7ea-b0c869edb935) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.657694096Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.65771208Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] b1b74c4b-9f65-4d10-b833-07ab3e622bae (e5df52ad-f85c-4cf3-b7ea-b0c869edb935) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.65773214Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.657749978Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] b1b74c4b-9f65-4d10-b833-07ab3e622bae (e5df52ad-f85c-4cf3-b7ea-b0c869edb935) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.657770297Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.657788315Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] b1b74c4b-9f65-4d10-b833-07ab3e622bae (e5df52ad-f85c-4cf3-b7ea-b0c869edb935) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.65780832Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.65782632Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] b1b74c4b-9f65-4d10-b833-07ab3e622bae (e5df52ad-f85c-4cf3-b7ea-b0c869edb935) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.65784643Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.657864176Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] b1b74c4b-9f65-4d10-b833-07ab3e622bae (e5df52ad-f85c-4cf3-b7ea-b0c869edb935) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.65788427Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.65790229Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] b1b74c4b-9f65-4d10-b833-07ab3e622bae (e5df52ad-f85c-4cf3-b7ea-b0c869edb935) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.65792245Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.657944401Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"b1b74c4b-9f65-4d10-b833-07ab3e622bae is now active with session: e5df52ad-f85c-4cf3-b7ea-b0c869edb935","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.657964936Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] DS Reports error Err(GenericError(\"bad\")) on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Write { dependencies: [], writes: [Write { eid: 0, offset: Block { value: 7, shift: 9 }, data: b\"\\x01\", block_context: BlockContext { hash: 0, encryption_context: None } }] }, state: ClientData([InProgress, InProgress, InProgress]), ack_status: AckReady, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.657997486Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[0] Reports error GenericError(\"bad\") on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Write { dependencies: [], writes: [Write { eid: 0, offset: Block { value: 7, shift: 9 }, data: b\"\\x01\", block_context: BlockContext { hash: 0, encryption_context: None } }] }, state: ClientData([InProgress, InProgress, InProgress]), ack_status: AckReady, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.658031103Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[0] client skip 1 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.65805089Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[0] changed 0 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.658069998Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[0] b1b74c4b-9f65-4d10-b833-07ab3e622bae (e5df52ad-f85c-4cf3-b7ea-b0c869edb935) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.658091956Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.658110118Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::read_after_write_fail_is_alright ... ok
{"msg":"Crucible stats registered with UUID: 1f4bf574-9a2c-47ce-ad31-29fae5234bd8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.658675606Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 1f4bf574-9a2c-47ce-ad31-29fae5234bd8 has session id: 900a3c87-43dc-46e0-875a-37c9b014533a","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.658709165Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Pop front: ReconcileIO { id: ReconciliationId(0), op: ExtentClose { repair_id: ReconciliationId(0), extent_id: 1 }, state: ClientData([New, New, New]) }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.658732025Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] rep_in_progress: return ReconcileIO { id: ReconciliationId(0), op: ExtentClose { repair_id: ReconciliationId(0), extent_id: 1 }, state: ClientData([InProgress, New, New]) }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.658753968Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] rep_in_progress: return ReconcileIO { id: ReconciliationId(0), op: ExtentClose { repair_id: ReconciliationId(0), extent_id: 1 }, state: ClientData([InProgress, InProgress, New]) }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.658777608Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] rep_in_progress: return ReconcileIO { id: ReconciliationId(0), op: ExtentClose { repair_id: ReconciliationId(0), extent_id: 1 }, state: ClientData([InProgress, InProgress, InProgress]) }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.658806478Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"Pop front: ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(0), extent_id: 1 }, state: ClientData([New, New, New]) }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.658830425Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::reconcile_leave_no_job_behind - should panic ... ok
{"msg":"Crucible stats registered with UUID: 6d5bfbb2-1b79-4c05-91a2-fac7055be051","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.659423223Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 6d5bfbb2-1b79-4c05-91a2-fac7055be051 has session id: 49ab1ee1-1712-4e07-a804-9b4a80711fd0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.659456419Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 6d5bfbb2-1b79-4c05-91a2-fac7055be051 (6bc62fb0-d4eb-4f0b-b738-a380a4407d9e) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.659487904Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.659512295Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 6d5bfbb2-1b79-4c05-91a2-fac7055be051 (6bc62fb0-d4eb-4f0b-b738-a380a4407d9e) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.659532947Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.659551184Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 6d5bfbb2-1b79-4c05-91a2-fac7055be051 (6bc62fb0-d4eb-4f0b-b738-a380a4407d9e) WaitQuorum New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.659571481Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.659589314Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 6d5bfbb2-1b79-4c05-91a2-fac7055be051 (6bc62fb0-d4eb-4f0b-b738-a380a4407d9e) WaitQuorum WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.659609624Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.659627578Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.659645614Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::reconcile_not_ready ... ok
{"msg":"Crucible stats registered with UUID: ca3987a1-46c1-434b-a038-228a35da78c7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.660208321Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible ca3987a1-46c1-434b-a038-228a35da78c7 has session id: 9d1682f1-07f7-44d9-aef4-5775641b29e0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.660241597Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Full repair list: {9: ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.660267659Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"Task list: [ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: 9, client_id: ClientId(0), flush_number: 22, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: 9 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: 9, source_client_id: ClientId(0), source_repair_address: 127.0.0.1:801, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: 9 }, state: ClientData([New, New, New]) }]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.66030728Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
test test::up_test::reconcile_rc_to_message ... ok
{"msg":"Crucible stats registered with UUID: fa02e367-7d9d-4c0f-b919-184b0105c3df","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.660847757Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible fa02e367-7d9d-4c0f-b919-184b0105c3df has session id: bc6e9d4b-bccc-4d02-a336-26fe6bdec7b4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.660881258Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Full repair list: {5: ExtentFix { source: ClientId(2), dest: [ClientId(0), ClientId(1)] }}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.660902043Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"Task list: [ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: 5, client_id: ClientId(2), flush_number: 66, gen_number: 77 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: 5 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: 5, source_client_id: ClientId(2), source_repair_address: 127.0.0.1:803, dest_clients: [ClientId(0), ClientId(1)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: 5 }, state: ClientData([New, New, New]) }]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.660939824Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
test test::up_test::reconcile_rc_to_message_two ... ok
{"msg":"Crucible stats registered with UUID: 069ec54d-55f6-4ab4-be42-cf366bacc424","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.661495897Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 069ec54d-55f6-4ab4-be42-cf366bacc424 has session id: e2c0f0cf-f258-4781-a5ae-c6516dfed062","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.661530384Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Pop front: ReconcileIO { id: ReconciliationId(0), op: ExtentClose { repair_id: ReconciliationId(0), extent_id: 1 }, state: ClientData([New, New, New]) }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.66155266Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::reconcile_rep_done_too_soon - should panic ... ok
{"msg":"Crucible stats registered with UUID: 97f6234d-70f6-49a4-86ce-e1f1d7dc2972","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.662146169Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 97f6234d-70f6-49a4-86ce-e1f1d7dc2972 has session id: bd6eeb18-14e9-4c0b-8a9d-55ce4c8e907a","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.662179331Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Pop front: ReconcileIO { id: ReconciliationId(0), op: ExtentClose { repair_id: ReconciliationId(0), extent_id: 1 }, state: ClientData([New, New, New]) }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.662207215Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] rep_in_progress: return ReconcileIO { id: ReconciliationId(0), op: ExtentClose { repair_id: ReconciliationId(0), extent_id: 1 }, state: ClientData([InProgress, New, New]) }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.662230373Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[0] rep_in_progress ignore submitted job ReconcileIO { id: ReconciliationId(0), op: ExtentClose { repair_id: ReconciliationId(0), extent_id: 1 }, state: ClientData([InProgress, New, New]) }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.662254135Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
test test::up_test::reconcile_rep_in_progress_bad1 - should panic ... ok
{"msg":"Crucible stats registered with UUID: 17a9ab2e-ac06-4d72-8d3c-04f05eee61d0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.662820614Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 17a9ab2e-ac06-4d72-8d3c-04f05eee61d0 has session id: 22fd7790-4122-4577-a195-dc808d47ccd5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.662853852Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::reconcile_rep_in_progress_none ... ok
{"msg":"Crucible stats registered with UUID: ac26e8b1-20d0-4338-89db-07298ddce80f","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.663395929Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible ac26e8b1-20d0-4338-89db-07298ddce80f has session id: 0600a358-4e81-43a6-bca2-cadd735b3342","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.663429075Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Pop front: ReconcileIO { id: ReconciliationId(0), op: ExtentClose { repair_id: ReconciliationId(0), extent_id: 1 }, state: ClientData([New, New, New]) }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.663451928Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::reconcile_repair_inprogress_not_done - should panic ... ok
{"msg":"Crucible stats registered with UUID: 4caf04f9-0583-4298-940c-baa864363578","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.664059149Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 4caf04f9-0583-4298-940c-baa864363578 has session id: c128e14b-5326-488d-990d-30965f450cdd","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.664092585Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Pop front: ReconcileIO { id: ReconciliationId(0), op: ExtentClose { repair_id: ReconciliationId(0), extent_id: 1 }, state: ClientData([New, New, New]) }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.664114956Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] rep_in_progress: return ReconcileIO { id: ReconciliationId(0), op: ExtentClose { repair_id: ReconciliationId(0), extent_id: 1 }, state: ClientData([InProgress, New, New]) }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.66413674Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] rep_in_progress: return ReconcileIO { id: ReconciliationId(0), op: ExtentClose { repair_id: ReconciliationId(0), extent_id: 1 }, state: ClientData([InProgress, InProgress, New]) }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.664160304Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] rep_in_progress: return ReconcileIO { id: ReconciliationId(0), op: ExtentClose { repair_id: ReconciliationId(0), extent_id: 1 }, state: ClientData([InProgress, InProgress, InProgress]) }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.664189036Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"Pop front: ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(0), extent_id: 1 }, state: ClientData([New, New, New]) }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.664212386Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] rep_in_progress: return ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(0), extent_id: 1 }, state: ClientData([InProgress, New, New]) }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.664233826Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] rep_in_progress: return ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(0), extent_id: 1 }, state: ClientData([InProgress, InProgress, New]) }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.664256384Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] rep_in_progress: return ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(0), extent_id: 1 }, state: ClientData([InProgress, InProgress, InProgress]) }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.664279151Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
test test::up_test::reconcile_repair_workflow_1 ... ok
{"msg":"Crucible stats registered with UUID: 67a60535-788d-4500-b81d-d0a82822f89a","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.664805126Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 67a60535-788d-4500-b81d-d0a82822f89a has session id: 12f33e27-3743-4af6-8c96-d9b9bee42ab8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.66483839Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Pop front: ReconcileIO { id: ReconciliationId(0), op: ExtentClose { repair_id: ReconciliationId(0), extent_id: 1 }, state: ClientData([New, New, New]) }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.664861106Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] rep_in_progress: return ReconcileIO { id: ReconciliationId(0), op: ExtentClose { repair_id: ReconciliationId(0), extent_id: 1 }, state: ClientData([InProgress, New, New]) }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.664882894Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] rep_in_progress: return ReconcileIO { id: ReconciliationId(0), op: ExtentClose { repair_id: ReconciliationId(0), extent_id: 1 }, state: ClientData([InProgress, Skipped, InProgress]) }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.664906216Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
test test::up_test::reconcile_repair_workflow_2 ... ok
{"msg":"Crucible stats registered with UUID: 1d64b798-4118-44cd-999f-1b34b470c1c5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.665449893Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 1d64b798-4118-44cd-999f-1b34b470c1c5 has session id: f89ef70f-6d50-41cc-81ba-90cdf0b6f139","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.665493494Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Mark 0 as FAILED REPAIR","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.665515921Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"Mark 2 as FAILED REPAIR","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.665535309Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"Clear out existing repair work queue","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.665560419Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
test test::up_test::reconcile_repair_workflow_not_repair ... ok
{"msg":"Crucible stats registered with UUID: a9b6d4e4-1a36-401a-a0ca-275c6c7578b5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.666143175Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible a9b6d4e4-1a36-401a-a0ca-275c6c7578b5 has session id: 9d51c81d-31db-4664-a3b9-830d11d61906","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.666176344Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Pop front: ReconcileIO { id: ReconciliationId(0), op: ExtentClose { repair_id: ReconciliationId(0), extent_id: 1 }, state: ClientData([New, New, New]) }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.666198722Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] rep_in_progress: return ReconcileIO { id: ReconciliationId(0), op: ExtentClose { repair_id: ReconciliationId(0), extent_id: 1 }, state: ClientData([InProgress, New, New]) }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.666220263Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] rep_in_progress: return ReconcileIO { id: ReconciliationId(0), op: ExtentClose { repair_id: ReconciliationId(0), extent_id: 1 }, state: ClientData([InProgress, InProgress, New]) }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.666243703Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] rep_in_progress: return ReconcileIO { id: ReconciliationId(0), op: ExtentClose { repair_id: ReconciliationId(0), extent_id: 1 }, state: ClientData([InProgress, InProgress, InProgress]) }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.666266462Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"Mark 0 as FAILED REPAIR","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.666285095Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"Mark 2 as FAILED REPAIR","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.666303827Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"Clear out existing repair work queue","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.666322891Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
test test::up_test::reconcile_repair_workflow_not_repair_later ... ok
{"msg":"Crucible stats registered with UUID: 02990135-0f1c-4862-aa7b-18aae9f44298","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.666842639Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 02990135-0f1c-4862-aa7b-18aae9f44298 has session id: 4e58b218-acbe-4114-82fb-ff057449c776","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.666876588Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Pop front: ReconcileIO { id: ReconciliationId(0), op: ExtentClose { repair_id: ReconciliationId(0), extent_id: 1 }, state: ClientData([New, New, New]) }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.666898938Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] rep_in_progress: return ReconcileIO { id: ReconciliationId(0), op: ExtentClose { repair_id: ReconciliationId(0), extent_id: 1 }, state: ClientData([InProgress, New, New]) }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.666920342Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] rep_in_progress: return ReconcileIO { id: ReconciliationId(0), op: ExtentClose { repair_id: ReconciliationId(0), extent_id: 1 }, state: ClientData([InProgress, InProgress, New]) }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.666943603Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] rep_in_progress: return ReconcileIO { id: ReconciliationId(0), op: ExtentClose { repair_id: ReconciliationId(0), extent_id: 1 }, state: ClientData([InProgress, InProgress, InProgress]) }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.666972421Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
test test::up_test::reconcile_repair_workflow_repair_later ... ok
{"msg":"Crucible stats registered with UUID: 9dac809d-cf03-4c57-baf9-7a5d9a4d890b","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.667518634Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 9dac809d-cf03-4c57-baf9-7a5d9a4d890b has session id: 82751d91-e856-4a21-91e7-8fe9f9112f2f","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.667552182Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Pop front: ReconcileIO { id: ReconciliationId(0), op: ExtentClose { repair_id: ReconciliationId(0), extent_id: 1 }, state: ClientData([New, New, New]) }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.667574563Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::reconcile_repair_workflow_too_soon - should panic ... ok
{"msg":"Crucible stats registered with UUID: ad352343-27a7-4725-9872-c157f535c275","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.668169997Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible ad352343-27a7-4725-9872-c157f535c275 has session id: d746cca1-bce1-4e32-93a9-4b9d5c136adb","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.668203082Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"ad352343-27a7-4725-9872-c157f535c275 is now active with session: 78993387-0c5c-4fe6-88b0-16fc8f959e40","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.668223767Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[rc] leave job 1000 on the queue when removing 1001 WorkCounts { active: 2, error: 0, skipped: 0, done: 1 }","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:11.668250442Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
test test::up_test::retire_dont_retire_everything ... ok
test test::up_test::send_io_live_repair_read ... ok
test test::up_test::send_io_live_repair_unwritten_write ... ok
test test::up_test::send_io_live_repair_write ... ok
{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.669583747Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: ad07efa3-3de2-4189-bc6a-b09c04a495e7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.669617011Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"00000000-0000-0000-0000-000000000000 is now active with session: ceba3bc0-7dc4-44df-bc80-c7cf0880210b","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.669637713Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::test_deps_all_writes_depend_on_flushes ... ok
{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.67047904Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: 618c5b6b-d33b-4afe-ad82-cf77a7a83de6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.670516346Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"00000000-0000-0000-0000-000000000000 is now active with session: 3531a35f-222b-44c8-a877-e791425e6b83","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.670538047Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::test_deps_big_read_depends_on_little_writes ... ok
{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.671249336Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: 31dc3ed7-1ad2-4a98-952c-1f8161910277","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.671282649Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"00000000-0000-0000-0000-000000000000 is now active with session: 94ef0595-d784-48ff-b4ed-fc1899b9063f","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.671303265Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::test_deps_big_write_depends_on_little_writes ... ok
{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.672029087Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: b388ce47-5823-4038-9d95-896435a0b663","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.672062297Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"00000000-0000-0000-0000-000000000000 is now active with session: c948856d-1826-4b6f-81ae-c01afbc7fc00","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.6720828Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::test_deps_depend_on_acked_work ... ok
{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.672763069Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: e6f69a31-a233-4bbe-a2ab-4d80a28a2311","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.67279676Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"00000000-0000-0000-0000-000000000000 is now active with session: d3721654-98f1-46a0-a6a8-eb39aadad20b","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.672817514Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::test_deps_flushes_depend_on_flushes ... ok
{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.673449759Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: f23d638d-669b-4254-9ccd-38027db9ed82","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.673493733Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"00000000-0000-0000-0000-000000000000 is now active with session: ef05052f-c079-4ccc-bc74-45387586e065","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.673517654Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::test_deps_flushes_depend_on_flushes_and_all_writes ... ok
{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.674353647Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: 0bff49f4-5635-420f-b796-2dfcf490acd9","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.674386837Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"00000000-0000-0000-0000-000000000000 is now active with session: 5d166cba-7c96-43c0-a592-3798947445d8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.674407498Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::test_deps_little_writes_depend_on_big_write ... ok
{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.675130982Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: 807ee680-d4fb-4886-8b4e-bfd2c3fb8ab2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.675164413Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"00000000-0000-0000-0000-000000000000 is now active with session: 0cba9a20-f1a7-429e-9250-230ecb23c8f5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.675185293Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::test_deps_little_writes_depend_on_big_write_chain ... ok
{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.676032106Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: dc2a0aee-f660-43a8-ae9e-b72ab5d152fb","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.676065416Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"00000000-0000-0000-0000-000000000000 is now active with session: aadd3d6a-038e-4c9b-a2a2-74a82ee4c9e8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.676086155Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::test_deps_multi_extent_batman ... ok
{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.676791975Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: 122e8b2e-3906-4dac-8d1f-c7464fc31067","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.676825145Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"00000000-0000-0000-0000-000000000000 is now active with session: 7d00589e-546c-4829-a81e-10828ef398cf","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.676845743Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::test_deps_multi_extent_there_and_back_again ... ok
{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.677651217Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: 26560027-2102-4c25-b236-ba8526b40216","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.677684348Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"00000000-0000-0000-0000-000000000000 is now active with session: 1a594dd3-ee0f-4008-bb83-e7d1e631db83","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.677705146Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::test_deps_multi_extent_write ... ok
{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.678411729Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: 535cbc81-fc9b-4386-aa56-07ab9dc76033","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.678444722Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"00000000-0000-0000-0000-000000000000 is now active with session: 1f47685e-52c8-4c03-9915-fcf8c7dbe1c2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.678466508Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::test_deps_multiple_reads_depend_on_write ... ok
{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.679167128Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: 8f651a7a-cf1f-4936-bc29-f132695fbc42","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.679206025Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"00000000-0000-0000-0000-000000000000 is now active with session: 6fabe596-c1dc-4a19-b4db-97108722e481","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.679227772Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::test_deps_read_depends_on_flush ... ok
{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.679890134Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: 32236327-9fd5-44b2-9553-13aa1be1d568","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.679923568Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"00000000-0000-0000-0000-000000000000 is now active with session: 8842029d-0daf-43ea-abd6-cafe80f8071a","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.679944117Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::test_deps_read_depends_on_write ... ok
{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.680576556Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: b2a26172-2a8e-462f-88bd-d44f36402cc3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.68060997Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"00000000-0000-0000-0000-000000000000 is now active with session: d5162543-b365-4a8a-83a1-a4a2614001f0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.680630739Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::test_deps_read_no_depend_on_read ... ok
{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.681260788Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: e0e730af-125e-4891-bcce-67baa3f14a83","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.681293862Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"00000000-0000-0000-0000-000000000000 is now active with session: a4a0763d-5745-40c2-9fea-748308d5166b","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.681314689Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::test_deps_read_write_batman ... ok
{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.682004867Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: e14a6fd5-813d-41c9-a19f-edd31c25b3a9","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.682037748Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"00000000-0000-0000-0000-000000000000 is now active with session: dc2df2a1-ca4d-41d5-8862-d9ed27e1a066","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.682058391Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::test_deps_read_write_ladder_1 ... ok
{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.682789141Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: c463f55f-b32e-4ced-b75b-0d61ec26249b","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.682827909Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"00000000-0000-0000-0000-000000000000 is now active with session: c8e7706b-8929-4422-9a66-1b3c76a30e88","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.682849954Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::test_deps_read_write_ladder_2 ... ok
{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.683556599Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: 362907f2-5304-4253-81c6-faf16908d778","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.683589944Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"00000000-0000-0000-0000-000000000000 is now active with session: 6248b885-469f-46ea-9f4e-d04639c16a5b","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.683610676Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::test_deps_read_write_ladder_3 ... ok
{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.684339149Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: 0ab7ae9e-fa1a-4dd5-8eee-e23a0b9c2538","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.684372234Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"00000000-0000-0000-0000-000000000000 is now active with session: 881418b6-b76c-41c8-afad-38189754b350","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.684392809Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::test_deps_write_unwrittens_depend_on_read ... ok
{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.685058699Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: 44a942e4-1b72-40e3-8c58-dfcb639b2a56","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.685092012Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"00000000-0000-0000-0000-000000000000 is now active with session: 485f8d12-ff26-4977-9b18-4a354c1cfe02","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.68511258Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::test_deps_writes_depend_on_overlapping_writes ... ok
{"msg":"Now move the NoOp job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.685681832Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now ACK the NoOp job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.685733083Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Finally, move the ReOpen job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.685767902Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now ACK the Reopen job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.685799661Z","hostname":"{ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
"msg{":""msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000"Extent 0 close id:1002 Failed: Error: bad","v,"":v"0:,"0name":,""cruciblename"":,""crucible"level":,"50level":30,"time":"2023-09-22T23:08:11.685841971Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{,""time":msg"":"2023-09-22T23:08:11.685844035Z",RE:0 Wait for result from reopen command 1003:4"","hostname"v":":0,"name":ip-10-150-1-74.us-west-2.compute.internal"","crucible"pid,"":level"4291:30}
,"time":"2023-09-22T23:08:11.685883654Z","hostname{":"ip-10-150-1-74.us-west-2.compute.internal",""pid":msg4291":"}
Crucible 00000000-0000-0000-0000-000000000000 has session id: c2ce6ac0-24a2-47bc-9b90-3a3ca5271ef3","v":0{,"name":""crucible"msg":,""level":30Extent 0 close id:1003 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.685920643Z",",time":""hostname":"2023-09-22T23:08:11.685926926Z","ip-10-150-1-74.us-west-2.compute.internal"hostname",:""pid":4291ip-10-150-1-74.us-west-2.compute.internal","pid"}:
4291}
{"{msg":""msg":"RE:0 Bailing with error"00000000-0000-0000-0000-000000000000 is now active with session: e5b04902-9ab3-41e0-9229-d2240b1e580f,""v",:"0v",:"0name",:""name":"crucible"crucible",","level"level"::4030,"time":"2023-09-22T23:08:11.685984773Z",",time"":"hostname":"2023-09-22T23:08:11.68598527Z","ip-10-150-1-74.us-west-2.compute.internal"hostname",":pid"":4291ip-10-150-1-74.us-west-2.compute.internal"},
"pid":4291}
test test::up_test::test_deps_writes_depend_on_overlapping_writes_and_flushes ... ok
{"msg":"Crucible stats registered with UUID: c2759bf2-aaf5-4d2a-82e7-2795ddf1ec4e","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.686461674Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible c2759bf2-aaf5-4d2a-82e7-2795ddf1ec4e has session id: f0f8df2c-4bdf-4ea7-a801-fb139630ee82","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.68651304Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] c2759bf2-aaf5-4d2a-82e7-2795ddf1ec4e (c90c1fcd-3603-46fd-b4f7-87867e35920a) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.686555286Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.686593675Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] c2759bf2-aaf5-4d2a-82e7-2795ddf1ec4e (c90c1fcd-3603-46fd-b4f7-87867e35920a) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.686630503Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.68666381Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] c2759bf2-aaf5-4d2a-82e7-2795ddf1ec4e (c90c1fcd-3603-46fd-b4f7-87867e35920a) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.686703999Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"{time":"2023-09-22T23:08:11.686742799Z","hostname":""msgip-10-150-1-74.us-west-2.compute.internal"",:""pid":4291}
Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v{":0,""name"msg"::""crucible","level":30[1] c2759bf2-aaf5-4d2a-82e7-2795ddf1ec4e (c90c1fcd-3603-46fd-b4f7-87867e35920a) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":","2023-09-22T23:08:11.686789248Z"time":,""hostname":"2023-09-22T23:08:11.686782543Z","ip-10-150-1-74.us-west-2.compute.internal"hostname",:""pid":4291ip-10-150-1-74.us-west-2.compute.internal"},
"pid":4291{}
"msg":"[1] Transition from New to WaitActive"{,"v":0,""name"msg"::""crucible","level":30Crucible 00000000-0000-0000-0000-000000000000 has session id: dd2e573b-5f55-4d37-ac90-b4479734e66a","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.68684055Z","hostname":","time"ip-10-150-1-74.us-west-2.compute.internal:"","pid":42912023-09-22T23:08:11.686848147Z",}"
hostname":"{ip-10-150-1-74.us-west-2.compute.internal","pid":4291"msg"}:
"{"msg":"[1] c2759bf2-aaf5-4d2a-82e7-2795ddf1ec4e (c90c1fcd-3603-46fd-b4f7-87867e35920a) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":00000000-0000-0000-0000-000000000000 is now active with session: a9e06fa5-3978-4b13-a30e-436871db1b0a"","cruciblev"":0,,""level"name"::"30crucible","level":30,"time":"2023-09-22T23:08:11.686900134Z",,""time"hostname":":"2023-09-22T23:08:11.686897953Z",ip-10-150-1-74.us-west-2.compute.internal"","hostname"pid":":4291}
ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.68695809Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] c2759bf2-aaf5-4d2a-82e7-2795ddf1ec4e (c90c1fcd-3603-46fd-b4f7-87867e35920a) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.687002748Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.687041365Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] c2759bf2-aaf5-4d2a-82e7-2795ddf1ec4e (c90c1fcd-3603-46fd-b4f7-87867e35920a) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.687082477Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.687117692Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] c2759bf2-aaf5-4d2a-82e7-2795ddf1ec4e (c90c1fcd-3603-46fd-b4f7-87867e35920a) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.68714217Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.687169717Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] c2759bf2-aaf5-4d2a-82e7-2795ddf1ec4e (c90c1fcd-3603-46fd-b4f7-87867e35920a) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.687207907Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.687245436Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"c2759bf2-aaf5-4d2a-82e7-2795ddf1ec4e is now active with session: c90c1fcd-3603-46fd-b4f7-87867e35920a","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.687285955Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] c2759bf2-aaf5-4d2a-82e7-2795ddf1ec4e (c90c1fcd-3603-46fd-b4f7-87867e35920a) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.687325242Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::test_deps_writes_depend_on_overlapping_writes_chain ... {ok"
msg":"[0] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.687365644Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] c2759bf2-aaf5-4d2a-82e7-2795ddf1ec4e (c90c1fcd-3603-46fd-b4f7-87867e35920a) Faulted Active Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.687388664Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.687406891Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] c2759bf2-aaf5-4d2a-82e7-2795ddf1ec4e (c90c1fcd-3603-46fd-b4f7-87867e35920a) LiveRepairReady Active Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.687427243Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.687445451Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for Close + ReOpen jobs","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.687469891Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.687489765Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 close id:1000 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.68750775Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from close command 1000:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.687526245Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.687679446Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: 3d043886-bbe0-4ac8-a93a-9043cf7c1b25","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.687708153Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"00000000-0000-0000-0000-000000000000 is now active with session: 60ccb8e7-7dbb-42d8-bef7-a0f8f8a16283","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.687728916Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::test_deps_writes_depend_on_read ... ok
test test::up_test::test_flush_does_not_consume_bw ... ok
test test::up_test::test_flush_does_not_consume_iops ... ok
test test::up_test::test_impossible_io ... ok
test test::up_test::test_iop_and_bw_limit ... ok
test test::up_test::test_iospan ... ok
test test::up_test::test_iospan_buffer_read_write ... ok
test test::up_test::test_no_iop_limit ... ok
{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.738146146Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: 512cafe5-b904-40b2-acda-3b38200b25bd","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.738197764Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"00000000-0000-0000-0000-000000000000 is now active with session: 4532d3f5-d6bd-4758-b8e8-875e64f31f49","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.738219575Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::test_read_flush_write_hash_mismatch ... ok
test test::up_test::test_set_bw_limit ... ok
test test::up_test::test_set_iop_limit ... ok
test test::up_test::test_upstairs_encryption_context_ok ... ok
test test::up_test::test_upstairs_encryption_context_wrong_nonce ... ok
test test::up_test::test_upstairs_encryption_context_wrong_tag ... ok
test test::up_test::test_upstairs_validate_encrypted_read_response ... ok
test test::up_test::test_upstairs_validate_encrypted_read_response_blank_block ... ok
test test::up_test::test_upstairs_validate_encrypted_read_response_multiple_contexts ... ok
test test::up_test::test_upstairs_validate_unencrypted_read_response ... ok
test test::up_test::test_upstairs_validate_unencrypted_read_response_blank_block ... ok
test test::up_test::test_upstairs_validate_unencrypted_read_response_multiple_contexts ... ok
test test::up_test::test_upstairs_validate_unencrypted_read_response_multiple_hashes ... ok
{"msg":"Crucible stats registered with UUID: aff9dff6-ec62-4129-9120-a7e33dd182ad","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.747379142Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible aff9dff6-ec62-4129-9120-a7e33dd182ad has session id: 6d3aa3c0-7944-4585-accc-c3aa0d2e66fb","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.747415066Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] aff9dff6-ec62-4129-9120-a7e33dd182ad (21a6bd28-e2c9-430e-b5fc-24e458ce8b1e) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.747445272Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.747470486Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] aff9dff6-ec62-4129-9120-a7e33dd182ad (21a6bd28-e2c9-430e-b5fc-24e458ce8b1e) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.74750241Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.7475226Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] aff9dff6-ec62-4129-9120-a7e33dd182ad (21a6bd28-e2c9-430e-b5fc-24e458ce8b1e) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.747543184Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.747561218Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] aff9dff6-ec62-4129-9120-a7e33dd182ad (21a6bd28-e2c9-430e-b5fc-24e458ce8b1e) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.747581321Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.747599281Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] aff9dff6-ec62-4129-9120-a7e33dd182ad (21a6bd28-e2c9-430e-b5fc-24e458ce8b1e) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.74761945Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.747637503Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] aff9dff6-ec62-4129-9120-a7e33dd182ad (21a6bd28-e2c9-430e-b5fc-24e458ce8b1e) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.747657503Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.7476754Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] aff9dff6-ec62-4129-9120-a7e33dd182ad (21a6bd28-e2c9-430e-b5fc-24e458ce8b1e) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.747695539Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.747713243Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] aff9dff6-ec62-4129-9120-a7e33dd182ad (21a6bd28-e2c9-430e-b5fc-24e458ce8b1e) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.747733267Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.747751181Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] aff9dff6-ec62-4129-9120-a7e33dd182ad (21a6bd28-e2c9-430e-b5fc-24e458ce8b1e) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.747771191Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.747794323Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"aff9dff6-ec62-4129-9120-a7e33dd182ad is now active with session: 21a6bd28-e2c9-430e-b5fc-24e458ce8b1e","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.747814862Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] aff9dff6-ec62-4129-9120-a7e33dd182ad (21a6bd28-e2c9-430e-b5fc-24e458ce8b1e) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.747834938Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.747852714Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] aff9dff6-ec62-4129-9120-a7e33dd182ad (21a6bd28-e2c9-430e-b5fc-24e458ce8b1e) Faulted Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.74787266Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.747890473Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] aff9dff6-ec62-4129-9120-a7e33dd182ad (21a6bd28-e2c9-430e-b5fc-24e458ce8b1e) Faulted Faulted Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.747910541Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.747928297Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"job 1000 skipped on all downstairs","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:11.747946051Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"Enqueue job 1000 goes straight to AckReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.747965641Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
test test::up_test::three_faulted_enqueue_will_handle_flush ... ok
{"msg":"Crucible stats registered with UUID: c48fedac-9929-4766-93c2-f75a1bb45a29","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.748533247Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible c48fedac-9929-4766-93c2-f75a1bb45a29 has session id: 8a456627-c0ec-44bd-b09a-b433e5213d9f","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.748567642Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] c48fedac-9929-4766-93c2-f75a1bb45a29 (a35ce5f9-6cf5-4fe6-a11d-68cc05477a1a) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.748589672Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.748608197Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] c48fedac-9929-4766-93c2-f75a1bb45a29 (a35ce5f9-6cf5-4fe6-a11d-68cc05477a1a) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.748628699Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.748646945Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] c48fedac-9929-4766-93c2-f75a1bb45a29 (a35ce5f9-6cf5-4fe6-a11d-68cc05477a1a) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.748667107Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.748690796Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] c48fedac-9929-4766-93c2-f75a1bb45a29 (a35ce5f9-6cf5-4fe6-a11d-68cc05477a1a) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.7487122Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.748730279Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] c48fedac-9929-4766-93c2-f75a1bb45a29 (a35ce5f9-6cf5-4fe6-a11d-68cc05477a1a) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.748750488Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.748768751Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] c48fedac-9929-4766-93c2-f75a1bb45a29 (a35ce5f9-6cf5-4fe6-a11d-68cc05477a1a) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.748788889Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.74880676Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] c48fedac-9929-4766-93c2-f75a1bb45a29 (a35ce5f9-6cf5-4fe6-a11d-68cc05477a1a) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.748827115Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.748844869Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] c48fedac-9929-4766-93c2-f75a1bb45a29 (a35ce5f9-6cf5-4fe6-a11d-68cc05477a1a) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.748864985Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.748883119Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] c48fedac-9929-4766-93c2-f75a1bb45a29 (a35ce5f9-6cf5-4fe6-a11d-68cc05477a1a) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.748903332Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.748921347Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"c48fedac-9929-4766-93c2-f75a1bb45a29 is now active with session: a35ce5f9-6cf5-4fe6-a11d-68cc05477a1a","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.748940985Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] c48fedac-9929-4766-93c2-f75a1bb45a29 (a35ce5f9-6cf5-4fe6-a11d-68cc05477a1a) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.748961222Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.748978994Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] c48fedac-9929-4766-93c2-f75a1bb45a29 (a35ce5f9-6cf5-4fe6-a11d-68cc05477a1a) Faulted Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.748999154Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.749017024Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] c48fedac-9929-4766-93c2-f75a1bb45a29 (a35ce5f9-6cf5-4fe6-a11d-68cc05477a1a) Faulted Faulted Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.749041611Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.749060347Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"job 1000 skipped on all downstairs","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:11.749078278Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"Enqueue job 1000 goes straight to AckReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.749097832Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"job 1001 skipped on all downstairs","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:11.749116524Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"Enqueue job 1001 goes straight to AckReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.749135529Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"job 1002 skipped on all downstairs","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:11.749154361Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"Enqueue job 1002 goes straight to AckReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.74917337Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
test test::up_test::three_faulted_enqueue_will_handle_many_ios ... ok
{"msg":"Crucible stats registered with UUID: cc0b1b6d-0d14-49fe-8849-9ff7b600a9b8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.749744316Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible cc0b1b6d-0d14-49fe-8849-9ff7b600a9b8 has session id: a3ae0350-4779-47cc-aadc-1326e13058a7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.749778692Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] cc0b1b6d-0d14-49fe-8849-9ff7b600a9b8 (47f7b332-dccf-4cd7-a278-f9f25a6aa948) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.749800636Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.749819293Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] cc0b1b6d-0d14-49fe-8849-9ff7b600a9b8 (47f7b332-dccf-4cd7-a278-f9f25a6aa948) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.749839746Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.749857866Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] cc0b1b6d-0d14-49fe-8849-9ff7b600a9b8 (47f7b332-dccf-4cd7-a278-f9f25a6aa948) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.749877929Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.74989584Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] cc0b1b6d-0d14-49fe-8849-9ff7b600a9b8 (47f7b332-dccf-4cd7-a278-f9f25a6aa948) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.749915934Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.749933763Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] cc0b1b6d-0d14-49fe-8849-9ff7b600a9b8 (47f7b332-dccf-4cd7-a278-f9f25a6aa948) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.749959988Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.749979639Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] cc0b1b6d-0d14-49fe-8849-9ff7b600a9b8 (47f7b332-dccf-4cd7-a278-f9f25a6aa948) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.749999717Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.750017532Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] cc0b1b6d-0d14-49fe-8849-9ff7b600a9b8 (47f7b332-dccf-4cd7-a278-f9f25a6aa948) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.750037721Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.750055516Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] cc0b1b6d-0d14-49fe-8849-9ff7b600a9b8 (47f7b332-dccf-4cd7-a278-f9f25a6aa948) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.750075805Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.750093949Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] cc0b1b6d-0d14-49fe-8849-9ff7b600a9b8 (47f7b332-dccf-4cd7-a278-f9f25a6aa948) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.75011417Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.750132232Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"cc0b1b6d-0d14-49fe-8849-9ff7b600a9b8 is now active with session: 47f7b332-dccf-4cd7-a278-f9f25a6aa948","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.750151933Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] cc0b1b6d-0d14-49fe-8849-9ff7b600a9b8 (47f7b332-dccf-4cd7-a278-f9f25a6aa948) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.750171987Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.750189789Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] cc0b1b6d-0d14-49fe-8849-9ff7b600a9b8 (47f7b332-dccf-4cd7-a278-f9f25a6aa948) Faulted Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.750209747Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.750227472Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] cc0b1b6d-0d14-49fe-8849-9ff7b600a9b8 (47f7b332-dccf-4cd7-a278-f9f25a6aa948) Faulted Faulted Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.750247485Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.750265374Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"job 1000 skipped on all downstairs","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:11.750287226Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"Enqueue job 1000 goes straight to AckReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.7503074Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
test test::up_test::three_faulted_enqueue_will_handle_read ... ok
{"msg":"Crucible stats registered with UUID: 140318df-e512-46f0-b064-327b8930ef8f","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.750832706Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 140318df-e512-46f0-b064-327b8930ef8f has session id: e343a590-4abc-41bd-bf7e-2a97e8a090f3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.75086663Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 140318df-e512-46f0-b064-327b8930ef8f (34028cb9-05d2-4203-a645-177faee3d76c) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.750888849Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.750907284Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 140318df-e512-46f0-b064-327b8930ef8f (34028cb9-05d2-4203-a645-177faee3d76c) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.75092775Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.750946003Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 140318df-e512-46f0-b064-327b8930ef8f (34028cb9-05d2-4203-a645-177faee3d76c) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.750966046Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.750984035Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 140318df-e512-46f0-b064-327b8930ef8f (34028cb9-05d2-4203-a645-177faee3d76c) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.751004193Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.751022146Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 140318df-e512-46f0-b064-327b8930ef8f (34028cb9-05d2-4203-a645-177faee3d76c) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.751042357Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.751060392Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 140318df-e512-46f0-b064-327b8930ef8f (34028cb9-05d2-4203-a645-177faee3d76c) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.751080388Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.751098433Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 140318df-e512-46f0-b064-327b8930ef8f (34028cb9-05d2-4203-a645-177faee3d76c) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.751118539Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.751142058Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 140318df-e512-46f0-b064-327b8930ef8f (34028cb9-05d2-4203-a645-177faee3d76c) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.751163689Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.751181697Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 140318df-e512-46f0-b064-327b8930ef8f (34028cb9-05d2-4203-a645-177faee3d76c) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.751201957Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.751219933Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"140318df-e512-46f0-b064-327b8930ef8f is now active with session: 34028cb9-05d2-4203-a645-177faee3d76c","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.751239328Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 140318df-e512-46f0-b064-327b8930ef8f (34028cb9-05d2-4203-a645-177faee3d76c) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.751259485Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.751277407Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 140318df-e512-46f0-b064-327b8930ef8f (34028cb9-05d2-4203-a645-177faee3d76c) Faulted Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.751297508Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.751315315Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 140318df-e512-46f0-b064-327b8930ef8f (34028cb9-05d2-4203-a645-177faee3d76c) Faulted Faulted Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.751335427Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.751353232Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"job 1000 skipped on all downstairs","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:11.751371056Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"Enqueue job 1000 goes straight to AckReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.751390568Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
test test::up_test::three_faulted_enqueue_will_handle_write ... ok
{"msg":"Crucible stats registered with UUID: 6d77b913-92ca-428b-a400-ba4b35c9827c","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.751914671Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 6d77b913-92ca-428b-a400-ba4b35c9827c has session id: 74dcbbc2-b4c0-4251-8701-6c7657e2728f","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.751948429Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 6d77b913-92ca-428b-a400-ba4b35c9827c (79976138-c5dd-4544-aaf9-3f7aa290bf7f) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.751969927Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.751988235Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 6d77b913-92ca-428b-a400-ba4b35c9827c (79976138-c5dd-4544-aaf9-3f7aa290bf7f) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.752014404Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.752033775Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 6d77b913-92ca-428b-a400-ba4b35c9827c (79976138-c5dd-4544-aaf9-3f7aa290bf7f) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.752053921Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.752071762Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 6d77b913-92ca-428b-a400-ba4b35c9827c (79976138-c5dd-4544-aaf9-3f7aa290bf7f) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.752091649Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.752109378Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 6d77b913-92ca-428b-a400-ba4b35c9827c (79976138-c5dd-4544-aaf9-3f7aa290bf7f) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.752129588Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.752147614Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 6d77b913-92ca-428b-a400-ba4b35c9827c (79976138-c5dd-4544-aaf9-3f7aa290bf7f) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.752167645Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.752185581Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 6d77b913-92ca-428b-a400-ba4b35c9827c (79976138-c5dd-4544-aaf9-3f7aa290bf7f) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.752205605Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.752223389Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 6d77b913-92ca-428b-a400-ba4b35c9827c (79976138-c5dd-4544-aaf9-3f7aa290bf7f) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.752243591Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.752261606Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 6d77b913-92ca-428b-a400-ba4b35c9827c (79976138-c5dd-4544-aaf9-3f7aa290bf7f) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.752281716Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.752299603Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"6d77b913-92ca-428b-a400-ba4b35c9827c is now active with session: 79976138-c5dd-4544-aaf9-3f7aa290bf7f","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.752319079Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 6d77b913-92ca-428b-a400-ba4b35c9827c (79976138-c5dd-4544-aaf9-3f7aa290bf7f) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.752343088Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.752361734Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 6d77b913-92ca-428b-a400-ba4b35c9827c (79976138-c5dd-4544-aaf9-3f7aa290bf7f) Faulted Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.7523818Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.752399728Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 6d77b913-92ca-428b-a400-ba4b35c9827c (79976138-c5dd-4544-aaf9-3f7aa290bf7f) Faulted Faulted Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.75241966Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.752437569Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"job 1000 skipped on all downstairs","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:11.75245521Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"Enqueue job 1000 goes straight to AckReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.752486606Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"job 1001 skipped on all downstairs","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:11.752510492Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"Enqueue job 1001 goes straight to AckReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.752529888Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"job 1002 skipped on all downstairs","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:11.752548637Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"Enqueue job 1002 goes straight to AckReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.752567647Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"job 1003 skipped on all downstairs","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:11.752586538Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"Enqueue job 1003 goes straight to AckReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.752605593Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"job 1004 skipped on all downstairs","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:11.752624433Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"Enqueue job 1004 goes straight to AckReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.752643499Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"job 1005 skipped on all downstairs","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:11.75266226Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"Enqueue job 1005 goes straight to AckReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.75268104Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
test test::up_test::three_faulted_retire_skipped_some_leave_some ... ok
{"msg":"Crucible stats registered with UUID: fc14036a-f2c9-4490-9298-18ffb7e63afc","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.753204734Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible fc14036a-f2c9-4490-9298-18ffb7e63afc has session id: fb167c65-c87d-4e51-b020-2ba8c05435e0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.753244335Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] fc14036a-f2c9-4490-9298-18ffb7e63afc (a61dd7d2-dd63-4e11-888a-505db3a9a408) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.753267473Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.753286017Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] fc14036a-f2c9-4490-9298-18ffb7e63afc (a61dd7d2-dd63-4e11-888a-505db3a9a408) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.753306552Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.753324793Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] fc14036a-f2c9-4490-9298-18ffb7e63afc (a61dd7d2-dd63-4e11-888a-505db3a9a408) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.753344897Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.753362813Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] fc14036a-f2c9-4490-9298-18ffb7e63afc (a61dd7d2-dd63-4e11-888a-505db3a9a408) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.753382991Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.753400884Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] fc14036a-f2c9-4490-9298-18ffb7e63afc (a61dd7d2-dd63-4e11-888a-505db3a9a408) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.753421127Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.753439284Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] fc14036a-f2c9-4490-9298-18ffb7e63afc (a61dd7d2-dd63-4e11-888a-505db3a9a408) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.753459529Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.753490403Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] fc14036a-f2c9-4490-9298-18ffb7e63afc (a61dd7d2-dd63-4e11-888a-505db3a9a408) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.753513981Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.753532001Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] fc14036a-f2c9-4490-9298-18ffb7e63afc (a61dd7d2-dd63-4e11-888a-505db3a9a408) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.753552092Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.753570016Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] fc14036a-f2c9-4490-9298-18ffb7e63afc (a61dd7d2-dd63-4e11-888a-505db3a9a408) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.753594565Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.753613359Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"fc14036a-f2c9-4490-9298-18ffb7e63afc is now active with session: a61dd7d2-dd63-4e11-888a-505db3a9a408","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.75363285Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] fc14036a-f2c9-4490-9298-18ffb7e63afc (a61dd7d2-dd63-4e11-888a-505db3a9a408) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.753652918Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.753670811Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] fc14036a-f2c9-4490-9298-18ffb7e63afc (a61dd7d2-dd63-4e11-888a-505db3a9a408) Faulted Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.753690771Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.753708586Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::two_faulted_downstairs_can_still_read ... ok
{"msg":"Crucible stats registered with UUID: 9e235c1f-09d1-4720-b751-3c1400b7944e","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.754231729Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 9e235c1f-09d1-4720-b751-3c1400b7944e has session id: 727a6b0e-52f2-451c-9060-bb324d36d8fd","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.754265291Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"9e235c1f-09d1-4720-b751-3c1400b7944e is now active with session: 9c7f28ef-46ce-4ea9-ad55-ea907e85426e","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.754286166Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] DS Reports error Err(GenericError(\"bad\")) on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Read { dependencies: [], requests: [ReadRequest { eid: 0, offset: Block { value: 7, shift: 9 } }] }, state: ClientData([InProgress, InProgress, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.754317254Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[0] Reports error GenericError(\"bad\") on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Read { dependencies: [], requests: [ReadRequest { eid: 0, offset: Block { value: 7, shift: 9 } }] }, state: ClientData([InProgress, InProgress, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.754348916Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[0] 1000 read error GenericError(\"bad\") DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Read { dependencies: [], requests: [ReadRequest { eid: 0, offset: Block { value: 7, shift: 9 } }] }, state: ClientData([Error(GenericError(\"bad\")), InProgress, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.754380702Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] DS Reports error Err(GenericError(\"bad\")) on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Read { dependencies: [], requests: [ReadRequest { eid: 0, offset: Block { value: 7, shift: 9 } }] }, state: ClientData([Error(GenericError(\"bad\")), InProgress, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.754418635Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] Reports error GenericError(\"bad\") on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Read { dependencies: [], requests: [ReadRequest { eid: 0, offset: Block { value: 7, shift: 9 } }] }, state: ClientData([Error(GenericError(\"bad\")), InProgress, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.754452842Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] 1000 read error GenericError(\"bad\") DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Read { dependencies: [], requests: [ReadRequest { eid: 0, offset: Block { value: 7, shift: 9 } }] }, state: ClientData([Error(GenericError(\"bad\")), Error(GenericError(\"bad\")), InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.754497918Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[0] DS Reports error Err(GenericError(\"bad\")) on job 1001, DownstairsIO { ds_id: JobId(1001), guest_id: 10, work: Read { dependencies: [], requests: [ReadRequest { eid: 0, offset: Block { value: 7, shift: 9 } }] }, state: ClientData([InProgress, InProgress, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.754532837Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[0] Reports error GenericError(\"bad\") on job 1001, DownstairsIO { ds_id: JobId(1001), guest_id: 10, work: Read { dependencies: [], requests: [ReadRequest { eid: 0, offset: Block { value: 7, shift: 9 } }] }, state: ClientData([InProgress, InProgress, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.754563214Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[0] 1001 read error GenericError(\"bad\") DownstairsIO { ds_id: JobId(1001), guest_id: 10, work: Read { dependencies: [], requests: [ReadRequest { eid: 0, offset: Block { value: 7, shift: 9 } }] }, state: ClientData([Error(GenericError(\"bad\")), InProgress, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.754594698Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] DS Reports error Err(GenericError(\"bad\")) on job 1001, DownstairsIO { ds_id: JobId(1001), guest_id: 10, work: Read { dependencies: [], requests: [ReadRequest { eid: 0, offset: Block { value: 7, shift: 9 } }] }, state: ClientData([Error(GenericError(\"bad\")), InProgress, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.754626039Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] Reports error GenericError(\"bad\") on job 1001, DownstairsIO { ds_id: JobId(1001), guest_id: 10, work: Read { dependencies: [], requests: [ReadRequest { eid: 0, offset: Block { value: 7, shift: 9 } }] }, state: ClientData([Error(GenericError(\"bad\")), InProgress, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.754661287Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] 1001 read error GenericError(\"bad\") DownstairsIO { ds_id: JobId(1001), guest_id: 10, work: Read { dependencies: [], requests: [ReadRequest { eid: 0, offset: Block { value: 7, shift: 9 } }] }, state: ClientData([Error(GenericError(\"bad\")), Error(GenericError(\"bad\")), InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.754694314Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
test test::up_test::work_assert_reads_do_not_cause_failure_state_transition ... ok
{"msg":"Crucible stats registered with UUID: 3d847606-c75c-46cf-82c9-8794a622434c","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.755221645Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 3d847606-c75c-46cf-82c9-8794a622434c has session id: 69bcabe1-4670-4eca-ae5c-d082c072d3d0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.75525527Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"3d847606-c75c-46cf-82c9-8794a622434c is now active with session: 69524606-a143-4db3-8351-b92a36cf2321","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.75527652Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] client re-new 1 jobs since flush 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.755294946Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
test test::up_test::work_completed_ack_read_replay ... ok
{"msg":"Crucible stats registered with UUID: 32742f7c-515e-43dd-bdeb-51ff74c46398","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.75590451Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 32742f7c-515e-43dd-bdeb-51ff74c46398 has session id: 33d6b101-42d9-4730-8445-e576abff9839","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.755938449Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"32742f7c-515e-43dd-bdeb-51ff74c46398 is now active with session: f0daa040-168e-4f27-9d99-6e7a270e32bb","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.755959407Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] client re-new 1 jobs since flush 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.755977812Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"REPLAY [0] read hash mismatch on id 1000\nExpected [Some(48a5a7677a8e488)]\nComputed [Some(ea9ca750a094f609)]\nguest_id:10 request:[ReadRequest { eid: 0, offset: Block { value: 7, shift: 9 } }]\njob state:ClientData([Done, InProgress, InProgress])","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.756004789Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
test test::up_test::work_completed_ack_read_replay_hash_mismatch ... ok
{"msg":"Crucible stats registered with UUID: 487cc194-afc0-45e3-8c68-666b799132d5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.756572783Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 487cc194-afc0-45e3-8c68-666b799132d5 has session id: 0a9c0c50-3be5-4fdc-91f5-aa6d365b9fc8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.756606945Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"487cc194-afc0-45e3-8c68-666b799132d5 is now active with session: c2032703-82ac-4414-a09a-b23f7e88407b","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.756627532Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] client re-new 1 jobs since flush 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.756645899Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"REPLAY [1] read hash mismatch on id 1000\nExpected [Some(48a5a7677a8e488)]\nComputed [Some(ea9ca750a094f609)]\nguest_id:10 request:[ReadRequest { eid: 0, offset: Block { value: 7, shift: 9 } }]\njob state:ClientData([Done, Done, InProgress])","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.75667198Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
test test::up_test::work_completed_ack_read_replay_two_hash_mismatch ... ok
{"msg":"Crucible stats registered with UUID: d10943d5-f081-48a9-9755-5b69eff5a43a","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.757225351Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible d10943d5-f081-48a9-9755-5b69eff5a43a has session id: 64671ce6-8a86-49b8-9f17-3184d1b12fd5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.757258736Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"d10943d5-f081-48a9-9755-5b69eff5a43a is now active with session: 5ef75342-87bc-420c-8b5d-c1f883e82c5a","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.757279542Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::work_completed_read_flush ... ok
{"msg":"Crucible stats registered with UUID: 46e3d841-473a-4479-8f4b-80bab25e2dc9","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.757926942Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 46e3d841-473a-4479-8f4b-80bab25e2dc9 has session id: e4ad383b-1180-4835-91a0-3c842f91f24c","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.757960274Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"46e3d841-473a-4479-8f4b-80bab25e2dc9 is now active with session: fa8fa0f3-1bb3-4442-bf94-cadaad9d806e","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.757980825Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] client re-new 1 jobs since flush 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.757999287Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"Remove read data for 1000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.758018408Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
test test::up_test::work_completed_read_replay ... ok
{"msg":"Crucible stats registered with UUID: ca559093-9d7d-488b-a4ba-f958fb30775b","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.758571666Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible ca559093-9d7d-488b-a4ba-f958fb30775b has session id: 0eca10ae-efc8-4813-8795-14dcc01bc5fc","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.758604825Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"ca559093-9d7d-488b-a4ba-f958fb30775b is now active with session: a6a9f1ee-69a5-4b6e-bffb-ea3a79d6305a","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.758631567Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] client re-new 1 jobs since flush 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.758651276Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] client re-new 1 jobs since flush 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.758670735Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"Remove read data for 1000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.758689527Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
test test::up_test::work_completed_two_read_replay ... ok
{"msg":"Crucible stats registered with UUID: 8a7b82aa-7655-47d5-9cc5-a40ab98dd0f7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.759250447Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 8a7b82aa-7655-47d5-9cc5-a40ab98dd0f7 has session id: 5888f378-0167-4ed0-87b6-b77df76ef320","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.759289014Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"8a7b82aa-7655-47d5-9cc5-a40ab98dd0f7 is now active with session: 9292b68c-d1fa-4d40-a83d-78aa9fe177c8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.75931052Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] client re-new 1 jobs since flush 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.759329209Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
test test::up_test::work_completed_write_ack_ready_replay_write ... ok
{"msg":"Crucible stats registered with UUID: f50f25cc-03f7-4f76-8f7a-654e90068744","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.759877755Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible f50f25cc-03f7-4f76-8f7a-654e90068744 has session id: c3637db0-52e8-4474-9a61-1055cd937119","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.759911306Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"f50f25cc-03f7-4f76-8f7a-654e90068744 is now active with session: e026a9da-2ed3-46e3-b787-5cd82887fab3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.759931908Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] client re-new 1 jobs since flush 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.759950218Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"Remove AckReady for Wu/F 1000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.759969519Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
test test::up_test::work_completed_write_ack_ready_replay_write_unwritten ... ok
{"msg":"Crucible stats registered with UUID: a240b570-e127-43c4-96a3-22778a1fa328","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.760515162Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible a240b570-e127-43c4-96a3-22778a1fa328 has session id: 0069abb3-2e5f-4951-9912-c3d9208f3a26","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.760548717Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"a240b570-e127-43c4-96a3-22778a1fa328 is now active with session: d6758c14-eeb5-41b0-9ed6-23e779b95294","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.760569457Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] client re-new 1 jobs since flush 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.760587934Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
test test::up_test::work_completed_write_acked_replay_write ... ok
{"msg":"Crucible stats registered with UUID: 967bffb3-d157-4029-a4b6-32ab3150ee8f","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.761143802Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 967bffb3-d157-4029-a4b6-32ab3150ee8f has session id: 5738ab77-8fc4-416f-b1e5-411f532b3563","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.761177202Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"967bffb3-d157-4029-a4b6-32ab3150ee8f is now active with session: 58ae5c4e-225e-4e3c-8714-e0261695db4b","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.761197937Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] client re-new 1 jobs since flush 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.761216362Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
test test::up_test::work_completed_write_acked_replay_write_unwritten ... ok
{"msg":"Crucible stats registered with UUID: f40934a3-fdee-44c9-9d71-a1d1d03fea19","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.761838917Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible f40934a3-fdee-44c9-9d71-a1d1d03fea19 has session id: b73f194c-9d4d-4c8e-9865-c567c649fc38","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.761877987Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"f40934a3-fdee-44c9-9d71-a1d1d03fea19 is now active with session: 8418f7e0-43db-406c-9f0b-702bdd2c897f","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.761900163Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::work_completed_write_flush ... ok
{"msg":"Crucible stats registered with UUID: 7bd3b9d3-f60f-42a6-9eda-04d5a1f1c6f1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.762551647Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 7bd3b9d3-f60f-42a6-9eda-04d5a1f1c6f1 has session id: b3d5524f-3e8f-4d47-b5bf-8ec35b91f9fa","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.762586482Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"7bd3b9d3-f60f-42a6-9eda-04d5a1f1c6f1 is now active with session: 51ee745f-e6b3-4b19-ab88-b71826522751","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.762607671Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::work_completed_write_unwritten_flush ... ok
{"msg":"Crucible stats registered with UUID: 12461ba7-0953-4115-b9c7-f5f2db94a25f","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.763196016Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 12461ba7-0953-4115-b9c7-f5f2db94a25f has session id: 19c7c0e9-2722-4c7e-96bd-3e13d708aecb","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.763229594Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"12461ba7-0953-4115-b9c7-f5f2db94a25f is now active with session: f984c317-2afb-4ac5-a655-52c94b263a05","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.763250265Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::work_delay_completion_flush_order_write ... ok
{"msg":"Crucible stats registered with UUID: 42f2d956-5bba-405a-8481-734c4465721c","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.763897979Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 42f2d956-5bba-405a-8481-734c4465721c has session id: b8d9ce3c-6c5e-4952-972d-b7ecfcaab302","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.76393135Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"42f2d956-5bba-405a-8481-734c4465721c is now active with session: 88ff3002-9740-423a-bbb2-f884972a2f8c","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.763952555Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::work_delay_completion_flush_order_write_unwritten ... ok
{"msg":"Crucible stats registered with UUID: 95feca81-c3d9-4608-80e5-1033c39f45c4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.764600006Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 95feca81-c3d9-4608-80e5-1033c39f45c4 has session id: bec7408f-7630-470d-b4a9-8c856037eab6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.764633945Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"95feca81-c3d9-4608-80e5-1033c39f45c4 is now active with session: c07153a0-e57e-470b-b4b4-c8c88b42ee17","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.764654946Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::work_delay_completion_flush_write ... ok
{"msg":"Crucible stats registered with UUID: 3321ed8d-989a-454d-a5f8-c78cc2732803","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.765316083Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 3321ed8d-989a-454d-a5f8-c78cc2732803 has session id: 8029d8c0-7998-4a28-88d6-16952816795a","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.765349793Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"3321ed8d-989a-454d-a5f8-c78cc2732803 is now active with session: 0657acee-3aa2-4b8f-9b44-83402e8dad84","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.765370794Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::work_delay_completion_flush_write_unwritten ... ok
{"msg":"Crucible stats registered with UUID: 44b091f9-cc65-47df-9514-3b0c43cb2566","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.766009679Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 44b091f9-cc65-47df-9514-3b0c43cb2566 has session id: 41c2e9a0-1d6f-43bf-9419-719b43de250d","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.766043594Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"44b091f9-cc65-47df-9514-3b0c43cb2566 is now active with session: 19565874-d9c4-4abf-a855-def36b2b783a","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.766064534Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] DS Reports error Err(GenericError(\"bad\")) on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 0, work: Flush { dependencies: [], flush_number: 10, gen_number: 0, snapshot_details: None, extent_limit: None }, state: ClientData([InProgress, InProgress, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.76609271Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[0] Reports error GenericError(\"bad\") on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 0, work: Flush { dependencies: [], flush_number: 10, gen_number: 0, snapshot_details: None, extent_limit: None }, state: ClientData([InProgress, InProgress, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.766122599Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
test test::up_test::work_flush_one_error_then_ok ... ok
{"msg":"Crucible stats registered with UUID: bb5e3b9e-cec5-4a34-9509-2d2d03f676dc","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.766683299Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible bb5e3b9e-cec5-4a34-9509-2d2d03f676dc has session id: 2b846f00-7bfb-4d28-8317-a7993b81af7a","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.766716746Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"bb5e3b9e-cec5-4a34-9509-2d2d03f676dc is now active with session: a4a4cfcf-ceb0-4c0b-b665-b118949ef7d1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.766743443Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::work_flush_snapshot_needs_three ... ok
{"msg":"Crucible stats registered with UUID: bd0c05db-c33d-4701-aed7-cd3200156978","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.767295146Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible bd0c05db-c33d-4701-aed7-cd3200156978 has session id: bceefb14-642d-4368-93e8-ad29c08fbc77","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.76732873Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"bd0c05db-c33d-4701-aed7-cd3200156978 is now active with session: bc1ab4ca-8dfa-4006-bcbf-88eede5e3b42","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.767349826Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::work_flush_three_ok ... ok
{"msg":"Crucible stats registered with UUID: e3168393-c2fa-4c5b-a918-146bdacd66ad","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.767904641Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible e3168393-c2fa-4c5b-a918-146bdacd66ad has session id: d6df6f8e-c3a6-436f-b7ab-618ce9b17356","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.767938254Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"e3168393-c2fa-4c5b-a918-146bdacd66ad is now active with session: dabdbf9f-c8e8-418c-b850-1c5c51575d75","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.767958966Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] DS Reports error Err(GenericError(\"bad\")) on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 0, work: Flush { dependencies: [], flush_number: 10, gen_number: 0, snapshot_details: None, extent_limit: None }, state: ClientData([InProgress, InProgress, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.767987165Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[0] Reports error GenericError(\"bad\") on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 0, work: Flush { dependencies: [], flush_number: 10, gen_number: 0, snapshot_details: None, extent_limit: None }, state: ClientData([InProgress, InProgress, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.768016439Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] DS Reports error Err(GenericError(\"bad\")) on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 0, work: Flush { dependencies: [], flush_number: 10, gen_number: 0, snapshot_details: None, extent_limit: None }, state: ClientData([Error(GenericError(\"bad\")), Done, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.768045477Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] Reports error GenericError(\"bad\") on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 0, work: Flush { dependencies: [], flush_number: 10, gen_number: 0, snapshot_details: None, extent_limit: None }, state: ClientData([Error(GenericError(\"bad\")), Done, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.768074439Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
test test::up_test::work_flush_two_errors_equals_fail ... ok
{"msg":"Crucible stats registered with UUID: 54379435-01b9-4102-9528-16f19bf3ab86","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.768652112Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 54379435-01b9-4102-9528-16f19bf3ab86 has session id: 8b86af86-7e60-46d0-bba4-f78f32e15b28","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.768685487Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"54379435-01b9-4102-9528-16f19bf3ab86 is now active with session: 0cd31d9d-58f5-4ad3-9b4f-107e13b35079","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.768706337Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::work_read_hash_mismatch ... ok
{"msg":"Crucible stats registered with UUID: 684a8c0d-d58a-426e-b44f-af7a313a5e8d","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.76947464Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 684a8c0d-d58a-426e-b44f-af7a313a5e8d has session id: 968fe636-6cc3-4343-8452-40171183a36f","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.769507762Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"684a8c0d-d58a-426e-b44f-af7a313a5e8d is now active with session: 2e418330-6130-4c3f-bc56-73f902aa2b9f","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.769528505Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::work_read_hash_mismatch_ack ... ok
{"msg":"Crucible stats registered with UUID: a8498796-f66a-43ec-9e57-6c73cf8382f7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.770184581Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible a8498796-f66a-43ec-9e57-6c73cf8382f7 has session id: 3380c477-6e82-43e8-b0fa-bf69ed3e8f1e","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.770218186Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"a8498796-f66a-43ec-9e57-6c73cf8382f7 is now active with session: 90280377-8ca1-461d-b335-83e34b288efe","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.770239075Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::work_read_hash_mismatch_inside ... ok
{"msg":"Crucible stats registered with UUID: 6499e0f3-38fd-4fde-bda2-e50b70ff0622","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.770954717Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 6499e0f3-38fd-4fde-bda2-e50b70ff0622 has session id: 5cb4d9a9-f270-4651-88b7-0d0d57976333","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.770988018Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"6499e0f3-38fd-4fde-bda2-e50b70ff0622 is now active with session: 22793feb-c68c-4a5f-9f86-ba4e929afb5f","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.771008687Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::work_read_hash_mismatch_no_data ... ok
{"msg":"Crucible stats registered with UUID: cc74aa76-3419-4349-a09c-27ed06d1c01f","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.771664365Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible cc74aa76-3419-4349-a09c-27ed06d1c01f has session id: fb7aefd3-8ae7-4f7f-8cf3-6b410a31d9e3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.771697493Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"cc74aa76-3419-4349-a09c-27ed06d1c01f is now active with session: e2f85476-f322-4b08-84d2-094d38df2f08","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.771718158Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::work_read_hash_mismatch_no_data_next ... ok
test test::up_test::work_read_hash_mismatch_third ... ok
{"msg":"Crucible stats registered with UUID: 774e44c7-f9d0-4ab2-bc7b-d62c3b99fd89","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.772994127Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 774e44c7-f9d0-4ab2-bc7b-d62c3b99fd89 has session id: 5a867bc5-f945-47b1-83c3-65528d4c5967","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.773033405Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"774e44c7-f9d0-4ab2-bc7b-d62c3b99fd89 is now active with session: f1d0da1e-baf4-4ea1-a759-9524dae41f33","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.773055503Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::work_read_hash_mismatch_third_ack ... ok
{"msg":"Crucible stats registered with UUID: 9408d4f5-4f72-4231-be2d-7992e1d0655a","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.773711249Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 9408d4f5-4f72-4231-be2d-7992e1d0655a has session id: d3ebd7d2-d13d-4651-847a-54fa059af30f","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.773744746Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"9408d4f5-4f72-4231-be2d-7992e1d0655a is now active with session: 7b635bfa-33f1-424f-bbe3-6c11912fa3b9","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.773765594Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] DS Reports error Err(GenericError(\"bad\")) on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Read { dependencies: [], requests: [ReadRequest { eid: 0, offset: Block { value: 7, shift: 9 } }] }, state: ClientData([InProgress, InProgress, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.773795793Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[0] Reports error GenericError(\"bad\") on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Read { dependencies: [], requests: [ReadRequest { eid: 0, offset: Block { value: 7, shift: 9 } }] }, state: ClientData([InProgress, InProgress, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.773827645Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[0] 1000 read error GenericError(\"bad\") DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Read { dependencies: [], requests: [ReadRequest { eid: 0, offset: Block { value: 7, shift: 9 } }] }, state: ClientData([Error(GenericError(\"bad\")), InProgress, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.773859301Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
test test::up_test::work_read_one_bad_two_ok ... ok
{"msg":"Crucible stats registered with UUID: d11acf42-c71d-4172-8ff1-ebfff311ac98","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.774430228Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible d11acf42-c71d-4172-8ff1-ebfff311ac98 has session id: 353d62dc-25aa-4615-be96-5097c15d0000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.774463674Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"d11acf42-c71d-4172-8ff1-ebfff311ac98 is now active with session: 00811f44-4b83-435a-aab7-eb3086b41cb9","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.774484452Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::work_read_one_ok ... ok
{"msg":"Crucible stats registered with UUID: 867579c1-1c7c-4587-b352-2937db1508c1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.775105384Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 867579c1-1c7c-4587-b352-2937db1508c1 has session id: 0aeda27e-f9ab-4818-b5b5-bfc9df18ab4e","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.775138663Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"867579c1-1c7c-4587-b352-2937db1508c1 is now active with session: 28880833-7f1f-47c0-a7db-be972653d01c","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.775159286Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] DS Reports error Err(GenericError(\"bad\")) on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Read { dependencies: [], requests: [ReadRequest { eid: 0, offset: Block { value: 7, shift: 9 } }] }, state: ClientData([InProgress, InProgress, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.775189585Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[0] Reports error GenericError(\"bad\") on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Read { dependencies: [], requests: [ReadRequest { eid: 0, offset: Block { value: 7, shift: 9 } }] }, state: ClientData([InProgress, InProgress, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.775221006Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[0] 1000 read error GenericError(\"bad\") DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Read { dependencies: [], requests: [ReadRequest { eid: 0, offset: Block { value: 7, shift: 9 } }] }, state: ClientData([Error(GenericError(\"bad\")), InProgress, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.775252414Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] DS Reports error Err(GenericError(\"bad\")) on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Read { dependencies: [], requests: [ReadRequest { eid: 0, offset: Block { value: 7, shift: 9 } }] }, state: ClientData([Error(GenericError(\"bad\")), InProgress, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.775284231Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] Reports error GenericError(\"bad\") on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Read { dependencies: [], requests: [ReadRequest { eid: 0, offset: Block { value: 7, shift: 9 } }] }, state: ClientData([Error(GenericError(\"bad\")), InProgress, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.775316679Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] 1000 read error GenericError(\"bad\") DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Read { dependencies: [], requests: [ReadRequest { eid: 0, offset: Block { value: 7, shift: 9 } }] }, state: ClientData([Error(GenericError(\"bad\")), Error(GenericError(\"bad\")), InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.775349192Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] DS Reports error Err(GenericError(\"bad\")) on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Read { dependencies: [], requests: [ReadRequest { eid: 0, offset: Block { value: 7, shift: 9 } }] }, state: ClientData([Error(GenericError(\"bad\")), Error(GenericError(\"bad\")), InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.775388665Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] Reports error GenericError(\"bad\") on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Read { dependencies: [], requests: [ReadRequest { eid: 0, offset: Block { value: 7, shift: 9 } }] }, state: ClientData([Error(GenericError(\"bad\")), Error(GenericError(\"bad\")), InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.775422971Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] 1000 read error GenericError(\"bad\") DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Read { dependencies: [], requests: [ReadRequest { eid: 0, offset: Block { value: 7, shift: 9 } }] }, state: ClientData([Error(GenericError(\"bad\")), Error(GenericError(\"bad\")), Error(GenericError(\"bad\"))]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.775455756Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
test test::up_test::work_read_three_bad ... ok
{"msg":"Crucible stats registered with UUID: 9149a4b9-99e7-4272-8e5a-a8e1d4a70d6b","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.776004988Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 9149a4b9-99e7-4272-8e5a-a8e1d4a70d6b has session id: a53f09aa-31e4-4f9a-9ef7-c5c2a9fe2f95","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.776038692Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"9149a4b9-99e7-4272-8e5a-a8e1d4a70d6b is now active with session: 7f1b7fd2-3cc2-4a18-a4ac-a2f025f22f53","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.776059482Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] DS Reports error Err(GenericError(\"bad\")) on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Read { dependencies: [], requests: [ReadRequest { eid: 0, offset: Block { value: 7, shift: 9 } }] }, state: ClientData([InProgress, InProgress, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.776089934Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[0] Reports error GenericError(\"bad\") on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Read { dependencies: [], requests: [ReadRequest { eid: 0, offset: Block { value: 7, shift: 9 } }] }, state: ClientData([InProgress, InProgress, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.776121057Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[0] 1000 read error GenericError(\"bad\") DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Read { dependencies: [], requests: [ReadRequest { eid: 0, offset: Block { value: 7, shift: 9 } }] }, state: ClientData([Error(GenericError(\"bad\")), InProgress, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.77615827Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] DS Reports error Err(GenericError(\"bad\")) on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Read { dependencies: [], requests: [ReadRequest { eid: 0, offset: Block { value: 7, shift: 9 } }] }, state: ClientData([Error(GenericError(\"bad\")), InProgress, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.77619145Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] Reports error GenericError(\"bad\") on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Read { dependencies: [], requests: [ReadRequest { eid: 0, offset: Block { value: 7, shift: 9 } }] }, state: ClientData([Error(GenericError(\"bad\")), InProgress, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.77622293Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] 1000 read error GenericError(\"bad\") DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Read { dependencies: [], requests: [ReadRequest { eid: 0, offset: Block { value: 7, shift: 9 } }] }, state: ClientData([Error(GenericError(\"bad\")), Error(GenericError(\"bad\")), InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.776254722Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
test test::up_test::work_read_two_bad_one_ok ... ok
{"msg":"Crucible stats registered with UUID: 9de103ab-4de1-4141-b9e2-0d4ad7cdf391","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.776820904Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 9de103ab-4de1-4141-b9e2-0d4ad7cdf391 has session id: 0bfe2bc3-2226-46ee-b197-26f5feee2d43","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.776854393Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"9de103ab-4de1-4141-b9e2-0d4ad7cdf391 is now active with session: 843e1bca-d671-479d-9c98-5719d9da9ec1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.776875083Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 9de103ab-4de1-4141-b9e2-0d4ad7cdf391 WARNING finish job 1000 when downstairs state:New","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:11.776895065Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 9de103ab-4de1-4141-b9e2-0d4ad7cdf391 WARNING finish job 1000 when downstairs state:New","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:11.77691487Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 9de103ab-4de1-4141-b9e2-0d4ad7cdf391 WARNING finish job 1000 when downstairs state:New","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:11.776940661Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] DS Reports error Err(GenericError(\"bad\")) on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Read { dependencies: [], requests: [ReadRequest { eid: 0, offset: Block { value: 7, shift: 9 } }] }, state: ClientData([Done, InProgress, Done]), ack_status: Acked, replay: false, data: Some([ReadResponse { eid: 0, offset: Block { value: 7, shift: 9 }, data: b\"\", block_contexts: [BlockContext { hash: 17241709254077376921, encryption_context: None }] }]), read_response_hashes: [Some(17241709254077376921)], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.776977137Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] Reports error GenericError(\"bad\") on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Read { dependencies: [], requests: [ReadRequest { eid: 0, offset: Block { value: 7, shift: 9 } }] }, state: ClientData([Done, InProgress, Done]), ack_status: Acked, replay: false, data: Some([ReadResponse { eid: 0, offset: Block { value: 7, shift: 9 }, data: b\"\", block_contexts: [BlockContext { hash: 17241709254077376921, encryption_context: None }] }]), read_response_hashes: [Some(17241709254077376921)], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.777013562Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] 1000 read error GenericError(\"bad\") DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Read { dependencies: [], requests: [ReadRequest { eid: 0, offset: Block { value: 7, shift: 9 } }] }, state: ClientData([Done, Error(GenericError(\"bad\")), Done]), ack_status: Acked, replay: false, data: Some([ReadResponse { eid: 0, offset: Block { value: 7, shift: 9 }, data: b\"\", block_contexts: [BlockContext { hash: 17241709254077376921, encryption_context: None }] }]), read_response_hashes: [Some(17241709254077376921)], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.7770497Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
test test::up_test::work_read_two_ok_one_bad ... ok
{"msg":"Crucible stats registered with UUID: 388dc96b-446c-4f84-aa2e-4a349a17bdab","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.777616348Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 388dc96b-446c-4f84-aa2e-4a349a17bdab has session id: 455f5f55-8830-4dab-8a60-cb7a980aa3c8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.77765032Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"388dc96b-446c-4f84-aa2e-4a349a17bdab is now active with session: 1ca4a2c2-3555-4d83-acfc-2381bc5f10ae","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.777671138Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] DS Reports error Err(GenericError(\"bad\")) on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: WriteUnwritten { dependencies: [], writes: [Write { eid: 0, offset: Block { value: 7, shift: 9 }, data: b\"\\x01\", block_context: BlockContext { hash: 0, encryption_context: None } }] }, state: ClientData([InProgress, InProgress, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.777704628Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[0] Reports error GenericError(\"bad\") on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: WriteUnwritten { dependencies: [], writes: [Write { eid: 0, offset: Block { value: 7, shift: 9 }, data: b\"\\x01\", block_context: BlockContext { hash: 0, encryption_context: None } }] }, state: ClientData([InProgress, InProgress, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.77774456Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] DS Reports error Err(GenericError(\"bad\")) on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: WriteUnwritten { dependencies: [], writes: [Write { eid: 0, offset: Block { value: 7, shift: 9 }, data: b\"\\x01\", block_context: BlockContext { hash: 0, encryption_context: None } }] }, state: ClientData([Error(GenericError(\"bad\")), InProgress, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.777781499Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] Reports error GenericError(\"bad\") on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: WriteUnwritten { dependencies: [], writes: [Write { eid: 0, offset: Block { value: 7, shift: 9 }, data: b\"\\x01\", block_context: BlockContext { hash: 0, encryption_context: None } }] }, state: ClientData([Error(GenericError(\"bad\")), InProgress, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.777815844Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
test test::up_test::work_write_errors_are_counted ... ok
{"msg":"Crucible stats registered with UUID: 584d6086-2bd6-4a47-9661-162273a4cbd8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.778367314Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 584d6086-2bd6-4a47-9661-162273a4cbd8 has session id: f720aa8e-a1f0-4c1e-8107-2d9e04c358ed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.778401174Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"584d6086-2bd6-4a47-9661-162273a4cbd8 is now active with session: daeed3aa-3720-4ae9-8698-570801ddc87d","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.778421837Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] DS Reports error Err(GenericError(\"bad\")) on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Write { dependencies: [], writes: [Write { eid: 0, offset: Block { value: 7, shift: 9 }, data: b\"\\x01\", block_context: BlockContext { hash: 0, encryption_context: None } }] }, state: ClientData([InProgress, InProgress, InProgress]), ack_status: AckReady, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.778455159Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[0] Reports error GenericError(\"bad\") on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Write { dependencies: [], writes: [Write { eid: 0, offset: Block { value: 7, shift: 9 }, data: b\"\\x01\", block_context: BlockContext { hash: 0, encryption_context: None } }] }, state: ClientData([InProgress, InProgress, InProgress]), ack_status: AckReady, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.7784897Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] DS Reports error Err(GenericError(\"bad\")) on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Write { dependencies: [], writes: [Write { eid: 0, offset: Block { value: 7, shift: 9 }, data: b\"\\x01\", block_context: BlockContext { hash: 0, encryption_context: None } }] }, state: ClientData([Error(GenericError(\"bad\")), InProgress, InProgress]), ack_status: AckReady, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.778532264Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] Reports error GenericError(\"bad\") on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Write { dependencies: [], writes: [Write { eid: 0, offset: Block { value: 7, shift: 9 }, data: b\"\\x01\", block_context: BlockContext { hash: 0, encryption_context: None } }] }, state: ClientData([Error(GenericError(\"bad\")), InProgress, InProgress]), ack_status: AckReady, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.778567903Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
test test::up_test::work_write_unwritten_errors_are_counted ... ok
{"msg":"Crucible stats registered with UUID: c3324075-9a8b-48e2-b877-1a956a371051","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.779132036Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible c3324075-9a8b-48e2-b877-1a956a371051 has session id: 2ec2225d-4d74-4b4d-ab98-ec5e43d1bb0d","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.779165797Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] c3324075-9a8b-48e2-b877-1a956a371051 (ea5b09c3-02ec-4513-8a6d-def06c87433b) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.779187368Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.779206003Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] c3324075-9a8b-48e2-b877-1a956a371051 (ea5b09c3-02ec-4513-8a6d-def06c87433b) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.779226427Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.779244821Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] c3324075-9a8b-48e2-b877-1a956a371051 (ea5b09c3-02ec-4513-8a6d-def06c87433b) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.779264908Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.779283076Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] c3324075-9a8b-48e2-b877-1a956a371051 (ea5b09c3-02ec-4513-8a6d-def06c87433b) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.779303299Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.779326636Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] c3324075-9a8b-48e2-b877-1a956a371051 (ea5b09c3-02ec-4513-8a6d-def06c87433b) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.779348154Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.779366359Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] c3324075-9a8b-48e2-b877-1a956a371051 (ea5b09c3-02ec-4513-8a6d-def06c87433b) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.779386439Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.779404349Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] c3324075-9a8b-48e2-b877-1a956a371051 (ea5b09c3-02ec-4513-8a6d-def06c87433b) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.779424521Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.779442504Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] c3324075-9a8b-48e2-b877-1a956a371051 (ea5b09c3-02ec-4513-8a6d-def06c87433b) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.779462648Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.779480962Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] c3324075-9a8b-48e2-b877-1a956a371051 (ea5b09c3-02ec-4513-8a6d-def06c87433b) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.779501233Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.779519126Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"c3324075-9a8b-48e2-b877-1a956a371051 is now active with session: ea5b09c3-02ec-4513-8a6d-def06c87433b","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.779538562Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] DS Reports error Err(GenericError(\"bad\")) on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Write { dependencies: [], writes: [Write { eid: 0, offset: Block { value: 7, shift: 9 }, data: b\"\\x01\", block_context: BlockContext { hash: 0, encryption_context: None } }] }, state: ClientData([InProgress, InProgress, InProgress]), ack_status: AckReady, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.779571029Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[0] Reports error GenericError(\"bad\") on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Write { dependencies: [], writes: [Write { eid: 0, offset: Block { value: 7, shift: 9 }, data: b\"\\x01\", block_context: BlockContext { hash: 0, encryption_context: None } }] }, state: ClientData([InProgress, InProgress, InProgress]), ack_status: AckReady, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.779604516Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[0] client skip 1 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.779628584Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[0] changed 0 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.779648587Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[0] c3324075-9a8b-48e2-b877-1a956a371051 (ea5b09c3-02ec-4513-8a6d-def06c87433b) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.779670058Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.779688009Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] DS Reports error Err(GenericError(\"bad\")) on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Write { dependencies: [], writes: [Write { eid: 0, offset: Block { value: 7, shift: 9 }, data: b\"\\x01\", block_context: BlockContext { hash: 0, encryption_context: None } }] }, state: ClientData([Error(GenericError(\"bad\")), InProgress, InProgress]), ack_status: AckReady, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.779720632Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] Reports error GenericError(\"bad\") on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Write { dependencies: [], writes: [Write { eid: 0, offset: Block { value: 7, shift: 9 }, data: b\"\\x01\", block_context: BlockContext { hash: 0, encryption_context: None } }] }, state: ClientData([Error(GenericError(\"bad\")), InProgress, InProgress]), ack_status: AckReady, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.779754308Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] client skip 1 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.779775049Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] changed 0 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.779794259Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] c3324075-9a8b-48e2-b877-1a956a371051 (ea5b09c3-02ec-4513-8a6d-def06c87433b) Faulted Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.779815558Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.779833386Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] DS Reports error Err(GenericError(\"bad\")) on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Write { dependencies: [], writes: [Write { eid: 0, offset: Block { value: 7, shift: 9 }, data: b\"\\x01\", block_context: BlockContext { hash: 0, encryption_context: None } }] }, state: ClientData([Error(GenericError(\"bad\")), Error(GenericError(\"bad\")), InProgress]), ack_status: AckReady, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.77986672Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] Reports error GenericError(\"bad\") on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Write { dependencies: [], writes: [Write { eid: 0, offset: Block { value: 7, shift: 9 }, data: b\"\\x01\", block_context: BlockContext { hash: 0, encryption_context: None } }] }, state: ClientData([Error(GenericError(\"bad\")), Error(GenericError(\"bad\")), InProgress]), ack_status: AckReady, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.779906558Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] client skip 1 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.779927582Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] changed 0 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.779946705Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] c3324075-9a8b-48e2-b877-1a956a371051 (ea5b09c3-02ec-4513-8a6d-def06c87433b) Faulted Faulted Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.779968885Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.779987104Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::work_writes_bad ... ok
{"msg":"Crucible stats registered with UUID: c804db91-8cb0-47a8-ae69-7e9036592932","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.780548802Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible c804db91-8cb0-47a8-ae69-7e9036592932 has session id: 062d0348-601a-40fd-8559-b52b9541cd0b","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.780582581Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] c804db91-8cb0-47a8-ae69-7e9036592932 (15a45cb2-9167-4006-9ad8-cbdd2abb55c1) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.780604014Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.780622544Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] c804db91-8cb0-47a8-ae69-7e9036592932 (15a45cb2-9167-4006-9ad8-cbdd2abb55c1) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.780642984Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.78066113Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] c804db91-8cb0-47a8-ae69-7e9036592932 (15a45cb2-9167-4006-9ad8-cbdd2abb55c1) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.780681268Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.780699184Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] c804db91-8cb0-47a8-ae69-7e9036592932 (15a45cb2-9167-4006-9ad8-cbdd2abb55c1) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.780718929Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.78073673Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] c804db91-8cb0-47a8-ae69-7e9036592932 (15a45cb2-9167-4006-9ad8-cbdd2abb55c1) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.78075714Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.780780828Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] c804db91-8cb0-47a8-ae69-7e9036592932 (15a45cb2-9167-4006-9ad8-cbdd2abb55c1) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.780802112Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.780820085Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] c804db91-8cb0-47a8-ae69-7e9036592932 (15a45cb2-9167-4006-9ad8-cbdd2abb55c1) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.780840198Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.780858115Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] c804db91-8cb0-47a8-ae69-7e9036592932 (15a45cb2-9167-4006-9ad8-cbdd2abb55c1) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.780878326Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.780896438Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] c804db91-8cb0-47a8-ae69-7e9036592932 (15a45cb2-9167-4006-9ad8-cbdd2abb55c1) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.780916502Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.78093433Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"c804db91-8cb0-47a8-ae69-7e9036592932 is now active with session: 15a45cb2-9167-4006-9ad8-cbdd2abb55c1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.780953701Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] DS Reports error Err(GenericError(\"bad\")) on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Write { dependencies: [], writes: [Write { eid: 0, offset: Block { value: 7, shift: 9 }, data: b\"\\x01\", block_context: BlockContext { hash: 0, encryption_context: None } }] }, state: ClientData([Done, InProgress, InProgress]), ack_status: AckReady, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.780985902Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] Reports error GenericError(\"bad\") on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Write { dependencies: [], writes: [Write { eid: 0, offset: Block { value: 7, shift: 9 }, data: b\"\\x01\", block_context: BlockContext { hash: 0, encryption_context: None } }] }, state: ClientData([Done, InProgress, InProgress]), ack_status: AckReady, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.781019782Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] client skip 1 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.781040325Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] changed 0 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.781063583Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] c804db91-8cb0-47a8-ae69-7e9036592932 (15a45cb2-9167-4006-9ad8-cbdd2abb55c1) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.781086273Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.781104543Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::write_after_write_fail_is_alright ... ok
{"msg":"Crucible stats registered with UUID: 1d0fc563-24c1-4002-a210-f2cb52557866","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.781675116Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 1d0fc563-24c1-4002-a210-f2cb52557866 has session id: 400507b3-3433-46f1-97f9-6f69f8272a87","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.781708544Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"1d0fc563-24c1-4002-a210-f2cb52557866 is now active with session: 35fd1c32-1e91-431a-be5b-1e2e51f05fc7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.78172941Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 1d0fc563-24c1-4002-a210-f2cb52557866 (35fd1c32-1e91-431a-be5b-1e2e51f05fc7) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.781749652Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.781768155Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 1d0fc563-24c1-4002-a210-f2cb52557866 (35fd1c32-1e91-431a-be5b-1e2e51f05fc7) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.781788416Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.781806595Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 1d0fc563-24c1-4002-a210-f2cb52557866 (35fd1c32-1e91-431a-be5b-1e2e51f05fc7) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.781826668Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.781844803Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 1d0fc563-24c1-4002-a210-f2cb52557866 (35fd1c32-1e91-431a-be5b-1e2e51f05fc7) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.781864928Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.781882834Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 1d0fc563-24c1-4002-a210-f2cb52557866 (35fd1c32-1e91-431a-be5b-1e2e51f05fc7) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.781903078Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.781921091Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 1d0fc563-24c1-4002-a210-f2cb52557866 (35fd1c32-1e91-431a-be5b-1e2e51f05fc7) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.781941128Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.781967124Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 1d0fc563-24c1-4002-a210-f2cb52557866 (35fd1c32-1e91-431a-be5b-1e2e51f05fc7) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.781988692Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.782006692Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 1d0fc563-24c1-4002-a210-f2cb52557866 (35fd1c32-1e91-431a-be5b-1e2e51f05fc7) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.782027087Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.782045371Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 1d0fc563-24c1-4002-a210-f2cb52557866 (35fd1c32-1e91-431a-be5b-1e2e51f05fc7) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.782065578Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.782083465Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 1d0fc563-24c1-4002-a210-f2cb52557866 (35fd1c32-1e91-431a-be5b-1e2e51f05fc7) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.782103593Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.782121457Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 1d0fc563-24c1-4002-a210-f2cb52557866 (35fd1c32-1e91-431a-be5b-1e2e51f05fc7) Active Faulted Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.782141664Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.782159665Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::write_double_skip ... ok
{"msg":"Crucible stats registered with UUID: 951798d4-06eb-46b9-b898-b1bc7cdbac03","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.782724151Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 951798d4-06eb-46b9-b898-b1bc7cdbac03 has session id: 20f02494-f73d-4af2-a509-6d1b03bd10d1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.782757366Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"951798d4-06eb-46b9-b898-b1bc7cdbac03 is now active with session: ccabf884-3bc2-43ec-876d-1738744a59bb","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.782778243Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 951798d4-06eb-46b9-b898-b1bc7cdbac03 (ccabf884-3bc2-43ec-876d-1738744a59bb) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.782798596Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.782816921Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 951798d4-06eb-46b9-b898-b1bc7cdbac03 (ccabf884-3bc2-43ec-876d-1738744a59bb) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.782837075Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.782860747Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 951798d4-06eb-46b9-b898-b1bc7cdbac03 (ccabf884-3bc2-43ec-876d-1738744a59bb) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.78288211Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.782900199Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 951798d4-06eb-46b9-b898-b1bc7cdbac03 (ccabf884-3bc2-43ec-876d-1738744a59bb) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.782920526Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.782938553Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 951798d4-06eb-46b9-b898-b1bc7cdbac03 (ccabf884-3bc2-43ec-876d-1738744a59bb) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.782958891Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.782976953Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 951798d4-06eb-46b9-b898-b1bc7cdbac03 (ccabf884-3bc2-43ec-876d-1738744a59bb) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.782996909Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.783014669Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 951798d4-06eb-46b9-b898-b1bc7cdbac03 (ccabf884-3bc2-43ec-876d-1738744a59bb) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.783034671Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.783052454Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 951798d4-06eb-46b9-b898-b1bc7cdbac03 (ccabf884-3bc2-43ec-876d-1738744a59bb) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.783072447Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.783090531Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 951798d4-06eb-46b9-b898-b1bc7cdbac03 (ccabf884-3bc2-43ec-876d-1738744a59bb) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.783110498Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.78312834Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 951798d4-06eb-46b9-b898-b1bc7cdbac03 (ccabf884-3bc2-43ec-876d-1738744a59bb) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.783148478Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.783166202Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] DS Reports error Err(GenericError(\"bad\")) on job 1010, DownstairsIO { ds_id: JobId(1010), guest_id: 19, work: Write { dependencies: [], writes: [Write { eid: 0, offset: Block { value: 7, shift: 9 }, data: b\"\\x01\", block_context: BlockContext { hash: 0, encryption_context: None } }] }, state: ClientData([Done, InProgress, Skipped]), ack_status: AckReady, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.783198242Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] Reports error GenericError(\"bad\") on job 1010, DownstairsIO { ds_id: JobId(1010), guest_id: 19, work: Write { dependencies: [], writes: [Write { eid: 0, offset: Block { value: 7, shift: 9 }, data: b\"\\x01\", block_context: BlockContext { hash: 0, encryption_context: None } }] }, state: ClientData([Done, InProgress, Skipped]), ack_status: AckReady, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.783236158Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"All done","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.783257141Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::write_fail_and_skip ... ok
{"msg":"Crucible stats registered with UUID: 50316693-20c8-443c-8411-a56850f10150","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.783819536Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 50316693-20c8-443c-8411-a56850f10150 has session id: 96bec26f-2277-4c5c-93d7-ba39f7d53456","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.783853765Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 50316693-20c8-443c-8411-a56850f10150 (df9b5363-3c6a-4de3-b7ee-261fe7d436a0) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.783875372Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.783893781Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 50316693-20c8-443c-8411-a56850f10150 (df9b5363-3c6a-4de3-b7ee-261fe7d436a0) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.783914135Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.783932255Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 50316693-20c8-443c-8411-a56850f10150 (df9b5363-3c6a-4de3-b7ee-261fe7d436a0) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.783952301Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.783970309Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 50316693-20c8-443c-8411-a56850f10150 (df9b5363-3c6a-4de3-b7ee-261fe7d436a0) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.783990436Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.784008394Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 50316693-20c8-443c-8411-a56850f10150 (df9b5363-3c6a-4de3-b7ee-261fe7d436a0) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.784028648Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.784046727Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 50316693-20c8-443c-8411-a56850f10150 (df9b5363-3c6a-4de3-b7ee-261fe7d436a0) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.784071754Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.784090931Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 50316693-20c8-443c-8411-a56850f10150 (df9b5363-3c6a-4de3-b7ee-261fe7d436a0) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.784111163Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.784129138Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 50316693-20c8-443c-8411-a56850f10150 (df9b5363-3c6a-4de3-b7ee-261fe7d436a0) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.784149274Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.78416725Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 50316693-20c8-443c-8411-a56850f10150 (df9b5363-3c6a-4de3-b7ee-261fe7d436a0) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.784187681Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.784205629Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"50316693-20c8-443c-8411-a56850f10150 is now active with session: df9b5363-3c6a-4de3-b7ee-261fe7d436a0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.784225193Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] DS Reports error Err(GenericError(\"bad\")) on job 1002, DownstairsIO { ds_id: JobId(1002), guest_id: 10, work: Write { dependencies: [], writes: [Write { eid: 0, offset: Block { value: 7, shift: 9 }, data: b\"\\x01\", block_context: BlockContext { hash: 0, encryption_context: None } }] }, state: ClientData([Done, Done, InProgress]), ack_status: AckReady, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.784257415Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] Reports error GenericError(\"bad\") on job 1002, DownstairsIO { ds_id: JobId(1002), guest_id: 10, work: Write { dependencies: [], writes: [Write { eid: 0, offset: Block { value: 7, shift: 9 }, data: b\"\\x01\", block_context: BlockContext { hash: 0, encryption_context: None } }] }, state: ClientData([Done, Done, InProgress]), ack_status: AckReady, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.784291912Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] client skip 4 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.78431241Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.784331582Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] 50316693-20c8-443c-8411-a56850f10150 (df9b5363-3c6a-4de3-b7ee-261fe7d436a0) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.784357572Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.784376576Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::write_fail_past_present_future ... ok
{"msg":"Crucible stats registered with UUID: f5dc7f31-30ff-4dc8-8ace-68972e04e382","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.784943057Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible f5dc7f31-30ff-4dc8-8ace-68972e04e382 has session id: e07a7c91-8c68-41d9-938e-06b8a87c4132","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.784976559Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] f5dc7f31-30ff-4dc8-8ace-68972e04e382 (0140ae24-3bbf-4340-90b6-64ff5c34b9d1) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.784998056Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.785016518Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] f5dc7f31-30ff-4dc8-8ace-68972e04e382 (0140ae24-3bbf-4340-90b6-64ff5c34b9d1) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.785036915Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.785055143Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] f5dc7f31-30ff-4dc8-8ace-68972e04e382 (0140ae24-3bbf-4340-90b6-64ff5c34b9d1) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.785075127Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.785092972Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] f5dc7f31-30ff-4dc8-8ace-68972e04e382 (0140ae24-3bbf-4340-90b6-64ff5c34b9d1) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.785113102Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.785131094Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] f5dc7f31-30ff-4dc8-8ace-68972e04e382 (0140ae24-3bbf-4340-90b6-64ff5c34b9d1) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.785151472Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.785169415Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] f5dc7f31-30ff-4dc8-8ace-68972e04e382 (0140ae24-3bbf-4340-90b6-64ff5c34b9d1) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.785189361Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.785207226Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] f5dc7f31-30ff-4dc8-8ace-68972e04e382 (0140ae24-3bbf-4340-90b6-64ff5c34b9d1) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.785227404Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.785250941Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] f5dc7f31-30ff-4dc8-8ace-68972e04e382 (0140ae24-3bbf-4340-90b6-64ff5c34b9d1) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.785272638Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.785290769Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] f5dc7f31-30ff-4dc8-8ace-68972e04e382 (0140ae24-3bbf-4340-90b6-64ff5c34b9d1) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.785310976Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.78532876Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"f5dc7f31-30ff-4dc8-8ace-68972e04e382 is now active with session: 0140ae24-3bbf-4340-90b6-64ff5c34b9d1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.78534833Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] DS Reports error Err(GenericError(\"bad\")) on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Write { dependencies: [], writes: [Write { eid: 0, offset: Block { value: 7, shift: 9 }, data: b\"\\x01\", block_context: BlockContext { hash: 0, encryption_context: None } }] }, state: ClientData([Done, InProgress, InProgress]), ack_status: AckReady, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.78538078Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] Reports error GenericError(\"bad\") on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Write { dependencies: [], writes: [Write { eid: 0, offset: Block { value: 7, shift: 9 }, data: b\"\\x01\", block_context: BlockContext { hash: 0, encryption_context: None } }] }, state: ClientData([Done, InProgress, InProgress]), ack_status: AckReady, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.785414327Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] client skip 2 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.785435105Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.785454242Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] f5dc7f31-30ff-4dc8-8ace-68972e04e382 (0140ae24-3bbf-4340-90b6-64ff5c34b9d1) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.785476089Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.785494372Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::write_fail_skips_inprogress_jobs ... ok
{"msg":"Crucible stats registered with UUID: 96ad6640-83d8-4bf4-ad16-c20b9e10e8b4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.786092667Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 96ad6640-83d8-4bf4-ad16-c20b9e10e8b4 has session id: f42bc518-963d-42fc-9e5c-737d117844c1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.786126348Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 96ad6640-83d8-4bf4-ad16-c20b9e10e8b4 (7b290629-99c5-4421-825c-072ef12b9607) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.786153464Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.786173123Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 96ad6640-83d8-4bf4-ad16-c20b9e10e8b4 (7b290629-99c5-4421-825c-072ef12b9607) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.786193494Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.786211654Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 96ad6640-83d8-4bf4-ad16-c20b9e10e8b4 (7b290629-99c5-4421-825c-072ef12b9607) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.786231443Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.786249299Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 96ad6640-83d8-4bf4-ad16-c20b9e10e8b4 (7b290629-99c5-4421-825c-072ef12b9607) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.786269106Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.786286778Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 96ad6640-83d8-4bf4-ad16-c20b9e10e8b4 (7b290629-99c5-4421-825c-072ef12b9607) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.786307097Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.786325199Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 96ad6640-83d8-4bf4-ad16-c20b9e10e8b4 (7b290629-99c5-4421-825c-072ef12b9607) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.786345288Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.786363099Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 96ad6640-83d8-4bf4-ad16-c20b9e10e8b4 (7b290629-99c5-4421-825c-072ef12b9607) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.78638334Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.78640126Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 96ad6640-83d8-4bf4-ad16-c20b9e10e8b4 (7b290629-99c5-4421-825c-072ef12b9607) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.786421177Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.786439094Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 96ad6640-83d8-4bf4-ad16-c20b9e10e8b4 (7b290629-99c5-4421-825c-072ef12b9607) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.786459163Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.786481284Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"96ad6640-83d8-4bf4-ad16-c20b9e10e8b4 is now active with session: 7b290629-99c5-4421-825c-072ef12b9607","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.786501764Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] DS Reports error Err(GenericError(\"bad\")) on job 1002, DownstairsIO { ds_id: JobId(1002), guest_id: 10, work: Write { dependencies: [], writes: [Write { eid: 0, offset: Block { value: 7, shift: 9 }, data: b\"\\x01\", block_context: BlockContext { hash: 0, encryption_context: None } }] }, state: ClientData([Done, Done, InProgress]), ack_status: AckReady, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.786533996Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] Reports error GenericError(\"bad\") on job 1002, DownstairsIO { ds_id: JobId(1002), guest_id: 10, work: Write { dependencies: [], writes: [Write { eid: 0, offset: Block { value: 7, shift: 9 }, data: b\"\\x01\", block_context: BlockContext { hash: 0, encryption_context: None } }] }, state: ClientData([Done, Done, InProgress]), ack_status: AckReady, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.786567184Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] client skip 3 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.786587078Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] changed 0 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.786606015Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] 96ad6640-83d8-4bf4-ad16-c20b9e10e8b4 (7b290629-99c5-4421-825c-072ef12b9607) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.786627875Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.786645931Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::write_fail_skips_many_jobs ... ok
{"msg":"Crucible stats registered with UUID: 79752912-ce8a-4d2f-a2a8-9a24046a746d","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.787199059Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 79752912-ce8a-4d2f-a2a8-9a24046a746d has session id: 0012ea56-2d1b-46fd-a597-6f481536df0a","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.787232369Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 79752912-ce8a-4d2f-a2a8-9a24046a746d (c027a9d2-7afa-471b-8cc7-9f51eefe6c60) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.787253944Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.787272431Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 79752912-ce8a-4d2f-a2a8-9a24046a746d (c027a9d2-7afa-471b-8cc7-9f51eefe6c60) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.787292824Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.787314379Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 79752912-ce8a-4d2f-a2a8-9a24046a746d (c027a9d2-7afa-471b-8cc7-9f51eefe6c60) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.787341231Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.787360626Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 79752912-ce8a-4d2f-a2a8-9a24046a746d (c027a9d2-7afa-471b-8cc7-9f51eefe6c60) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.787380682Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.787398305Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 79752912-ce8a-4d2f-a2a8-9a24046a746d (c027a9d2-7afa-471b-8cc7-9f51eefe6c60) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.787418607Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.7874366Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 79752912-ce8a-4d2f-a2a8-9a24046a746d (c027a9d2-7afa-471b-8cc7-9f51eefe6c60) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.787456653Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.787474481Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 79752912-ce8a-4d2f-a2a8-9a24046a746d (c027a9d2-7afa-471b-8cc7-9f51eefe6c60) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.787494525Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.787512163Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 79752912-ce8a-4d2f-a2a8-9a24046a746d (c027a9d2-7afa-471b-8cc7-9f51eefe6c60) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.787532404Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.787550319Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 79752912-ce8a-4d2f-a2a8-9a24046a746d (c027a9d2-7afa-471b-8cc7-9f51eefe6c60) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.787570243Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.787587939Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"79752912-ce8a-4d2f-a2a8-9a24046a746d is now active with session: c027a9d2-7afa-471b-8cc7-9f51eefe6c60","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.787607411Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] DS Reports error Err(GenericError(\"bad\")) on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Write { dependencies: [], writes: [Write { eid: 0, offset: Block { value: 7, shift: 9 }, data: b\"\\x01\", block_context: BlockContext { hash: 0, encryption_context: None } }] }, state: ClientData([Done, InProgress, InProgress]), ack_status: AckReady, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.787639562Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] Reports error GenericError(\"bad\") on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Write { dependencies: [], writes: [Write { eid: 0, offset: Block { value: 7, shift: 9 }, data: b\"\\x01\", block_context: BlockContext { hash: 0, encryption_context: None } }] }, state: ClientData([Done, InProgress, InProgress]), ack_status: AckReady, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.787676905Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] client skip 2 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.787697698Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.78771666Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] 79752912-ce8a-4d2f-a2a8-9a24046a746d (c027a9d2-7afa-471b-8cc7-9f51eefe6c60) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.787738487Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.787756626Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::write_fail_skips_new_jobs ... ok
{"msg":"Crucible stats registered with UUID: 63029047-35d1-4ade-9268-107c50cf3afa","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.788308969Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 63029047-35d1-4ade-9268-107c50cf3afa has session id: ed76b75f-9c74-481b-9727-a52c6e51f95f","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.788342773Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"63029047-35d1-4ade-9268-107c50cf3afa is now active with session: 477d83cc-37f0-408d-bb6d-d256eecf22be","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.788363537Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 63029047-35d1-4ade-9268-107c50cf3afa (477d83cc-37f0-408d-bb6d-d256eecf22be) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.78838379Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.788402352Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 63029047-35d1-4ade-9268-107c50cf3afa (477d83cc-37f0-408d-bb6d-d256eecf22be) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.78842268Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.788440729Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 63029047-35d1-4ade-9268-107c50cf3afa (477d83cc-37f0-408d-bb6d-d256eecf22be) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.788460703Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.788478547Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 63029047-35d1-4ade-9268-107c50cf3afa (477d83cc-37f0-408d-bb6d-d256eecf22be) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.788498691Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.788521963Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 63029047-35d1-4ade-9268-107c50cf3afa (477d83cc-37f0-408d-bb6d-d256eecf22be) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.788543434Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.788561464Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 63029047-35d1-4ade-9268-107c50cf3afa (477d83cc-37f0-408d-bb6d-d256eecf22be) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.788581666Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.788599705Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 63029047-35d1-4ade-9268-107c50cf3afa (477d83cc-37f0-408d-bb6d-d256eecf22be) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.788619741Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.788637579Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 63029047-35d1-4ade-9268-107c50cf3afa (477d83cc-37f0-408d-bb6d-d256eecf22be) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.788657848Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.788675761Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 63029047-35d1-4ade-9268-107c50cf3afa (477d83cc-37f0-408d-bb6d-d256eecf22be) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.788695811Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.788713735Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 63029047-35d1-4ade-9268-107c50cf3afa (477d83cc-37f0-408d-bb6d-d256eecf22be) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.788733664Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.788751442Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::write_single_skip ... ok
{"msg":"Crucible stats registered with UUID: 66e232ce-a5ce-4efc-a04a-375d43621361","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.789302489Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 66e232ce-a5ce-4efc-a04a-375d43621361 has session id: 5eb51674-bb08-490c-ae56-70af044f1f67","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.789336166Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"66e232ce-a5ce-4efc-a04a-375d43621361 is now active with session: 28e2db0e-563f-4b11-a465-a811b0c7f927","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.789357058Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 66e232ce-a5ce-4efc-a04a-375d43621361 (28e2db0e-563f-4b11-a465-a811b0c7f927) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.789377427Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.789401405Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 66e232ce-a5ce-4efc-a04a-375d43621361 (28e2db0e-563f-4b11-a465-a811b0c7f927) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.789423065Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.789441306Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 66e232ce-a5ce-4efc-a04a-375d43621361 (28e2db0e-563f-4b11-a465-a811b0c7f927) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.789461476Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.789479475Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 66e232ce-a5ce-4efc-a04a-375d43621361 (28e2db0e-563f-4b11-a465-a811b0c7f927) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.789499874Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.789517772Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 66e232ce-a5ce-4efc-a04a-375d43621361 (28e2db0e-563f-4b11-a465-a811b0c7f927) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.789538087Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.789556011Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 66e232ce-a5ce-4efc-a04a-375d43621361 (28e2db0e-563f-4b11-a465-a811b0c7f927) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.789575897Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.789593891Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 66e232ce-a5ce-4efc-a04a-375d43621361 (28e2db0e-563f-4b11-a465-a811b0c7f927) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.78961387Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.789631997Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 66e232ce-a5ce-4efc-a04a-375d43621361 (28e2db0e-563f-4b11-a465-a811b0c7f927) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.789652166Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.789670254Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 66e232ce-a5ce-4efc-a04a-375d43621361 (28e2db0e-563f-4b11-a465-a811b0c7f927) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.789690301Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.789708056Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 66e232ce-a5ce-4efc-a04a-375d43621361 (28e2db0e-563f-4b11-a465-a811b0c7f927) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.78973269Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.78975137Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 66e232ce-a5ce-4efc-a04a-375d43621361 (28e2db0e-563f-4b11-a465-a811b0c7f927) Active Faulted Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.789771403Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.789789329Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::write_unwritten_double_skip ... ok
{"msg":"Crucible stats registered with UUID: ff99b38c-4c8d-474d-956d-3205797ca532","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.790338934Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible ff99b38c-4c8d-474d-956d-3205797ca532 has session id: 7e2791a2-3f3e-4452-97d1-5eaf0c1a357d","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.790372338Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"ff99b38c-4c8d-474d-956d-3205797ca532 is now active with session: d1637da7-c607-4a1e-a7e6-5b2b3ea1094f","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.790393032Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] ff99b38c-4c8d-474d-956d-3205797ca532 (d1637da7-c607-4a1e-a7e6-5b2b3ea1094f) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.790413307Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.79043171Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] ff99b38c-4c8d-474d-956d-3205797ca532 (d1637da7-c607-4a1e-a7e6-5b2b3ea1094f) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.79045197Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.790469928Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] ff99b38c-4c8d-474d-956d-3205797ca532 (d1637da7-c607-4a1e-a7e6-5b2b3ea1094f) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.790490626Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.79050862Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] ff99b38c-4c8d-474d-956d-3205797ca532 (d1637da7-c607-4a1e-a7e6-5b2b3ea1094f) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.790528537Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.790546298Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] ff99b38c-4c8d-474d-956d-3205797ca532 (d1637da7-c607-4a1e-a7e6-5b2b3ea1094f) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.790566411Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.790584348Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] ff99b38c-4c8d-474d-956d-3205797ca532 (d1637da7-c607-4a1e-a7e6-5b2b3ea1094f) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.790610105Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.790629255Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] ff99b38c-4c8d-474d-956d-3205797ca532 (d1637da7-c607-4a1e-a7e6-5b2b3ea1094f) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.790649406Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.790667143Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] ff99b38c-4c8d-474d-956d-3205797ca532 (d1637da7-c607-4a1e-a7e6-5b2b3ea1094f) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.790687317Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.79070517Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] ff99b38c-4c8d-474d-956d-3205797ca532 (d1637da7-c607-4a1e-a7e6-5b2b3ea1094f) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.790725254Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.790743015Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] ff99b38c-4c8d-474d-956d-3205797ca532 (d1637da7-c607-4a1e-a7e6-5b2b3ea1094f) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.790762956Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.790780612Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] DS Reports error Err(GenericError(\"bad\")) on job 1010, DownstairsIO { ds_id: JobId(1010), guest_id: 19, work: WriteUnwritten { dependencies: [], writes: [Write { eid: 0, offset: Block { value: 7, shift: 9 }, data: b\"\\x01\", block_context: BlockContext { hash: 0, encryption_context: None } }] }, state: ClientData([Done, InProgress, Skipped]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.790813156Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] Reports error GenericError(\"bad\") on job 1010, DownstairsIO { ds_id: JobId(1010), guest_id: 19, work: WriteUnwritten { dependencies: [], writes: [Write { eid: 0, offset: Block { value: 7, shift: 9 }, data: b\"\\x01\", block_context: BlockContext { hash: 0, encryption_context: None } }] }, state: ClientData([Done, InProgress, Skipped]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:11.790846796Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"All done","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.790866369Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::write_unwritten_fail_and_skip ... ok
{"msg":"Crucible stats registered with UUID: 4cb1df6d-b8a4-4fed-a080-b7c433de4b52","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.791425178Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 4cb1df6d-b8a4-4fed-a080-b7c433de4b52 has session id: a8d6868e-530f-486d-a724-3af44cf7c6ac","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.791464099Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"4cb1df6d-b8a4-4fed-a080-b7c433de4b52 is now active with session: f2c4566a-ab5e-4396-96d2-c94e90ee81ee","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.791486242Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 4cb1df6d-b8a4-4fed-a080-b7c433de4b52 (f2c4566a-ab5e-4396-96d2-c94e90ee81ee) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.791506629Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.791524897Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 4cb1df6d-b8a4-4fed-a080-b7c433de4b52 (f2c4566a-ab5e-4396-96d2-c94e90ee81ee) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.791544962Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.791562981Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 4cb1df6d-b8a4-4fed-a080-b7c433de4b52 (f2c4566a-ab5e-4396-96d2-c94e90ee81ee) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.791582976Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.791600713Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 4cb1df6d-b8a4-4fed-a080-b7c433de4b52 (f2c4566a-ab5e-4396-96d2-c94e90ee81ee) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.791620504Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.791638342Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 4cb1df6d-b8a4-4fed-a080-b7c433de4b52 (f2c4566a-ab5e-4396-96d2-c94e90ee81ee) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.791658513Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.791676577Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 4cb1df6d-b8a4-4fed-a080-b7c433de4b52 (f2c4566a-ab5e-4396-96d2-c94e90ee81ee) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.791696497Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.791714424Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 4cb1df6d-b8a4-4fed-a080-b7c433de4b52 (f2c4566a-ab5e-4396-96d2-c94e90ee81ee) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.791734509Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.791752154Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 4cb1df6d-b8a4-4fed-a080-b7c433de4b52 (f2c4566a-ab5e-4396-96d2-c94e90ee81ee) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.791772237Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.791794148Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 4cb1df6d-b8a4-4fed-a080-b7c433de4b52 (f2c4566a-ab5e-4396-96d2-c94e90ee81ee) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.791814858Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.7918329Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 4cb1df6d-b8a4-4fed-a080-b7c433de4b52 (f2c4566a-ab5e-4396-96d2-c94e90ee81ee) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.791853138Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:11.791870742Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test test::up_test::write_unwritten_single_skip ... ok
test test_buffer_len ... ok
test test_buffer_len_after_clone ... ok
test test_buffer_len_index_overflow - should panic ... ok
test test_buffer_len_over_block_size ... ok
test test_return_iops ... ok
test volume::test::construct_file_block_io ... ok
test volume::test::test_affected_subvolumes ... ok
test volume::test::test_correct_blocks_returned_multiple_subvolumes_1 ... ok
test volume::test::test_correct_blocks_returned_multiple_subvolumes_2 ... ok
test volume::test::test_correct_blocks_returned_multiple_subvolumes_3 ... ok
test volume::test::test_correct_blocks_returned_one_subvolume ... ok
test volume::test::test_correct_blocks_returned_three_subvolumes ... ok
test volume::test::test_drop_then_recreate_test ... ok
test volume::test::test_in_memory_block_io ... ok
test volume::test::test_no_read_only_parent_for_lba_range ... ok
test volume::test::test_out_of_bounds ... ok
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:12.425739624Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:12.426728889Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:12.442838077Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:12.442890554Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Finally, move the ReOpen job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:12.569755914Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now ACK the reopen job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:12.569806794Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Bailing with error","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:12.569844636Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"err:1 or:0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:12.570035027Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible stats registered with UUID: ea9c2828-4faa-4745-9992-cc45fe6bf51b","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:12.570493482Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible ea9c2828-4faa-4745-9992-cc45fe6bf51b has session id: 2f505f29-5b73-4e36-b406-71bbf06e30ad","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:12.570548553Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] ea9c2828-4faa-4745-9992-cc45fe6bf51b (62b807af-88b8-4100-a47a-2cd55774bf63) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:12.570597688Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:12.570632829Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] ea9c2828-4faa-4745-9992-cc45fe6bf51b (62b807af-88b8-4100-a47a-2cd55774bf63) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:12.570669153Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30{"msg":","time":"2023-09-22T23:08:12.570701841Z[0] DS Reports error Err(GenericError("\",bad\""hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] ea9c2828-4faa-4745-9992-cc45fe6bf51b (62b807af-88b8-4100-a47a-2cd55774bf63) WaitQuorum New New ds_transition to Active","v":0)) on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentFlushClose { dependencies: [], extent: 0, flush_number: 0, gen_number: 0, source_downstairs: ClientId(2), repair_downstairs: [ClientId(1)] }, state: ClientData([InProgress, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) },""name":",crucible"","v"level"::300,"name":"crucible","level":50,"time":"2023-09-22T23:08:12.570744763Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":","time"[0] Transition from WaitQuorum to Active:"","v":02023-09-22T23:08:12.570750532Z","name",:""crucible"hostname",:""level":30ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
,"time":"2023-09-22T23:08:12.570783378Z"{,"hostname":""msg":"ip-10-150-1-74.us-west-2.compute.internal","[0] Reports error GenericError(\"pid":bad4291\"}
{"msg":"[1] ea9c2828-4faa-4745-9992-cc45fe6bf51b (62b807af-88b8-4100-a47a-2cd55774bf63) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30) on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentFlushClose { dependencies: [], extent: 0, flush_number: 0, gen_number: 0, source_downstairs: ClientId(2), repair_downstairs: [ClientId(1)] }, state: ClientData([InProgress, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:12.57082591Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal",",pid"":time"4291:"}
2023-09-22T23:08:12.570833444Z","{hostname":""msg":"ip-10-150-1-74.us-west-2.compute.internal","pid":[1] Transition from New to WaitActive4291",,""":v"":0downstairs,""}name"
:"crucible","{level":30"msg":"[0] client skip 2 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:12.570871833Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"time":}"
2023-09-22T23:08:12.57088064Z","{hostname":""msg":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
[1] ea9c2828-4faa-4745-9992-cc45fe6bf51b (62b807af-88b8-4100-a47a-2cd55774bf63) Active WaitActive New ds_transition to WaitQuorum","v{":0,""msg"name"::""crucible","[0] changed 1 jobs to fault skipped"level",:"30v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:12.570918465Z",,""hostname"time"::""2023-09-22T23:08:12.570922871Z"ip-10-150-1-74.us-west-2.compute.internal",,""pid"hostname"::4291"}
ip-10-150-1-74.us-west-2.compute.internal","pid":{4291,""":"msg":"downstairs"}
[1] Transition from WaitActive to WaitQuorum","{v":0,""name"msg"::""crucible","level":30[0] bb7a424d-9b30-4a13-96a7-a41c0f6628b1 (5a93f758-ea9a-4a78-ad67-6f82f309f96e) Active LiveRepair Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:12.570965018Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":,4291"time":"}
2023-09-22T23:08:12.570974091Z","hostname{":""msg":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
[1] ea9c2828-4faa-4745-9992-cc45fe6bf51b (62b807af-88b8-4100-a47a-2cd55774bf63) Active WaitQuorum New ds_transition to Active","v":0,{"name":"crucible""msg",":"level":30[0] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:12.571014615Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,}"
time":"{2023-09-22T23:08:12.571021778Z",""msg":hostname"":"[1] Transition from WaitQuorum to Active"ip-10-150-1-74.us-west-2.compute.internal",,""v"pid"::04291,"}name
":"crucible"{,"level"":msg30":"Now ACK the close job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:12.5710622Z","hostname":","time":"ip-10-150-1-74.us-west-2.compute.internal","pid":2023-09-22T23:08:12.571069067Z"4291,"}hostname"
:"{ip-10-150-1-74.us-west-2.compute.internal",""msg"pid"::4291"}
{"[2] ea9c2828-4faa-4745-9992-cc45fe6bf51b (62b807af-88b8-4100-a47a-2cd55774bf63) Active Active New ds_transition to WaitActive"msg",:""v":0,"Waiting for 3 jobs (currently 2)"name",:""v":crucible"0,","level"name"::30"crucible","level":30,"time":"2023-09-22T23:08:12.571112777Z",",hostname"":time"":"ip-10-150-1-74.us-west-2.compute.internal"2023-09-22T23:08:12.571115454Z,"",pid"":4291hostname":"}
ip-10-150-1-74.us-west-2.compute.internal","pid"{:4291"}
msg":"{[2] Transition from New to WaitActive","v"":msg"0:,""name":"crucible"Extent 0 close id:1000 Failed: Error: bad",","level"v"::300,"name":"crucible","level":50,"time":"2023-09-22T23:08:12.571161046Z",","time":hostname"":"2023-09-22T23:08:12.571164917Z",ip-10-150-1-74.us-west-2.compute.internal"","hostnamepid""::"4291}
ip-10-150-1-74.us-west-2.compute.internal","pid":{4291}
"msg":"{"msg":"[1] client skip 2 in process jobs because fault","v":0,[2] ea9c2828-4faa-4745-9992-cc45fe6bf51b (62b807af-88b8-4100-a47a-2cd55774bf63) Active Active WaitActive ds_transition to WaitQuorum"","namev""::"0,crucible"","name":level"":30crucible","level":30,"time":","2023-09-22T23:08:12.571210519Z"time",:""hostname":"2023-09-22T23:08:12.5712126Z","ip-10-150-1-74.us-west-2.compute.internal"hostname,"":pid"":4291,"":ip-10-150-1-74.us-west-2.compute.internal"","downstairs"pid"}:
4291}
{"{msg":""msg":"[1] changed 1 jobs to fault skipped","v":0,"[2] Transition from WaitActive to WaitQuorum"name":,""vcrucible"",:"0level,"":name30":"crucible","level":30,"time":"2023-09-22T23:08:12.571260186Z",",hostname"":"time":"ip-10-150-1-74.us-west-2.compute.internal2023-09-22T23:08:12.571263196Z"",,""hostname"pid"::"4291,"":"ip-10-150-1-74.us-west-2.compute.internal"downstairs",}"
pid":4291{}
"msg":"{"msg":"[1] bb7a424d-9b30-4a13-96a7-a41c0f6628b1 (5a93f758-ea9a-4a78-ad67-6f82f309f96e) Faulted LiveRepair Active ds_transition to Faulted","v":0,"name"[2] ea9c2828-4faa-4745-9992-cc45fe6bf51b (62b807af-88b8-4100-a47a-2cd55774bf63) Active Active WaitQuorum ds_transition to Active":","crucible"v",:"0level",:"30name":"crucible","level":30,"time":"2023-09-22T23:08:12.571312175Z","hostname",:""time":"ip-10-150-1-74.us-west-2.compute.internal"2023-09-22T23:08:12.571315449Z",","pid":hostname"4291:"}
ip-10-150-1-74.us-west-2.compute.internal","pid{":4291"msg"}:
"{[1] Transition from LiveRepair to Faulted"","msg"v"::"0,"name":"[2] Transition from WaitQuorum to Active"crucible",,""v":level0":,"30name":"crucible","level":30,"time":"2023-09-22T23:08:12.571359089Z","hostname",:""time":"ip-10-150-1-74.us-west-2.compute.internal"2023-09-22T23:08:12.571362359Z",","pid":hostname"4291:"}
ip-10-150-1-74.us-west-2.compute.internal","pid":{4291}"
msg":"{RE:0 Wait for result from repair command 1001:2""msg",:""v":0,"name":"crucible","level":30ea9c2828-4faa-4745-9992-cc45fe6bf51b is now active with session: 62b807af-88b8-4100-a47a-2cd55774bf63","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:12.571401213Z",",time":""hostname":"2023-09-22T23:08:12.571406838Z","ip-10-150-1-74.us-west-2.compute.internal"hostname,"":"pid":4291ip-10-150-1-74.us-west-2.compute.internal}"
,"pid":4291}
{"msg":"[0] ea9c2828-4faa-4745-9992-cc45fe6bf51b (62b807af-88b8-4100-a47a-2cd55774bf63) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:12.571545219Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:12.571590417Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] ea9c2828-4faa-4745-9992-cc45fe6bf51b (62b807af-88b8-4100-a47a-2cd55774bf63) Faulted Active Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:12.57163536Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:12.5716764Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] ea9c2828-4faa-4745-9992-cc45fe6bf51b (62b807af-88b8-4100-a47a-2cd55774bf63) LiveRepairReady Active Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:12.571720749Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:12.571762137Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for Close + ReOpen jobs","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:12.57179744Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:12.571837079Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 close id:1000 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:12.571876363Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from close command 1000:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:12.571916457Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for 3 jobs (currently 2)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:12.574729738Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"No repair needed for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:12.57478005Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:12.574824498Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now ACK the close job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:12.68799694Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for 3 jobs (currently 2)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:12.688046906Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"No repair needed for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:12.688081192Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:12.688117304Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test volume::test::test_parent_initialized_read_only_region_one_subvolume ... ok
{"msg":"Waiting for 3 jobs (currently 2)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:13.572049685Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"No repair needed for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:13.572127741Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:13.572166495Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:13.572314027Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Extent 0 close id:1001 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:13.572346769Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:13.572383503Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:13.576087978Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:13.576142749Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] DS Reports error Err(GenericError(\"bad\")) on job 1001, DownstairsIO { ds_id: JobId(1001), guest_id: 2, work: ExtentLiveNoOp { dependencies: [JobId(1000)] }, state: ClientData([Done, Done, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:13.689334823Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] Reports error GenericError(\"bad\") on job 1001, DownstairsIO { ds_id: JobId(1001), guest_id: 2, work: ExtentLiveNoOp { dependencies: [JobId(1000)] }, state: ClientData([Done, Done, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:13.689403495Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] client skip 3 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:13.689485247Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:13.689527788Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] c2759bf2-aaf5-4d2a-82e7-2795ddf1ec4e (c90c1fcd-3603-46fd-b4f7-87867e35920a) LiveRepair Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:13.689566254Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:13.689586598Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:13.689604618Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Extent 0 close id:1001 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:13.689622504Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] client skip 3 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:13.68964071Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[0] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:13.68965976Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[0] c2759bf2-aaf5-4d2a-82e7-2795ddf1ec4e (c90c1fcd-3603-46fd-b4f7-87867e35920a) LiveRepair Active Faulted ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:13.689681187Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from LiveRepair to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:13.689705992Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:13.689725289Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now move the NoOp job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.572717585Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now ACK the NoOp job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.572771206Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Finally, move the ReOpen job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.572791208Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now ACK the Reopen job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.572809116Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Extent 0 close id:1002 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:14.572837966Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from reopen command 1003:4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.572857844Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Extent 0 close id:1003 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:14.572876401Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Bailing with error","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:14.572894481Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
----------------------------------------------------------------
 Crucible gen:0 GIO:true work queues:  Upstairs:0  downstairs:4
GW_ID      ACK  DSID    TYPE  BLOCKS  DS:0  DS:1  DS:2  REPLAY
    1    Acked  1000  FClose       0   Err  Done  Done  false
    2    Acked  1001    NoOp       0  Skip  Skip  Done  false
    3    Acked  1002    NoOp       0  Skip  Skip  Done  false
    4    Acked  1003  Reopen       0  Skip  Skip  Done  false
   STATES      DS:0   DS:1   DS:2   TOTAL
    New           0      0      0      0
    Sent          0      0      0      0
    Done          0      1      4      5
    Skipped       3      3      0      6
    Error         1      0      0      1
Last Flush: 0 0 0 
Downstairs last five completed:
Upstairs last five completed:      4    3    2    1
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.573472496Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.573526518Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible stats registered with UUID: 4bf3db3d-50a4-4818-86ec-4e619b46b97c","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.573809699Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 4bf3db3d-50a4-4818-86ec-4e619b46b97c has session id: 62650f10-827e-49f2-8652-65f6b8972ff8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.573866044Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 4bf3db3d-50a4-4818-86ec-4e619b46b97c (bf820409-3b29-4b81-8028-3c3affe54370) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.573903761Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.573934913Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 4bf3db3d-50a4-4818-86ec-4e619b46b97c (bf820409-3b29-4b81-8028-3c3affe54370) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.573980229Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.574013566Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 4bf3db3d-50a4-4818-86ec-4e619b46b97c (bf820409-3b29-4b81-8028-3c3affe54370) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.574046778Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.574076478Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 4bf3db3d-50a4-4818-86ec-4e619b46b97c (bf820409-3b29-4b81-8028-3c3affe54370) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.574111735Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.574143974Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 4bf3db3d-50a4-4818-86ec-4e619b46b97c (bf820409-3b29-4b81-8028-3c3affe54370) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.574177528Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.574207087Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 4bf3db3d-50a4-4818-86ec-4e619b46b97c (bf820409-3b29-4b81-8028-3c3affe54370) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.574240231Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.574269495Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 4bf3db3d-50a4-4818-86ec-4e619b46b97c (bf820409-3b29-4b81-8028-3c3affe54370) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.574302616Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.574331777Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 4bf3db3d-50a4-4818-86ec-4e619b46b97c (bf820409-3b29-4b81-8028-3c3affe54370) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.574387091Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.574421074Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 4bf3db3d-50a4-4818-86ec-4e619b46b97c (bf820409-3b29-4b81-8028-3c3affe54370) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.574454858Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.574484326Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"4bf3db3d-50a4-4818-86ec-4e619b46b97c is now active with session: bf820409-3b29-4b81-8028-3c3affe54370","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.574515978Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 4bf3db3d-50a4-4818-86ec-4e619b46b97c (bf820409-3b29-4b81-8028-3c3affe54370) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.57455282Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.574593529Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 4bf3db3d-50a4-4818-86ec-4e619b46b97c (bf820409-3b29-4b81-8028-3c3affe54370) Active Faulted Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.574628448Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.574658093Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 4bf3db3d-50a4-4818-86ec-4e619b46b97c (bf820409-3b29-4b81-8028-3c3affe54370) Active LiveRepairReady Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.574693719Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.574724004Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for Close + ReOpen jobs","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.5747531Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.574783421Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 close id:1000 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.574812767Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from close command 1000:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.574842413Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now move the NoOp job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.577382429Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] DS Reports error Err(GenericError(\"bad\")) on job 1003, DownstairsIO { ds_id: JobId(1003), guest_id: 4, work: ExtentLiveReopen { dependencies: [JobId(1000), JobId(1001), JobId(1002)], extent: 0 }, state: ClientData([InProgress, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:14.577488903Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[0] Reports error GenericError(\"bad\") on job 1003, DownstairsIO { ds_id: JobId(1003), guest_id: 4, work: ExtentLiveReopen { dependencies: [JobId(1000), JobId(1001), JobId(1002)], extent: 0 }, state: ClientData([InProgress, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:14.577552122Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[0] client skip 4 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.577597441Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[0] changed 0 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.577638558Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[0] e5daa207-6492-4466-92d0-cc614d6c56ce (8a6b2194-d5ad-4cea-9c66-5ded3e31a721) Active LiveRepair Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.577682185Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.577725707Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from reopen command 1003:4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.577764707Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Extent 0 close id:1003 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:14.57780158Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] client skip 4 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.577833358Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] changed 0 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.577853751Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] e5daa207-6492-4466-92d0-cc614d6c56ce (8a6b2194-d5ad-4cea-9c66-5ded3e31a721) Faulted LiveRepair Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.577876028Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from LiveRepair to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.577894053Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Bailing with error","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:14.577911333Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible stats registered with UUID: d53fef88-17c8-441f-9cbe-e9f976239c5e","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.578284791Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible d53fef88-17c8-441f-9cbe-e9f976239c5e has session id: 1717818e-3bcd-47d2-8b65-284c19fd759f","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.578338345Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] d53fef88-17c8-441f-9cbe-e9f976239c5e (36f4bb72-2a32-45e5-94a6-6cee1f536fe0) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.578379856Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.578417052Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] d53fef88-17c8-441f-9cbe-e9f976239c5e (36f4bb72-2a32-45e5-94a6-6cee1f536fe0) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.578457024Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.578495532Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] d53fef88-17c8-441f-9cbe-e9f976239c5e (36f4bb72-2a32-45e5-94a6-6cee1f536fe0) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.578538325Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.578578003Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] d53fef88-17c8-441f-9cbe-e9f976239c5e (36f4bb72-2a32-45e5-94a6-6cee1f536fe0) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.578620788Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.578652867Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] d53fef88-17c8-441f-9cbe-e9f976239c5e (36f4bb72-2a32-45e5-94a6-6cee1f536fe0) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.578681071Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.578700484Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] d53fef88-17c8-441f-9cbe-e9f976239c5e (36f4bb72-2a32-45e5-94a6-6cee1f536fe0) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.578720573Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.578738455Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] d53fef88-17c8-441f-9cbe-e9f976239c5e (36f4bb72-2a32-45e5-94a6-6cee1f536fe0) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.57875831Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.578776273Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] d53fef88-17c8-441f-9cbe-e9f976239c5e (36f4bb72-2a32-45e5-94a6-6cee1f536fe0) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.578796469Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.578814338Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] d53fef88-17c8-441f-9cbe-e9f976239c5e (36f4bb72-2a32-45e5-94a6-6cee1f536fe0) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.578834337Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.578852101Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"d53fef88-17c8-441f-9cbe-e9f976239c5e is now active with session: 36f4bb72-2a32-45e5-94a6-6cee1f536fe0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.578871378Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] d53fef88-17c8-441f-9cbe-e9f976239c5e (36f4bb72-2a32-45e5-94a6-6cee1f536fe0) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.578891421Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.578909098Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] d53fef88-17c8-441f-9cbe-e9f976239c5e (36f4bb72-2a32-45e5-94a6-6cee1f536fe0) Active Faulted Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.578929145Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.578947175Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] d53fef88-17c8-441f-9cbe-e9f976239c5e (36f4bb72-2a32-45e5-94a6-6cee1f536fe0) Active LiveRepairReady Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.578967602Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.578985584Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for Close + ReOpen jobs","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.579007824Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.579027091Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 close id:1000 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.579044892Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from close command 1000:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.57906289Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now move the NoOp job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.689840554Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now ACK the NoOp job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.68989005Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Finally, move the ReOpen job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.689930542Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now ACK the Reopen job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.689969392Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Extent 0 close id:1002 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:14.690007433Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from reopen command 1003:4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.690041302Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Extent 0 close id:1003 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:14.690074493Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Bailing with error","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:14.690111052Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible stats registered with UUID: 678cfcb8-a5e0-4920-b486-0e81743672c3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.690489689Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 678cfcb8-a5e0-4920-b486-0e81743672c3 has session id: 033e8bcc-deed-4cb7-8711-a570c88db5ff","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.690547861Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 678cfcb8-a5e0-4920-b486-0e81743672c3 (d9d086f6-80b5-4805-9193-71fbf1abd250) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.690592939Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.690626165Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 678cfcb8-a5e0-4920-b486-0e81743672c3 (d9d086f6-80b5-4805-9193-71fbf1abd250) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.690662236Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.690693998Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 678cfcb8-a5e0-4920-b486-0e81743672c3 (d9d086f6-80b5-4805-9193-71fbf1abd250) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.690729204Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.690761435Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 678cfcb8-a5e0-4920-b486-0e81743672c3 (d9d086f6-80b5-4805-9193-71fbf1abd250) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.690805597Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.690838908Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 678cfcb8-a5e0-4920-b486-0e81743672c3 (d9d086f6-80b5-4805-9193-71fbf1abd250) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.690874539Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.690906294Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 678cfcb8-a5e0-4920-b486-0e81743672c3 (d9d086f6-80b5-4805-9193-71fbf1abd250) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.690941376Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.690972969Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 678cfcb8-a5e0-4920-b486-0e81743672c3 (d9d086f6-80b5-4805-9193-71fbf1abd250) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.691008187Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.691039807Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 678cfcb8-a5e0-4920-b486-0e81743672c3 (d9d086f6-80b5-4805-9193-71fbf1abd250) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.691075542Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.691107183Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 678cfcb8-a5e0-4920-b486-0e81743672c3 (d9d086f6-80b5-4805-9193-71fbf1abd250) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.691142666Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.691174293Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"678cfcb8-a5e0-4920-b486-0e81743672c3 is now active with session: d9d086f6-80b5-4805-9193-71fbf1abd250","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.691208341Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 678cfcb8-a5e0-4920-b486-0e81743672c3 (d9d086f6-80b5-4805-9193-71fbf1abd250) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.691243585Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.69127506Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 678cfcb8-a5e0-4920-b486-0e81743672c3 (d9d086f6-80b5-4805-9193-71fbf1abd250) Active Faulted Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.691310909Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.691342392Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 678cfcb8-a5e0-4920-b486-0e81743672c3 (d9d086f6-80b5-4805-9193-71fbf1abd250) Active LiveRepairReady Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.691384384Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.691439556Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for Close + ReOpen jobs","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.691471631Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.691504669Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 close id:1000 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.691536187Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from close command 1000:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:14.691568102Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test volume::test::test_parent_initialized_read_only_region_with_multiple_sub_volumes_1 ... ok
----------------------------------------------------------------
 Crucible gen:0 GIO:true work queues:  Upstairs:2  downstairs:4
GW_ID      ACK  DSID    TYPE  BLOCKS  DS:0  DS:1  DS:2  REPLAY
    1    Acked  1000  FClose       0  Done  Done  Done  false
    2    Acked  1001    NoOp       0  Done  Done  Done  false
    3 NotAcked  1002    NoOp       0   New   New   New  false
    4 NotAcked  1003  Reopen       0   New   New   New  false
   STATES      DS:0   DS:1   DS:2   TOTAL
    New           2      2      2      6
    Sent          0      0      0      0
    Done          2      2      2      6
    Skipped       0      0      0      0
    Error         0      0      0      0
Last Flush: 0 0 0 
Downstairs last five completed:
Upstairs last five completed:      2    1
{"msg":"Now move the NoOp job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:15.444237938Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Finally, move the ReOpen job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:15.444315537Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now ACK the reopen job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:15.44435662Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from reopen command 1003:4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:15.44439534Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"jobs are: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:15.444443823Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible stats registered with UUID: 65a3de01-2768-4988-b51a-36173a07e08d","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:15.444926044Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 65a3de01-2768-4988-b51a-36173a07e08d has session id: dbce7671-3ed0-4ad9-a0ef-2beabf4b973c","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:15.444985083Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 65a3de01-2768-4988-b51a-36173a07e08d (178fe1c3-e441-4bd3-8a70-92b9df375450) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:15.44503241Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:15.445075613Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 65a3de01-2768-4988-b51a-36173a07e08d (178fe1c3-e441-4bd3-8a70-92b9df375450) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:15.445119979Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:15.445172293Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 65a3de01-2768-4988-b51a-36173a07e08d (178fe1c3-e441-4bd3-8a70-92b9df375450) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:15.445217065Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:15.445257487Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 65a3de01-2768-4988-b51a-36173a07e08d (178fe1c3-e441-4bd3-8a70-92b9df375450) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:15.445300123Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:15.445340933Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 65a3de01-2768-4988-b51a-36173a07e08d (178fe1c3-e441-4bd3-8a70-92b9df375450) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:15.445383367Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:15.445422101Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 65a3de01-2768-4988-b51a-36173a07e08d (178fe1c3-e441-4bd3-8a70-92b9df375450) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:15.445462847Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:15.445503721Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 65a3de01-2768-4988-b51a-36173a07e08d (178fe1c3-e441-4bd3-8a70-92b9df375450) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:15.445546971Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:15.445587394Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 65a3de01-2768-4988-b51a-36173a07e08d (178fe1c3-e441-4bd3-8a70-92b9df375450) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:15.445631021Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:15.445671566Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 65a3de01-2768-4988-b51a-36173a07e08d (178fe1c3-e441-4bd3-8a70-92b9df375450) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:15.445706974Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:15.445742507Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"65a3de01-2768-4988-b51a-36173a07e08d is now active with session: 178fe1c3-e441-4bd3-8a70-92b9df375450","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:15.445783723Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 65a3de01-2768-4988-b51a-36173a07e08d (178fe1c3-e441-4bd3-8a70-92b9df375450) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:15.445827382Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:15.445872977Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 65a3de01-2768-4988-b51a-36173a07e08d (178fe1c3-e441-4bd3-8a70-92b9df375450) Active Active Faulted ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:15.445911044Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:15.445943634Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 65a3de01-2768-4988-b51a-36173a07e08d (178fe1c3-e441-4bd3-8a70-92b9df375450) Active Active LiveRepairReady ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:15.445979659Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:15.446011871Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for Close + ReOpen jobs","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:15.446043262Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:15.446075707Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 close id:1000 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:15.446107271Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from close command 1000:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:15.446139826Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now move the NoOp job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:15.5738975Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] DS Reports error Err(GenericError(\"bad\")) on job 1002, DownstairsIO { ds_id: JobId(1002), guest_id: 3, work: ExtentLiveNoOp { dependencies: [JobId(1000), JobId(1001)] }, state: ClientData([Done, Done, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:15.573948399Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] Reports error GenericError(\"bad\") on job 1002, DownstairsIO { ds_id: JobId(1002), guest_id: 3, work: ExtentLiveNoOp { dependencies: [JobId(1000), JobId(1001)] }, state: ClientData([Done, Done, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:15.573981194Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] client skip 4 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:15.574001754Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:15.574020902Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] ea9c2828-4faa-4745-9992-cc45fe6bf51b (62b807af-88b8-4100-a47a-2cd55774bf63) LiveRepair Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:15.574042726Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:15.574061092Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Extent 0 close id:1002 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:15.574087829Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] client skip 4 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:15.574107776Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[0] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:15.574126967Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[0] ea9c2828-4faa-4745-9992-cc45fe6bf51b (62b807af-88b8-4100-a47a-2cd55774bf63) LiveRepair Active Faulted ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:15.574148566Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from LiveRepair to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:15.574166647Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from reopen command 1003:4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:15.574184871Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] DS Reports error Err(GenericError(\"bad\")) on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentFlushClose { dependencies: [], extent: 0, flush_number: 0, gen_number: 0, source_downstairs: ClientId(2), repair_downstairs: [ClientId(1)] }, state: ClientData([Done, InProgress, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:15.574627241Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] Reports error GenericError(\"bad\") on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentFlushClose { dependencies: [], extent: 0, flush_number: 0, gen_number: 0, source_downstairs: ClientId(2), repair_downstairs: [ClientId(1)] }, state: ClientData([Done, InProgress, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:15.574700159Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] client skip 2 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:15.574747095Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:15.574788755Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] 4bf3db3d-50a4-4818-86ec-4e619b46b97c (bf820409-3b29-4b81-8028-3c3affe54370) Active LiveRepair Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:15.574828148Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from LiveRepair to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:15.574848501Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now ACK the close job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:15.574865974Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for 3 jobs (currently 2)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:15.57488369Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Extent 0 close id:1000 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:15.574901915Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:15.57492694Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for 3 jobs (currently 2)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:15.578963924Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"No repair needed for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:15.579011516Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:15.579055251Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now ACK the close job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:15.691966078Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for 3 jobs (currently 2)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:15.692014738Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"No repair needed for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:15.692055066Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:15.692096934Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test volume::test::test_parent_initialized_read_only_region_with_multiple_sub_volumes_2 ... ok
test volume::test::test_parent_uninitialized_read_only_region_one_subvolume ... ok
test volume::test::test_parent_uninitialized_read_only_region_with_multiple_sub_volumes_1 ... ok
test volume::test::test_parent_uninitialized_read_only_region_with_multiple_sub_volumes_2 ... ok
test volume::test::test_read_only_parent_for_lba_range ... ok
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 5 < scrub_point 0. Check with your parent
block 6 < scrub_point 0. Check with your parent
block 7 < scrub_point 0. Check with your parent
block 8 < scrub_point 0. Check with your parent
block 9 < scrub_point 0. Check with your parent
block 0+1 <= scrub_point 1. No parent check
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 5 < scrub_point 1. Check with your parent
block 6 < scrub_point 1. Check with your parent
block 7 < scrub_point 1. Check with your parent
block 8 < scrub_point 1. Check with your parent
block 9 < scrub_point 1. Check with your parent
block 0+1 <= scrub_point 2. No parent check
block 1+1 <= scrub_point 2. No parent check
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 5 < scrub_point 2. Check with your parent
block 6 < scrub_point 2. Check with your parent
block 7 < scrub_point 2. Check with your parent
block 8 < scrub_point 2. Check with your parent
block 9 < scrub_point 2. Check with your parent
block 0+1 <= scrub_point 3. No parent check
block 1+1 <= scrub_point 3. No parent check
block 2+1 <= scrub_point 3. No parent check
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 5 < scrub_point 3. Check with your parent
block 6 < scrub_point 3. Check with your parent
block 7 < scrub_point 3. Check with your parent
block 8 < scrub_point 3. Check with your parent
block 9 < scrub_point 3. Check with your parent
block 0+1 <= scrub_point 4. No parent check
block 1+1 <= scrub_point 4. No parent check
block 2+1 <= scrub_point 4. No parent check
block 3+1 <= scrub_point 4. No parent check
block 4 < scrub_point 4. Check with your parent
block 5 < scrub_point 4. Check with your parent
block 6 < scrub_point 4. Check with your parent
block 7 < scrub_point 4. Check with your parent
block 8 < scrub_point 4. Check with your parent
block 9 < scrub_point 4. Check with your parent
block 0+1 <= scrub_point 5. No parent check
block 1+1 <= scrub_point 5. No parent check
block 2+1 <= scrub_point 5. No parent check
block 3+1 <= scrub_point 5. No parent check
block 4+1 <= scrub_point 5. No parent check
block 5 < scrub_point 5. Check with your parent
block 6 < scrub_point 5. Check with your parent
block 7 < scrub_point 5. Check with your parent
block 8 < scrub_point 5. Check with your parent
block 9 < scrub_point 5. Check with your parent
block 0+1 <= scrub_point 6. No parent check
block 1+1 <= scrub_point 6. No parent check
block 2+1 <= scrub_point 6. No parent check
block 3+1 <= scrub_point 6. No parent check
block 4+1 <= scrub_point 6. No parent check
block 5+1 <= scrub_point 6. No parent check
block 6 < scrub_point 6. Check with your parent
block 7 < scrub_point 6. Check with your parent
block 8 < scrub_point 6. Check with your parent
block 9 < scrub_point 6. Check with your parent
block 0+1 <= scrub_point 7. No parent check
block 1+1 <= scrub_point 7. No parent check
block 2+1 <= scrub_point 7. No parent check
block 3+1 <= scrub_point 7. No parent check
block 4+1 <= scrub_point 7. No parent check
block 5+1 <= scrub_point 7. No parent check
block 6+1 <= scrub_point 7. No parent check
block 7 < scrub_point 7. Check with your parent
block 8 < scrub_point 7. Check with your parent
block 9 < scrub_point 7. Check with your parent
block 0+1 <= scrub_point 8. No parent check
block 1+1 <= scrub_point 8. No parent check
block 2+1 <= scrub_point 8. No parent check
block 3+1 <= scrub_point 8. No parent check
block 4+1 <= scrub_point 8. No parent check
block 5+1 <= scrub_point 8. No parent check
block 6+1 <= scrub_point 8. No parent check
block 7+1 <= scrub_point 8. No parent check
block 8 < scrub_point 8. Check with your parent
block 9 < scrub_point 8. Check with your parent
block 0+1 <= scrub_point 9. No parent check
block 1+1 <= scrub_point 9. No parent check
block 2+1 <= scrub_point 9. No parent check
block 3+1 <= scrub_point 9. No parent check
block 4+1 <= scrub_point 9. No parent check
block 5+1 <= scrub_point 9. No parent check
block 6+1 <= scrub_point 9. No parent check
block 7+1 <= scrub_point 9. No parent check
block 8+1 <= scrub_point 9. No parent check
block 9 < scrub_point 9. Check with your parent
block 0+1 <= scrub_point 10. No parent check
block 1+1 <= scrub_point 10. No parent check
block 2+1 <= scrub_point 10. No parent check
block 3+1 <= scrub_point 10. No parent check
block 4+1 <= scrub_point 10. No parent check
block 5+1 <= scrub_point 10. No parent check
block 6+1 <= scrub_point 10. No parent check
block 7+1 <= scrub_point 10. No parent check
block 8+1 <= scrub_point 10. No parent check
block 9+1 <= scrub_point 10. No parent check
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 5 < scrub_point 0. Check with your parent
block 6 < scrub_point 0. Check with your parent
block 7 < scrub_point 0. Check with your parent
block 8 < scrub_point 0. Check with your parent
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 5 < scrub_point 1. Check with your parent
block 6 < scrub_point 1. Check with your parent
block 7 < scrub_point 1. Check with your parent
block 8 < scrub_point 1. Check with your parent
block 0+2 <= scrub_point 2. No parent check
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 5 < scrub_point 2. Check with your parent
block 6 < scrub_point 2. Check with your parent
block 7 < scrub_point 2. Check with your parent
block 8 < scrub_point 2. Check with your parent
block 0+2 <= scrub_point 3. No parent check
block 1+2 <= scrub_point 3. No parent check
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 5 < scrub_point 3. Check with your parent
block 6 < scrub_point 3. Check with your parent
block 7 < scrub_point 3. Check with your parent
block 8 < scrub_point 3. Check with your parent
block 0+2 <= scrub_point 4. No parent check
block 1+2 <= scrub_point 4. No parent check
block 2+2 <= scrub_point 4. No parent check
block 3 < scrub_point 4. Check with your parent
block 4 < scrub_point 4. Check with your parent
block 5 < scrub_point 4. Check with your parent
block 6 < scrub_point 4. Check with your parent
block 7 < scrub_point 4. Check with your parent
block 8 < scrub_point 4. Check with your parent
block 0+2 <= scrub_point 5. No parent check
block 1+2 <= scrub_point 5. No parent check
block 2+2 <= scrub_point 5. No parent check
block 3+2 <= scrub_point 5. No parent check
block 4 < scrub_point 5. Check with your parent
block 5 < scrub_point 5. Check with your parent
block 6 < scrub_point 5. Check with your parent
block 7 < scrub_point 5. Check with your parent
block 8 < scrub_point 5. Check with your parent
block 0+2 <= scrub_point 6. No parent check
block 1+2 <= scrub_point 6. No parent check
block 2+2 <= scrub_point 6. No parent check
block 3+2 <= scrub_point 6. No parent check
block 4+2 <= scrub_point 6. No parent check
block 5 < scrub_point 6. Check with your parent
block 6 < scrub_point 6. Check with your parent
block 7 < scrub_point 6. Check with your parent
block 8 < scrub_point 6. Check with your parent
block 0+2 <= scrub_point 7. No parent check
block 1+2 <= scrub_point 7. No parent check
block 2+2 <= scrub_point 7. No parent check
block 3+2 <= scrub_point 7. No parent check
block 4+2 <= scrub_point 7. No parent check
block 5+2 <= scrub_point 7. No parent check
block 6 < scrub_point 7. Check with your parent
block 7 < scrub_point 7. Check with your parent
block 8 < scrub_point 7. Check with your parent
block 0+2 <= scrub_point 8. No parent check
block 1+2 <= scrub_point 8. No parent check
block 2+2 <= scrub_point 8. No parent check
block 3+2 <= scrub_point 8. No parent check
block 4+2 <= scrub_point 8. No parent check
block 5+2 <= scrub_point 8. No parent check
block 6+2 <= scrub_point 8. No parent check
block 7 < scrub_point 8. Check with your parent
block 8 < scrub_point 8. Check with your parent
block 0+2 <= scrub_point 9. No parent check
block 1+2 <= scrub_point 9. No parent check
block 2+2 <= scrub_point 9. No parent check
block 3+2 <= scrub_point 9. No parent check
block 4+2 <= scrub_point 9. No parent check
block 5+2 <= scrub_point 9. No parent check
block 6+2 <= scrub_point 9. No parent check
block 7+2 <= scrub_point 9. No parent check
block 8 < scrub_point 9. Check with your parent
block 0+2 <= scrub_point 10. No parent check
block 1+2 <= scrub_point 10. No parent check
block 2+2 <= scrub_point 10. No parent check
block 3+2 <= scrub_point 10. No parent check
block 4+2 <= scrub_point 10. No parent check
block 5+2 <= scrub_point 10. No parent check
block 6+2 <= scrub_point 10. No parent check
block 7+2 <= scrub_point 10. No parent check
block 8+2 <= scrub_point 10. No parent check
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 5 < scrub_point 0. Check with your parent
block 6 < scrub_point 0. Check with your parent
block 7 < scrub_point 0. Check with your parent
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 5 < scrub_point 1. Check with your parent
block 6 < scrub_point 1. Check with your parent
block 7 < scrub_point 1. Check with your parent
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 5 < scrub_point 2. Check with your parent
block 6 < scrub_point 2. Check with your parent
block 7 < scrub_point 2. Check with your parent
block 0+3 <= scrub_point 3. No parent check
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 5 < scrub_point 3. Check with your parent
block 6 < scrub_point 3. Check with your parent
block 7 < scrub_point 3. Check with your parent
block 0+3 <= scrub_point 4. No parent check
block 1+3 <= scrub_point 4. No parent check
block 2 < scrub_point 4. Check with your parent
block 3 < scrub_point 4. Check with your parent
block 4 < scrub_point 4. Check with your parent
block 5 < scrub_point 4. Check with your parent
block 6 < scrub_point 4. Check with your parent
block 7 < scrub_point 4. Check with your parent
block 0+3 <= scrub_point 5. No parent check
block 1+3 <= scrub_point 5. No parent check
block 2+3 <= scrub_point 5. No parent check
block 3 < scrub_point 5. Check with your parent
block 4 < scrub_point 5. Check with your parent
block 5 < scrub_point 5. Check with your parent
block 6 < scrub_point 5. Check with your parent
block 7 < scrub_point 5. Check with your parent
block 0+3 <= scrub_point 6. No parent check
block 1+3 <= scrub_point 6. No parent check
block 2+3 <= scrub_point 6. No parent check
block 3+3 <= scrub_point 6. No parent check
block 4 < scrub_point 6. Check with your parent
block 5 < scrub_point 6. Check with your parent
block 6 < scrub_point 6. Check with your parent
block 7 < scrub_point 6. Check with your parent
block 0+3 <= scrub_point 7. No parent check
block 1+3 <= scrub_point 7. No parent check
block 2+3 <= scrub_point 7. No parent check
block 3+3 <= scrub_point 7. No parent check
block 4+3 <= scrub_point 7. No parent check
block 5 < scrub_point 7. Check with your parent
block 6 < scrub_point 7. Check with your parent
block 7 < scrub_point 7. Check with your parent
block 0+3 <= scrub_point 8. No parent check
block 1+3 <= scrub_point 8. No parent check
block 2+3 <= scrub_point 8. No parent check
block 3+3 <= scrub_point 8. No parent check
block 4+3 <= scrub_point 8. No parent check
block 5+3 <= scrub_point 8. No parent check
block 6 < scrub_point 8. Check with your parent
block 7 < scrub_point 8. Check with your parent
block 0+3 <= scrub_point 9. No parent check
block 1+3 <= scrub_point 9. No parent check
block 2+3 <= scrub_point 9. No parent check
block 3+3 <= scrub_point 9. No parent check
block 4+3 <= scrub_point 9. No parent check
block 5+3 <= scrub_point 9. No parent check
block 6+3 <= scrub_point 9. No parent check
block 7 < scrub_point 9. Check with your parent
block 0+3 <= scrub_point 10. No parent check
block 1+3 <= scrub_point 10. No parent check
block 2+3 <= scrub_point 10. No parent check
block 3+3 <= scrub_point 10. No parent check
block 4+3 <= scrub_point 10. No parent check
block 5+3 <= scrub_point 10. No parent check
block 6+3 <= scrub_point 10. No parent check
block 7+3 <= scrub_point 10. No parent check
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 5 < scrub_point 0. Check with your parent
block 6 < scrub_point 0. Check with your parent
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 5 < scrub_point 1. Check with your parent
block 6 < scrub_point 1. Check with your parent
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 5 < scrub_point 2. Check with your parent
block 6 < scrub_point 2. Check with your parent
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 5 < scrub_point 3. Check with your parent
block 6 < scrub_point 3. Check with your parent
block 0+4 <= scrub_point 4. No parent check
block 1 < scrub_point 4. Check with your parent
block 2 < scrub_point 4. Check with your parent
block 3 < scrub_point 4. Check with your parent
block 4 < scrub_point 4. Check with your parent
block 5 < scrub_point 4. Check with your parent
block 6 < scrub_point 4. Check with your parent
block 0+4 <= scrub_point 5. No parent check
block 1+4 <= scrub_point 5. No parent check
block 2 < scrub_point 5. Check with your parent
block 3 < scrub_point 5. Check with your parent
block 4 < scrub_point 5. Check with your parent
block 5 < scrub_point 5. Check with your parent
block 6 < scrub_point 5. Check with your parent
block 0+4 <= scrub_point 6. No parent check
block 1+4 <= scrub_point 6. No parent check
block 2+4 <= scrub_point 6. No parent check
block 3 < scrub_point 6. Check with your parent
block 4 < scrub_point 6. Check with your parent
block 5 < scrub_point 6. Check with your parent
block 6 < scrub_point 6. Check with your parent
block 0+4 <= scrub_point 7. No parent check
block 1+4 <= scrub_point 7. No parent check
block 2+4 <= scrub_point 7. No parent check
block 3+4 <= scrub_point 7. No parent check
block 4 < scrub_point 7. Check with your parent
block 5 < scrub_point 7. Check with your parent
block 6 < scrub_point 7. Check with your parent
block 0+4 <= scrub_point 8. No parent check
block 1+4 <= scrub_point 8. No parent check
block 2+4 <= scrub_point 8. No parent check
block 3+4 <= scrub_point 8. No parent check
block 4+4 <= scrub_point 8. No parent check
block 5 < scrub_point 8. Check with your parent
block 6 < scrub_point 8. Check with your parent
block 0+4 <= scrub_point 9. No parent check
block 1+4 <= scrub_point 9. No parent check
block 2+4 <= scrub_point 9. No parent check
block 3+4 <= scrub_point 9. No parent check
block 4+4 <= scrub_point 9. No parent check
block 5+4 <= scrub_point 9. No parent check
block 6 < scrub_point 9. Check with your parent
block 0+4 <= scrub_point 10. No parent check
block 1+4 <= scrub_point 10. No parent check
block 2+4 <= scrub_point 10. No parent check
block 3+4 <= scrub_point 10. No parent check
block 4+4 <= scrub_point 10. No parent check
block 5+4 <= scrub_point 10. No parent check
block 6+4 <= scrub_point 10. No parent check
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 5 < scrub_point 0. Check with your parent
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 5 < scrub_point 1. Check with your parent
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 5 < scrub_point 2. Check with your parent
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 5 < scrub_point 3. Check with your parent
block 0 < scrub_point 4. Check with your parent
block 1 < scrub_point 4. Check with your parent
block 2 < scrub_point 4. Check with your parent
block 3 < scrub_point 4. Check with your parent
block 4 < scrub_point 4. Check with your parent
block 5 < scrub_point 4. Check with your parent
block 0+5 <= scrub_point 5. No parent check
block 1 < scrub_point 5. Check with your parent
block 2 < scrub_point 5. Check with your parent
block 3 < scrub_point 5. Check with your parent
block 4 < scrub_point 5. Check with your parent
block 5 < scrub_point 5. Check with your parent
block 0+5 <= scrub_point 6. No parent check
block 1+5 <= scrub_point 6. No parent check
block 2 < scrub_point 6. Check with your parent
block 3 < scrub_point 6. Check with your parent
block 4 < scrub_point 6. Check with your parent
block 5 < scrub_point 6. Check with your parent
block 0+5 <= scrub_point 7. No parent check
block 1+5 <= scrub_point 7. No parent check
block 2+5 <= scrub_point 7. No parent check
block 3 < scrub_point 7. Check with your parent
block 4 < scrub_point 7. Check with your parent
block 5 < scrub_point 7. Check with your parent
block 0+5 <= scrub_point 8. No parent check
block 1+5 <= scrub_point 8. No parent check
block 2+5 <= scrub_point 8. No parent check
block 3+5 <= scrub_point 8. No parent check
block 4 < scrub_point 8. Check with your parent
block 5 < scrub_point 8. Check with your parent
block 0+5 <= scrub_point 9. No parent check
block 1+5 <= scrub_point 9. No parent check
block 2+5 <= scrub_point 9. No parent check
block 3+5 <= scrub_point 9. No parent check
block 4+5 <= scrub_point 9. No parent check
block 5 < scrub_point 9. Check with your parent
block 0+5 <= scrub_point 10. No parent check
block 1+5 <= scrub_point 10. No parent check
block 2+5 <= scrub_point 10. No parent check
block 3+5 <= scrub_point 10. No parent check
block 4+5 <= scrub_point 10. No parent check
block 5+5 <= scrub_point 10. No parent check
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 0 < scrub_point 4. Check with your parent
block 1 < scrub_point 4. Check with your parent
block 2 < scrub_point 4. Check with your parent
block 3 < scrub_point 4. Check with your parent
block 4 < scrub_point 4. Check with your parent
block 0 < scrub_point 5. Check with your parent
block 1 < scrub_point 5. Check with your parent
block 2 < scrub_point 5. Check with your parent
block 3 < scrub_point 5. Check with your parent
block 4 < scrub_point 5. Check with your parent
block 0+6 <= scrub_point 6. No parent check
block 1 < scrub_point 6. Check with your parent
block 2 < scrub_point 6. Check with your parent
block 3 < scrub_point 6. Check with your parent
block 4 < scrub_point 6. Check with your parent
block 0+6 <= scrub_point 7. No parent check
block 1+6 <= scrub_point 7. No parent check
block 2 < scrub_point 7. Check with your parent
block 3 < scrub_point 7. Check with your parent
block 4 < scrub_point 7. Check with your parent
block 0+6 <= scrub_point 8. No parent check
block 1+6 <= scrub_point 8. No parent check
block 2+6 <= scrub_point 8. No parent check
block 3 < scrub_point 8. Check with your parent
block 4 < scrub_point 8. Check with your parent
block 0+6 <= scrub_point 9. No parent check
block 1+6 <= scrub_point 9. No parent check
block 2+6 <= scrub_point 9. No parent check
block 3+6 <= scrub_point 9. No parent check
block 4 < scrub_point 9. Check with your parent
block 0+6 <= scrub_point 10. No parent check
block 1+6 <= scrub_point 10. No parent check
block 2+6 <= scrub_point 10. No parent check
block 3+6 <= scrub_point 10. No parent check
block 4+6 <= scrub_point 10. No parent check
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 0 < scrub_point 4. Check with your parent
block 1 < scrub_point 4. Check with your parent
block 2 < scrub_point 4. Check with your parent
block 3 < scrub_point 4. Check with your parent
block 0 < scrub_point 5. Check with your parent
block 1 < scrub_point 5. Check with your parent
block 2 < scrub_point 5. Check with your parent
block 3 < scrub_point 5. Check with your parent
block 0 < scrub_point 6. Check with your parent
block 1 < scrub_point 6. Check with your parent
block 2 < scrub_point 6. Check with your parent
block 3 < scrub_point 6. Check with your parent
block 0+7 <= scrub_point 7. No parent check
block 1 < scrub_point 7. Check with your parent
block 2 < scrub_point 7. Check with your parent
block 3 < scrub_point 7. Check with your parent
block 0+7 <= scrub_point 8. No parent check
block 1+7 <= scrub_point 8. No parent check
block 2 < scrub_point 8. Check with your parent
block 3 < scrub_point 8. Check with your parent
block 0+7 <= scrub_point 9. No parent check
block 1+7 <= scrub_point 9. No parent check
block 2+7 <= scrub_point 9. No parent check
block 3 < scrub_point 9. Check with your parent
block 0+7 <= scrub_point 10. No parent check
block 1+7 <= scrub_point 10. No parent check
block 2+7 <= scrub_point 10. No parent check
block 3+7 <= scrub_point 10. No parent check
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 0 < scrub_point 4. Check with your parent
block 1 < scrub_point 4. Check with your parent
block 2 < scrub_point 4. Check with your parent
block 0 < scrub_point 5. Check with your parent
block 1 < scrub_point 5. Check with your parent
block 2 < scrub_point 5. Check with your parent
block 0 < scrub_point 6. Check with your parent
block 1 < scrub_point 6. Check with your parent
block 2 < scrub_point 6. Check with your parent
block 0 < scrub_point 7. Check with your parent
block 1 < scrub_point 7. Check with your parent
block 2 < scrub_point 7. Check with your parent
block 0+8 <= scrub_point 8. No parent check
block 1 < scrub_point 8. Check with your parent
block 2 < scrub_point 8. Check with your parent
block 0+8 <= scrub_point 9. No parent check
block 1+8 <= scrub_point 9. No parent check
block 2 < scrub_point 9. Check with your parent
block 0+8 <= scrub_point 10. No parent check
block 1+8 <= scrub_point 10. No parent check
block 2+8 <= scrub_point 10. No parent check
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 0 < scrub_point 4. Check with your parent
block 1 < scrub_point 4. Check with your parent
block 0 < scrub_point 5. Check with your parent
block 1 < scrub_point 5. Check with your parent
block 0 < scrub_point 6. Check with your parent
block 1 < scrub_point 6. Check with your parent
block 0 < scrub_point 7. Check with your parent
block 1 < scrub_point 7. Check with your parent
block 0 < scrub_point 8. Check with your parent
block 1 < scrub_point 8. Check with your parent
block 0+9 <= scrub_point 9. No parent check
block 1 < scrub_point 9. Check with your parent
block 0+9 <= scrub_point 10. No parent check
block 1+9 <= scrub_point 10. No parent check
test volume::test::test_scrub_point_subvolume_equal ... ok
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 0+1 <= scrub_point 1. No parent check
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 0+1 <= scrub_point 2. No parent check
block 1+1 <= scrub_point 2. No parent check
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 0+1 <= scrub_point 3. No parent check
block 1+1 <= scrub_point 3. No parent check
block 2+1 <= scrub_point 3. No parent check
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 0+1 <= scrub_point 4. No parent check
block 1+1 <= scrub_point 4. No parent check
block 2+1 <= scrub_point 4. No parent check
block 3+1 <= scrub_point 4. No parent check
block 4 < scrub_point 4. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 0+1 <= scrub_point 5. No parent check
block 1+1 <= scrub_point 5. No parent check
block 2+1 <= scrub_point 5. No parent check
block 3+1 <= scrub_point 5. No parent check
block 4+1 <= scrub_point 5. No parent check
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 0+2 <= scrub_point 2. No parent check
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 0+2 <= scrub_point 3. No parent check
block 1+2 <= scrub_point 3. No parent check
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 0+2 <= scrub_point 4. No parent check
block 1+2 <= scrub_point 4. No parent check
block 2+2 <= scrub_point 4. No parent check
block 3 < scrub_point 4. Check with your parent
block 4 < scrub_point 4. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 0+2 <= scrub_point 5. No parent check
block 1+2 <= scrub_point 5. No parent check
block 2+2 <= scrub_point 5. No parent check
block 3+2 <= scrub_point 5. No parent check
block 4 < scrub_point 5. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 0+3 <= scrub_point 3. No parent check
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 0+3 <= scrub_point 4. No parent check
block 1+3 <= scrub_point 4. No parent check
block 2 < scrub_point 4. Check with your parent
block 3 < scrub_point 4. Check with your parent
block 4 < scrub_point 4. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 0+3 <= scrub_point 5. No parent check
block 1+3 <= scrub_point 5. No parent check
block 2+3 <= scrub_point 5. No parent check
block 3 < scrub_point 5. Check with your parent
block 4 < scrub_point 5. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 0+4 <= scrub_point 4. No parent check
block 1 < scrub_point 4. Check with your parent
block 2 < scrub_point 4. Check with your parent
block 3 < scrub_point 4. Check with your parent
block 4 < scrub_point 4. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 0+4 <= scrub_point 5. No parent check
block 1+4 <= scrub_point 5. No parent check
block 2 < scrub_point 5. Check with your parent
block 3 < scrub_point 5. Check with your parent
block 4 < scrub_point 5. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 5 > parent 5. Go to SubVolume
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 5 > parent 5. Go to SubVolume
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 5 > parent 5. Go to SubVolume
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 5 > parent 5. Go to SubVolume
block 0 < scrub_point 4. Check with your parent
block 1 < scrub_point 4. Check with your parent
block 2 < scrub_point 4. Check with your parent
block 3 < scrub_point 4. Check with your parent
block 4 < scrub_point 4. Check with your parent
block 5 > parent 5. Go to SubVolume
block 0+5 <= scrub_point 5. No parent check
block 1 < scrub_point 5. Check with your parent
block 2 < scrub_point 5. Check with your parent
block 3 < scrub_point 5. Check with your parent
block 4 < scrub_point 5. Check with your parent
block 5 > parent 5. Go to SubVolume
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 0 < scrub_point 4. Check with your parent
block 1 < scrub_point 4. Check with your parent
block 2 < scrub_point 4. Check with your parent
block 3 < scrub_point 4. Check with your parent
block 4 < scrub_point 4. Check with your parent
block 0 < scrub_point 5. Check with your parent
block 1 < scrub_point 5. Check with your parent
block 2 < scrub_point 5. Check with your parent
block 3 < scrub_point 5. Check with your parent
block 4 < scrub_point 5. Check with your parent
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 0 < scrub_point 4. Check with your parent
block 1 < scrub_point 4. Check with your parent
block 2 < scrub_point 4. Check with your parent
block 3 < scrub_point 4. Check with your parent
block 0 < scrub_point 5. Check with your parent
block 1 < scrub_point 5. Check with your parent
block 2 < scrub_point 5. Check with your parent
block 3 < scrub_point 5. Check with your parent
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 0 < scrub_point 4. Check with your parent
block 1 < scrub_point 4. Check with your parent
block 2 < scrub_point 4. Check with your parent
block 0 < scrub_point 5. Check with your parent
block 1 < scrub_point 5. Check with your parent
block 2 < scrub_point 5. Check with your parent
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 0 < scrub_point 4. Check with your parent
block 1 < scrub_point 4. Check with your parent
block 0 < scrub_point 5. Check with your parent
block 1 < scrub_point 5. Check with your parent
test volume::test::test_scrub_point_subvolume_smaller ... ok
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 5 < scrub_point 0. Check with your parent
block 6 < scrub_point 0. Check with your parent
block 7 < scrub_point 0. Check with your parent
block 8 > parent 8. Go to SubVolume
block 9 > parent 8. Go to SubVolume
block 0+1 <= scrub_point 1. No parent check
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 5 < scrub_point 1. Check with your parent
block 6 < scrub_point 1. Check with your parent
block 7 < scrub_point 1. Check with your parent
block 8 > parent 8. Go to SubVolume
block 9 > parent 8. Go to SubVolume
block 0+1 <= scrub_point 2. No parent check
block 1+1 <= scrub_point 2. No parent check
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 5 < scrub_point 2. Check with your parent
block 6 < scrub_point 2. Check with your parent
block 7 < scrub_point 2. Check with your parent
block 8 > parent 8. Go to SubVolume
block 9 > parent 8. Go to SubVolume
block 0+1 <= scrub_point 3. No parent check
block 1+1 <= scrub_point 3. No parent check
block 2+1 <= scrub_point 3. No parent check
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 5 < scrub_point 3. Check with your parent
block 6 < scrub_point 3. Check with your parent
block 7 < scrub_point 3. Check with your parent
block 8 > parent 8. Go to SubVolume
block 9 > parent 8. Go to SubVolume
block 0+1 <= scrub_point 4. No parent check
block 1+1 <= scrub_point 4. No parent check
block 2+1 <= scrub_point 4. No parent check
block 3+1 <= scrub_point 4. No parent check
block 4 < scrub_point 4. Check with your parent
block 5 < scrub_point 4. Check with your parent
block 6 < scrub_point 4. Check with your parent
block 7 < scrub_point 4. Check with your parent
block 8 > parent 8. Go to SubVolume
block 9 > parent 8. Go to SubVolume
block 0+1 <= scrub_point 5. No parent check
block 1+1 <= scrub_point 5. No parent check
block 2+1 <= scrub_point 5. No parent check
block 3+1 <= scrub_point 5. No parent check
block 4+1 <= scrub_point 5. No parent check
block 5 < scrub_point 5. Check with your parent
block 6 < scrub_point 5. Check with your parent
block 7 < scrub_point 5. Check with your parent
block 8 > parent 8. Go to SubVolume
block 9 > parent 8. Go to SubVolume
block 0+1 <= scrub_point 6. No parent check
block 1+1 <= scrub_point 6. No parent check
block 2+1 <= scrub_point 6. No parent check
block 3+1 <= scrub_point 6. No parent check
block 4+1 <= scrub_point 6. No parent check
block 5+1 <= scrub_point 6. No parent check
block 6 < scrub_point 6. Check with your parent
block 7 < scrub_point 6. Check with your parent
block 8 > parent 8. Go to SubVolume
block 9 > parent 8. Go to SubVolume
block 0+1 <= scrub_point 7. No parent check
block 1+1 <= scrub_point 7. No parent check
block 2+1 <= scrub_point 7. No parent check
block 3+1 <= scrub_point 7. No parent check
block 4+1 <= scrub_point 7. No parent check
block 5+1 <= scrub_point 7. No parent check
block 6+1 <= scrub_point 7. No parent check
block 7 < scrub_point 7. Check with your parent
block 8 > parent 8. Go to SubVolume
block 9 > parent 8. Go to SubVolume
block 0+1 <= scrub_point 8. No parent check
block 1+1 <= scrub_point 8. No parent check
block 2+1 <= scrub_point 8. No parent check
block 3+1 <= scrub_point 8. No parent check
block 4+1 <= scrub_point 8. No parent check
block 5+1 <= scrub_point 8. No parent check
block 6+1 <= scrub_point 8. No parent check
block 7+1 <= scrub_point 8. No parent check
block 8 > parent 8. Go to SubVolume
block 9 > parent 8. Go to SubVolume
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 5 < scrub_point 0. Check with your parent
block 6 < scrub_point 0. Check with your parent
block 7 < scrub_point 0. Check with your parent
block 8 > parent 8. Go to SubVolume
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 5 < scrub_point 1. Check with your parent
block 6 < scrub_point 1. Check with your parent
block 7 < scrub_point 1. Check with your parent
block 8 > parent 8. Go to SubVolume
block 0+2 <= scrub_point 2. No parent check
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 5 < scrub_point 2. Check with your parent
block 6 < scrub_point 2. Check with your parent
block 7 < scrub_point 2. Check with your parent
block 8 > parent 8. Go to SubVolume
block 0+2 <= scrub_point 3. No parent check
block 1+2 <= scrub_point 3. No parent check
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 5 < scrub_point 3. Check with your parent
block 6 < scrub_point 3. Check with your parent
block 7 < scrub_point 3. Check with your parent
block 8 > parent 8. Go to SubVolume
block 0+2 <= scrub_point 4. No parent check
block 1+2 <= scrub_point 4. No parent check
block 2+2 <= scrub_point 4. No parent check
block 3 < scrub_point 4. Check with your parent
block 4 < scrub_point 4. Check with your parent
block 5 < scrub_point 4. Check with your parent
block 6 < scrub_point 4. Check with your parent
block 7 < scrub_point 4. Check with your parent
block 8 > parent 8. Go to SubVolume
block 0+2 <= scrub_point 5. No parent check
block 1+2 <= scrub_point 5. No parent check
block 2+2 <= scrub_point 5. No parent check
block 3+2 <= scrub_point 5. No parent check
block 4 < scrub_point 5. Check with your parent
block 5 < scrub_point 5. Check with your parent
block 6 < scrub_point 5. Check with your parent
block 7 < scrub_point 5. Check with your parent
block 8 > parent 8. Go to SubVolume
block 0+2 <= scrub_point 6. No parent check
block 1+2 <= scrub_point 6. No parent check
block 2+2 <= scrub_point 6. No parent check
block 3+2 <= scrub_point 6. No parent check
block 4+2 <= scrub_point 6. No parent check
block 5 < scrub_point 6. Check with your parent
block 6 < scrub_point 6. Check with your parent
block 7 < scrub_point 6. Check with your parent
block 8 > parent 8. Go to SubVolume
block 0+2 <= scrub_point 7. No parent check
block 1+2 <= scrub_point 7. No parent check
block 2+2 <= scrub_point 7. No parent check
block 3+2 <= scrub_point 7. No parent check
block 4+2 <= scrub_point 7. No parent check
block 5+2 <= scrub_point 7. No parent check
block 6 < scrub_point 7. Check with your parent
block 7 < scrub_point 7. Check with your parent
block 8 > parent 8. Go to SubVolume
block 0+2 <= scrub_point 8. No parent check
block 1+2 <= scrub_point 8. No parent check
block 2+2 <= scrub_point 8. No parent check
block 3+2 <= scrub_point 8. No parent check
block 4+2 <= scrub_point 8. No parent check
block 5+2 <= scrub_point 8. No parent check
block 6+2 <= scrub_point 8. No parent check
block 7 < scrub_point 8. Check with your parent
block 8 > parent 8. Go to SubVolume
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 5 < scrub_point 0. Check with your parent
block 6 < scrub_point 0. Check with your parent
block 7 < scrub_point 0. Check with your parent
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 5 < scrub_point 1. Check with your parent
block 6 < scrub_point 1. Check with your parent
block 7 < scrub_point 1. Check with your parent
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 5 < scrub_point 2. Check with your parent
block 6 < scrub_point 2. Check with your parent
block 7 < scrub_point 2. Check with your parent
block 0+3 <= scrub_point 3. No parent check
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 5 < scrub_point 3. Check with your parent
block 6 < scrub_point 3. Check with your parent
block 7 < scrub_point 3. Check with your parent
block 0+3 <= scrub_point 4. No parent check
block 1+3 <= scrub_point 4. No parent check
block 2 < scrub_point 4. Check with your parent
block 3 < scrub_point 4. Check with your parent
block 4 < scrub_point 4. Check with your parent
block 5 < scrub_point 4. Check with your parent
block 6 < scrub_point 4. Check with your parent
block 7 < scrub_point 4. Check with your parent
block 0+3 <= scrub_point 5. No parent check
block 1+3 <= scrub_point 5. No parent check
block 2+3 <= scrub_point 5. No parent check
block 3 < scrub_point 5. Check with your parent
block 4 < scrub_point 5. Check with your parent
block 5 < scrub_point 5. Check with your parent
block 6 < scrub_point 5. Check with your parent
block 7 < scrub_point 5. Check with your parent
block 0+3 <= scrub_point 6. No parent check
block 1+3 <= scrub_point 6. No parent check
block 2+3 <= scrub_point 6. No parent check
block 3+3 <= scrub_point 6. No parent check
block 4 < scrub_point 6. Check with your parent
block 5 < scrub_point 6. Check with your parent
block 6 < scrub_point 6. Check with your parent
block 7 < scrub_point 6. Check with your parent
block 0+3 <= scrub_point 7. No parent check
block 1+3 <= scrub_point 7. No parent check
block 2+3 <= scrub_point 7. No parent check
block 3+3 <= scrub_point 7. No parent check
block 4+3 <= scrub_point 7. No parent check
block 5 < scrub_point 7. Check with your parent
block 6 < scrub_point 7. Check with your parent
block 7 < scrub_point 7. Check with your parent
block 0+3 <= scrub_point 8. No parent check
block 1+3 <= scrub_point 8. No parent check
block 2+3 <= scrub_point 8. No parent check
block 3+3 <= scrub_point 8. No parent check
block 4+3 <= scrub_point 8. No parent check
block 5+3 <= scrub_point 8. No parent check
block 6 < scrub_point 8. Check with your parent
block 7 < scrub_point 8. Check with your parent
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 5 < scrub_point 0. Check with your parent
block 6 < scrub_point 0. Check with your parent
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 5 < scrub_point 1. Check with your parent
block 6 < scrub_point 1. Check with your parent
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 5 < scrub_point 2. Check with your parent
block 6 < scrub_point 2. Check with your parent
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 5 < scrub_point 3. Check with your parent
block 6 < scrub_point 3. Check with your parent
block 0+4 <= scrub_point 4. No parent check
block 1 < scrub_point 4. Check with your parent
block 2 < scrub_point 4. Check with your parent
block 3 < scrub_point 4. Check with your parent
block 4 < scrub_point 4. Check with your parent
block 5 < scrub_point 4. Check with your parent
block 6 < scrub_point 4. Check with your parent
block 0+4 <= scrub_point 5. No parent check
block 1+4 <= scrub_point 5. No parent check
block 2 < scrub_point 5. Check with your parent
block 3 < scrub_point 5. Check with your parent
block 4 < scrub_point 5. Check with your parent
block 5 < scrub_point 5. Check with your parent
block 6 < scrub_point 5. Check with your parent
block 0+4 <= scrub_point 6. No parent check
block 1+4 <= scrub_point 6. No parent check
block 2+4 <= scrub_point 6. No parent check
block 3 < scrub_point 6. Check with your parent
block 4 < scrub_point 6. Check with your parent
block 5 < scrub_point 6. Check with your parent
block 6 < scrub_point 6. Check with your parent
block 0+4 <= scrub_point 7. No parent check
block 1+4 <= scrub_point 7. No parent check
block 2+4 <= scrub_point 7. No parent check
block 3+4 <= scrub_point 7. No parent check
block 4 < scrub_point 7. Check with your parent
block 5 < scrub_point 7. Check with your parent
block 6 < scrub_point 7. Check with your parent
block 0+4 <= scrub_point 8. No parent check
block 1+4 <= scrub_point 8. No parent check
block 2+4 <= scrub_point 8. No parent check
block 3+4 <= scrub_point 8. No parent check
block 4+4 <= scrub_point 8. No parent check
block 5 < scrub_point 8. Check with your parent
block 6 < scrub_point 8. Check with your parent
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 5 < scrub_point 0. Check with your parent
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 5 < scrub_point 1. Check with your parent
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 5 < scrub_point 2. Check with your parent
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 5 < scrub_point 3. Check with your parent
block 0 < scrub_point 4. Check with your parent
block 1 < scrub_point 4. Check with your parent
block 2 < scrub_point 4. Check with your parent
block 3 < scrub_point 4. Check with your parent
block 4 < scrub_point 4. Check with your parent
block 5 < scrub_point 4. Check with your parent
block 0+5 <= scrub_point 5. No parent check
block 1 < scrub_point 5. Check with your parent
block 2 < scrub_point 5. Check with your parent
block 3 < scrub_point 5. Check with your parent
block 4 < scrub_point 5. Check with your parent
block 5 < scrub_point 5. Check with your parent
block 0+5 <= scrub_point 6. No parent check
block 1+5 <= scrub_point 6. No parent check
block 2 < scrub_point 6. Check with your parent
block 3 < scrub_point 6. Check with your parent
block 4 < scrub_point 6. Check with your parent
block 5 < scrub_point 6. Check with your parent
block 0+5 <= scrub_point 7. No parent check
block 1+5 <= scrub_point 7. No parent check
block 2+5 <= scrub_point 7. No parent check
block 3 < scrub_point 7. Check with your parent
block 4 < scrub_point 7. Check with your parent
block 5 < scrub_point 7. Check with your parent
block 0+5 <= scrub_point 8. No parent check
block 1+5 <= scrub_point 8. No parent check
block 2+5 <= scrub_point 8. No parent check
block 3+5 <= scrub_point 8. No parent check
block 4 < scrub_point 8. Check with your parent
block 5 < scrub_point 8. Check with your parent
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 0 < scrub_point 4. Check with your parent
block 1 < scrub_point 4. Check with your parent
block 2 < scrub_point 4. Check with your parent
block 3 < scrub_point 4. Check with your parent
block 4 < scrub_point 4. Check with your parent
block 0 < scrub_point 5. Check with your parent
block 1 < scrub_point 5. Check with your parent
block 2 < scrub_point 5. Check with your parent
block 3 < scrub_point 5. Check with your parent
block 4 < scrub_point 5. Check with your parent
block 0+6 <= scrub_point 6. No parent check
block 1 < scrub_point 6. Check with your parent
block 2 < scrub_point 6. Check with your parent
block 3 < scrub_point 6. Check with your parent
block 4 < scrub_point 6. Check with your parent
block 0+6 <= scrub_point 7. No parent check
block 1+6 <= scrub_point 7. No parent check
block 2 < scrub_point 7. Check with your parent
block 3 < scrub_point 7. Check with your parent
block 4 < scrub_point 7. Check with your parent
block 0+6 <= scrub_point 8. No parent check
block 1+6 <= scrub_point 8. No parent check
block 2+6 <= scrub_point 8. No parent check
block 3 < scrub_point 8. Check with your parent
block 4 < scrub_point 8. Check with your parent
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 0 < scrub_point 4. Check with your parent
block 1 < scrub_point 4. Check with your parent
block 2 < scrub_point 4. Check with your parent
block 3 < scrub_point 4. Check with your parent
block 0 < scrub_point 5. Check with your parent
block 1 < scrub_point 5. Check with your parent
block 2 < scrub_point 5. Check with your parent
block 3 < scrub_point 5. Check with your parent
block 0 < scrub_point 6. Check with your parent
block 1 < scrub_point 6. Check with your parent
block 2 < scrub_point 6. Check with your parent
block 3 < scrub_point 6. Check with your parent
block 0+7 <= scrub_point 7. No parent check
block 1 < scrub_point 7. Check with your parent
block 2 < scrub_point 7. Check with your parent
block 3 < scrub_point 7. Check with your parent
block 0+7 <= scrub_point 8. No parent check
block 1+7 <= scrub_point 8. No parent check
block 2 < scrub_point 8. Check with your parent
block 3 < scrub_point 8. Check with your parent
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 0 < scrub_point 4. Check with your parent
block 1 < scrub_point 4. Check with your parent
block 2 < scrub_point 4. Check with your parent
block 0 < scrub_point 5. Check with your parent
block 1 < scrub_point 5. Check with your parent
block 2 < scrub_point 5. Check with your parent
block 0 < scrub_point 6. Check with your parent
block 1 < scrub_point 6. Check with your parent
block 2 < scrub_point 6. Check with your parent
block 0 < scrub_point 7. Check with your parent
block 1 < scrub_point 7. Check with your parent
block 2 < scrub_point 7. Check with your parent
block 0+8 <= scrub_point 8. No parent check
block 1 < scrub_point 8. Check with your parent
block 2 < scrub_point 8. Check with your parent
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 0 < scrub_point 4. Check with your parent
block 1 < scrub_point 4. Check with your parent
block 0 < scrub_point 5. Check with your parent
block 1 < scrub_point 5. Check with your parent
block 0 < scrub_point 6. Check with your parent
block 1 < scrub_point 6. Check with your parent
block 0 < scrub_point 7. Check with your parent
block 1 < scrub_point 7. Check with your parent
block 0 < scrub_point 8. Check with your parent
block 1 < scrub_point 8. Check with your parent
test volume::test::test_scrub_point_two_subvolume_equal ... ok
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 10 > parent 5. Go to SubVolume
block 11 > parent 5. Go to SubVolume
block 12 > parent 5. Go to SubVolume
block 13 > parent 5. Go to SubVolume
block 14 > parent 5. Go to SubVolume
block 0+1 <= scrub_point 1. No parent check
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 10 > parent 5. Go to SubVolume
block 11 > parent 5. Go to SubVolume
block 12 > parent 5. Go to SubVolume
block 13 > parent 5. Go to SubVolume
block 14 > parent 5. Go to SubVolume
block 0+1 <= scrub_point 2. No parent check
block 1+1 <= scrub_point 2. No parent check
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 10 > parent 5. Go to SubVolume
block 11 > parent 5. Go to SubVolume
block 12 > parent 5. Go to SubVolume
block 13 > parent 5. Go to SubVolume
block 14 > parent 5. Go to SubVolume
block 0+1 <= scrub_point 3. No parent check
block 1+1 <= scrub_point 3. No parent check
block 2+1 <= scrub_point 3. No parent check
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 10 > parent 5. Go to SubVolume
block 11 > parent 5. Go to SubVolume
block 12 > parent 5. Go to SubVolume
block 13 > parent 5. Go to SubVolume
block 14 > parent 5. Go to SubVolume
block 0+1 <= scrub_point 4. No parent check
block 1+1 <= scrub_point 4. No parent check
block 2+1 <= scrub_point 4. No parent check
block 3+1 <= scrub_point 4. No parent check
block 4 < scrub_point 4. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 10 > parent 5. Go to SubVolume
block 11 > parent 5. Go to SubVolume
block 12 > parent 5. Go to SubVolume
block 13 > parent 5. Go to SubVolume
block 14 > parent 5. Go to SubVolume
block 0+1 <= scrub_point 5. No parent check
block 1+1 <= scrub_point 5. No parent check
block 2+1 <= scrub_point 5. No parent check
block 3+1 <= scrub_point 5. No parent check
block 4+1 <= scrub_point 5. No parent check
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 10 > parent 5. Go to SubVolume
block 11 > parent 5. Go to SubVolume
block 12 > parent 5. Go to SubVolume
block 13 > parent 5. Go to SubVolume
block 14 > parent 5. Go to SubVolume
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 10 > parent 5. Go to SubVolume
block 11 > parent 5. Go to SubVolume
block 12 > parent 5. Go to SubVolume
block 13 > parent 5. Go to SubVolume
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 10 > parent 5. Go to SubVolume
block 11 > parent 5. Go to SubVolume
block 12 > parent 5. Go to SubVolume
block 13 > parent 5. Go to SubVolume
block 0+2 <= scrub_point 2. No parent check
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 10 > parent 5. Go to SubVolume
block 11 > parent 5. Go to SubVolume
block 12 > parent 5. Go to SubVolume
block 13 > parent 5. Go to SubVolume
block 0+2 <= scrub_point 3. No parent check
block 1+2 <= scrub_point 3. No parent check
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 10 > parent 5. Go to SubVolume
block 11 > parent 5. Go to SubVolume
block 12 > parent 5. Go to SubVolume
block 13 > parent 5. Go to SubVolume
block 0+2 <= scrub_point 4. No parent check
block 1+2 <= scrub_point 4. No parent check
block 2+2 <= scrub_point 4. No parent check
block 3 < scrub_point 4. Check with your parent
block 4 < scrub_point 4. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 10 > parent 5. Go to SubVolume
block 11 > parent 5. Go to SubVolume
block 12 > parent 5. Go to SubVolume
block 13 > parent 5. Go to SubVolume
block 0+2 <= scrub_point 5. No parent check
block 1+2 <= scrub_point 5. No parent check
block 2+2 <= scrub_point 5. No parent check
block 3+2 <= scrub_point 5. No parent check
block 4 < scrub_point 5. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 10 > parent 5. Go to SubVolume
block 11 > parent 5. Go to SubVolume
block 12 > parent 5. Go to SubVolume
block 13 > parent 5. Go to SubVolume
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 10 > parent 5. Go to SubVolume
block 11 > parent 5. Go to SubVolume
block 12 > parent 5. Go to SubVolume
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 10 > parent 5. Go to SubVolume
block 11 > parent 5. Go to SubVolume
block 12 > parent 5. Go to SubVolume
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 10 > parent 5. Go to SubVolume
block 11 > parent 5. Go to SubVolume
block 12 > parent 5. Go to SubVolume
block 0+3 <= scrub_point 3. No parent check
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 10 > parent 5. Go to SubVolume
block 11 > parent 5. Go to SubVolume
block 12 > parent 5. Go to SubVolume
block 0+3 <= scrub_point 4. No parent check
block 1+3 <= scrub_point 4. No parent check
block 2 < scrub_point 4. Check with your parent
block 3 < scrub_point 4. Check with your parent
block 4 < scrub_point 4. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 10 > parent 5. Go to SubVolume
block 11 > parent 5. Go to SubVolume
block 12 > parent 5. Go to SubVolume
block 0+3 <= scrub_point 5. No parent check
block 1+3 <= scrub_point 5. No parent check
block 2+3 <= scrub_point 5. No parent check
block 3 < scrub_point 5. Check with your parent
block 4 < scrub_point 5. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 10 > parent 5. Go to SubVolume
block 11 > parent 5. Go to SubVolume
block 12 > parent 5. Go to SubVolume
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 10 > parent 5. Go to SubVolume
block 11 > parent 5. Go to SubVolume
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 10 > parent 5. Go to SubVolume
block 11 > parent 5. Go to SubVolume
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 10 > parent 5. Go to SubVolume
block 11 > parent 5. Go to SubVolume
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 10 > parent 5. Go to SubVolume
block 11 > parent 5. Go to SubVolume
block 0+4 <= scrub_point 4. No parent check
block 1 < scrub_point 4. Check with your parent
block 2 < scrub_point 4. Check with your parent
block 3 < scrub_point 4. Check with your parent
block 4 < scrub_point 4. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 10 > parent 5. Go to SubVolume
block 11 > parent 5. Go to SubVolume
block 0+4 <= scrub_point 5. No parent check
block 1+4 <= scrub_point 5. No parent check
block 2 < scrub_point 5. Check with your parent
block 3 < scrub_point 5. Check with your parent
block 4 < scrub_point 5. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 10 > parent 5. Go to SubVolume
block 11 > parent 5. Go to SubVolume
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 10 > parent 5. Go to SubVolume
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 10 > parent 5. Go to SubVolume
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 10 > parent 5. Go to SubVolume
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 10 > parent 5. Go to SubVolume
block 0 < scrub_point 4. Check with your parent
block 1 < scrub_point 4. Check with your parent
block 2 < scrub_point 4. Check with your parent
block 3 < scrub_point 4. Check with your parent
block 4 < scrub_point 4. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 10 > parent 5. Go to SubVolume
block 0+5 <= scrub_point 5. No parent check
block 1 < scrub_point 5. Check with your parent
block 2 < scrub_point 5. Check with your parent
block 3 < scrub_point 5. Check with your parent
block 4 < scrub_point 5. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 10 > parent 5. Go to SubVolume
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 0 < scrub_point 4. Check with your parent
block 1 < scrub_point 4. Check with your parent
block 2 < scrub_point 4. Check with your parent
block 3 < scrub_point 4. Check with your parent
block 4 < scrub_point 4. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 0 < scrub_point 5. Check with your parent
block 1 < scrub_point 5. Check with your parent
block 2 < scrub_point 5. Check with your parent
block 3 < scrub_point 5. Check with your parent
block 4 < scrub_point 5. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 0 < scrub_point 4. Check with your parent
block 1 < scrub_point 4. Check with your parent
block 2 < scrub_point 4. Check with your parent
block 3 < scrub_point 4. Check with your parent
block 4 < scrub_point 4. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 0 < scrub_point 5. Check with your parent
block 1 < scrub_point 5. Check with your parent
block 2 < scrub_point 5. Check with your parent
block 3 < scrub_point 5. Check with your parent
block 4 < scrub_point 5. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 0 < scrub_point 4. Check with your parent
block 1 < scrub_point 4. Check with your parent
block 2 < scrub_point 4. Check with your parent
block 3 < scrub_point 4. Check with your parent
block 4 < scrub_point 4. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 0 < scrub_point 5. Check with your parent
block 1 < scrub_point 5. Check with your parent
block 2 < scrub_point 5. Check with your parent
block 3 < scrub_point 5. Check with your parent
block 4 < scrub_point 5. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 0 < scrub_point 4. Check with your parent
block 1 < scrub_point 4. Check with your parent
block 2 < scrub_point 4. Check with your parent
block 3 < scrub_point 4. Check with your parent
block 4 < scrub_point 4. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 0 < scrub_point 5. Check with your parent
block 1 < scrub_point 5. Check with your parent
block 2 < scrub_point 5. Check with your parent
block 3 < scrub_point 5. Check with your parent
block 4 < scrub_point 5. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 5 > parent 5. Go to SubVolume
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 5 > parent 5. Go to SubVolume
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 5 > parent 5. Go to SubVolume
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 5 > parent 5. Go to SubVolume
block 0 < scrub_point 4. Check with your parent
block 1 < scrub_point 4. Check with your parent
block 2 < scrub_point 4. Check with your parent
block 3 < scrub_point 4. Check with your parent
block 4 < scrub_point 4. Check with your parent
block 5 > parent 5. Go to SubVolume
block 0 < scrub_point 5. Check with your parent
block 1 < scrub_point 5. Check with your parent
block 2 < scrub_point 5. Check with your parent
block 3 < scrub_point 5. Check with your parent
block 4 < scrub_point 5. Check with your parent
block 5 > parent 5. Go to SubVolume
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 0 < scrub_point 4. Check with your parent
block 1 < scrub_point 4. Check with your parent
block 2 < scrub_point 4. Check with your parent
block 3 < scrub_point 4. Check with your parent
block 4 < scrub_point 4. Check with your parent
block 0 < scrub_point 5. Check with your parent
block 1 < scrub_point 5. Check with your parent
block 2 < scrub_point 5. Check with your parent
block 3 < scrub_point 5. Check with your parent
block 4 < scrub_point 5. Check with your parent
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 0 < scrub_point 4. Check with your parent
block 1 < scrub_point 4. Check with your parent
block 2 < scrub_point 4. Check with your parent
block 3 < scrub_point 4. Check with your parent
block 0 < scrub_point 5. Check with your parent
block 1 < scrub_point 5. Check with your parent
block 2 < scrub_point 5. Check with your parent
block 3 < scrub_point 5. Check with your parent
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 0 < scrub_point 4. Check with your parent
block 1 < scrub_point 4. Check with your parent
block 2 < scrub_point 4. Check with your parent
block 0 < scrub_point 5. Check with your parent
block 1 < scrub_point 5. Check with your parent
block 2 < scrub_point 5. Check with your parent
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 0 < scrub_point 4. Check with your parent
block 1 < scrub_point 4. Check with your parent
block 0 < scrub_point 5. Check with your parent
block 1 < scrub_point 5. Check with your parent
test volume::test::test_scrub_point_two_subvolume_smaller_1 ... ok
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 0+1 <= scrub_point 1. No parent check
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 0+1 <= scrub_point 2. No parent check
block 1+1 <= scrub_point 2. No parent check
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 0+1 <= scrub_point 3. No parent check
block 1+1 <= scrub_point 3. No parent check
block 2+1 <= scrub_point 3. No parent check
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 0+1 <= scrub_point 4. No parent check
block 1+1 <= scrub_point 4. No parent check
block 2+1 <= scrub_point 4. No parent check
block 3+1 <= scrub_point 4. No parent check
block 4 < scrub_point 4. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 0+1 <= scrub_point 5. No parent check
block 1+1 <= scrub_point 5. No parent check
block 2+1 <= scrub_point 5. No parent check
block 3+1 <= scrub_point 5. No parent check
block 4+1 <= scrub_point 5. No parent check
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 0+2 <= scrub_point 2. No parent check
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 0+2 <= scrub_point 3. No parent check
block 1+2 <= scrub_point 3. No parent check
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 0+2 <= scrub_point 4. No parent check
block 1+2 <= scrub_point 4. No parent check
block 2+2 <= scrub_point 4. No parent check
block 3 < scrub_point 4. Check with your parent
block 4 < scrub_point 4. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 0+2 <= scrub_point 5. No parent check
block 1+2 <= scrub_point 5. No parent check
block 2+2 <= scrub_point 5. No parent check
block 3+2 <= scrub_point 5. No parent check
block 4 < scrub_point 5. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 0+3 <= scrub_point 3. No parent check
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 0+3 <= scrub_point 4. No parent check
block 1+3 <= scrub_point 4. No parent check
block 2 < scrub_point 4. Check with your parent
block 3 < scrub_point 4. Check with your parent
block 4 < scrub_point 4. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 0+3 <= scrub_point 5. No parent check
block 1+3 <= scrub_point 5. No parent check
block 2+3 <= scrub_point 5. No parent check
block 3 < scrub_point 5. Check with your parent
block 4 < scrub_point 5. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 0+4 <= scrub_point 4. No parent check
block 1 < scrub_point 4. Check with your parent
block 2 < scrub_point 4. Check with your parent
block 3 < scrub_point 4. Check with your parent
block 4 < scrub_point 4. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 0+4 <= scrub_point 5. No parent check
block 1+4 <= scrub_point 5. No parent check
block 2 < scrub_point 5. Check with your parent
block 3 < scrub_point 5. Check with your parent
block 4 < scrub_point 5. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 5 > parent 5. Go to SubVolume
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 5 > parent 5. Go to SubVolume
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 5 > parent 5. Go to SubVolume
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 5 > parent 5. Go to SubVolume
block 0 < scrub_point 4. Check with your parent
block 1 < scrub_point 4. Check with your parent
block 2 < scrub_point 4. Check with your parent
block 3 < scrub_point 4. Check with your parent
block 4 < scrub_point 4. Check with your parent
block 5 > parent 5. Go to SubVolume
block 0+5 <= scrub_point 5. No parent check
block 1 < scrub_point 5. Check with your parent
block 2 < scrub_point 5. Check with your parent
block 3 < scrub_point 5. Check with your parent
block 4 < scrub_point 5. Check with your parent
block 5 > parent 5. Go to SubVolume
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 0 < scrub_point 4. Check with your parent
block 1 < scrub_point 4. Check with your parent
block 2 < scrub_point 4. Check with your parent
block 3 < scrub_point 4. Check with your parent
block 4 < scrub_point 4. Check with your parent
block 0 < scrub_point 5. Check with your parent
block 1 < scrub_point 5. Check with your parent
block 2 < scrub_point 5. Check with your parent
block 3 < scrub_point 5. Check with your parent
block 4 < scrub_point 5. Check with your parent
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 0 < scrub_point 4. Check with your parent
block 1 < scrub_point 4. Check with your parent
block 2 < scrub_point 4. Check with your parent
block 3 < scrub_point 4. Check with your parent
block 0 < scrub_point 5. Check with your parent
block 1 < scrub_point 5. Check with your parent
block 2 < scrub_point 5. Check with your parent
block 3 < scrub_point 5. Check with your parent
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 0 < scrub_point 4. Check with your parent
block 1 < scrub_point 4. Check with your parent
block 2 < scrub_point 4. Check with your parent
block 0 < scrub_point 5. Check with your parent
block 1 < scrub_point 5. Check with your parent
block 2 < scrub_point 5. Check with your parent
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 0 < scrub_point 4. Check with your parent
block 1 < scrub_point 4. Check with your parent
block 0 < scrub_point 5. Check with your parent
block 1 < scrub_point 5. Check with your parent
test volume::test::test_scrub_point_two_subvolume_smaller_2 ... ok
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 5 < scrub_point 0. Check with your parent
block 6 < scrub_point 0. Check with your parent
block 7 < scrub_point 0. Check with your parent
block 8 > parent 8. Go to SubVolume
block 9 > parent 8. Go to SubVolume
block 0+1 <= scrub_point 1. No parent check
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 5 < scrub_point 1. Check with your parent
block 6 < scrub_point 1. Check with your parent
block 7 < scrub_point 1. Check with your parent
block 8 > parent 8. Go to SubVolume
block 9 > parent 8. Go to SubVolume
block 0+1 <= scrub_point 2. No parent check
block 1+1 <= scrub_point 2. No parent check
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 5 < scrub_point 2. Check with your parent
block 6 < scrub_point 2. Check with your parent
block 7 < scrub_point 2. Check with your parent
block 8 > parent 8. Go to SubVolume
block 9 > parent 8. Go to SubVolume
block 0+1 <= scrub_point 3. No parent check
block 1+1 <= scrub_point 3. No parent check
block 2+1 <= scrub_point 3. No parent check
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 5 < scrub_point 3. Check with your parent
block 6 < scrub_point 3. Check with your parent
block 7 < scrub_point 3. Check with your parent
block 8 > parent 8. Go to SubVolume
block 9 > parent 8. Go to SubVolume
block 0+1 <= scrub_point 4. No parent check
block 1+1 <= scrub_point 4. No parent check
block 2+1 <= scrub_point 4. No parent check
block 3+1 <= scrub_point 4. No parent check
block 4 < scrub_point 4. Check with your parent
block 5 < scrub_point 4. Check with your parent
block 6 < scrub_point 4. Check with your parent
block 7 < scrub_point 4. Check with your parent
block 8 > parent 8. Go to SubVolume
block 9 > parent 8. Go to SubVolume
block 0+1 <= scrub_point 5. No parent check
block 1+1 <= scrub_point 5. No parent check
block 2+1 <= scrub_point 5. No parent check
block 3+1 <= scrub_point 5. No parent check
block 4+1 <= scrub_point 5. No parent check
block 5 < scrub_point 5. Check with your parent
block 6 < scrub_point 5. Check with your parent
block 7 < scrub_point 5. Check with your parent
block 8 > parent 8. Go to SubVolume
block 9 > parent 8. Go to SubVolume
block 0+1 <= scrub_point 6. No parent check
block 1+1 <= scrub_point 6. No parent check
block 2+1 <= scrub_point 6. No parent check
block 3+1 <= scrub_point 6. No parent check
block 4+1 <= scrub_point 6. No parent check
block 5+1 <= scrub_point 6. No parent check
block 6 < scrub_point 6. Check with your parent
block 7 < scrub_point 6. Check with your parent
block 8 > parent 8. Go to SubVolume
block 9 > parent 8. Go to SubVolume
block 0+1 <= scrub_point 7. No parent check
block 1+1 <= scrub_point 7. No parent check
block 2+1 <= scrub_point 7. No parent check
block 3+1 <= scrub_point 7. No parent check
block 4+1 <= scrub_point 7. No parent check
block 5+1 <= scrub_point 7. No parent check
block 6+1 <= scrub_point 7. No parent check
block 7 < scrub_point 7. Check with your parent
block 8 > parent 8. Go to SubVolume
block 9 > parent 8. Go to SubVolume
block 0+1 <= scrub_point 8. No parent check
block 1+1 <= scrub_point 8. No parent check
block 2+1 <= scrub_point 8. No parent check
block 3+1 <= scrub_point 8. No parent check
block 4+1 <= scrub_point 8. No parent check
block 5+1 <= scrub_point 8. No parent check
block 6+1 <= scrub_point 8. No parent check
block 7+1 <= scrub_point 8. No parent check
block 8 > parent 8. Go to SubVolume
block 9 > parent 8. Go to SubVolume
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 5 < scrub_point 0. Check with your parent
block 6 < scrub_point 0. Check with your parent
block 7 < scrub_point 0. Check with your parent
block 8 > parent 8. Go to SubVolume
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 5 < scrub_point 1. Check with your parent
block 6 < scrub_point 1. Check with your parent
block 7 < scrub_point 1. Check with your parent
block 8 > parent 8. Go to SubVolume
block 0+2 <= scrub_point 2. No parent check
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 5 < scrub_point 2. Check with your parent
block 6 < scrub_point 2. Check with your parent
block 7 < scrub_point 2. Check with your parent
block 8 > parent 8. Go to SubVolume
block 0+2 <= scrub_point 3. No parent check
block 1+2 <= scrub_point 3. No parent check
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 5 < scrub_point 3. Check with your parent
block 6 < scrub_point 3. Check with your parent
block 7 < scrub_point 3. Check with your parent
block 8 > parent 8. Go to SubVolume
block 0+2 <= scrub_point 4. No parent check
block 1+2 <= scrub_point 4. No parent check
block 2+2 <= scrub_point 4. No parent check
block 3 < scrub_point 4. Check with your parent
block 4 < scrub_point 4. Check with your parent
block 5 < scrub_point 4. Check with your parent
block 6 < scrub_point 4. Check with your parent
block 7 < scrub_point 4. Check with your parent
block 8 > parent 8. Go to SubVolume
block 0+2 <= scrub_point 5. No parent check
block 1+2 <= scrub_point 5. No parent check
block 2+2 <= scrub_point 5. No parent check
block 3+2 <= scrub_point 5. No parent check
block 4 < scrub_point 5. Check with your parent
block 5 < scrub_point 5. Check with your parent
block 6 < scrub_point 5. Check with your parent
block 7 < scrub_point 5. Check with your parent
block 8 > parent 8. Go to SubVolume
block 0+2 <= scrub_point 6. No parent check
block 1+2 <= scrub_point 6. No parent check
block 2+2 <= scrub_point 6. No parent check
block 3+2 <= scrub_point 6. No parent check
block 4+2 <= scrub_point 6. No parent check
block 5 < scrub_point 6. Check with your parent
block 6 < scrub_point 6. Check with your parent
block 7 < scrub_point 6. Check with your parent
block 8 > parent 8. Go to SubVolume
block 0+2 <= scrub_point 7. No parent check
block 1+2 <= scrub_point 7. No parent check
block 2+2 <= scrub_point 7. No parent check
block 3+2 <= scrub_point 7. No parent check
block 4+2 <= scrub_point 7. No parent check
block 5+2 <= scrub_point 7. No parent check
block 6 < scrub_point 7. Check with your parent
block 7 < scrub_point 7. Check with your parent
block 8 > parent 8. Go to SubVolume
block 0+2 <= scrub_point 8. No parent check
block 1+2 <= scrub_point 8. No parent check
block 2+2 <= scrub_point 8. No parent check
block 3+2 <= scrub_point 8. No parent check
block 4+2 <= scrub_point 8. No parent check
block 5+2 <= scrub_point 8. No parent check
block 6+2 <= scrub_point 8. No parent check
block 7 < scrub_point 8. Check with your parent
block 8 > parent 8. Go to SubVolume
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 5 < scrub_point 0. Check with your parent
block 6 < scrub_point 0. Check with your parent
block 7 < scrub_point 0. Check with your parent
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 5 < scrub_point 1. Check with your parent
block 6 < scrub_point 1. Check with your parent
block 7 < scrub_point 1. Check with your parent
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 5 < scrub_point 2. Check with your parent
block 6 < scrub_point 2. Check with your parent
block 7 < scrub_point 2. Check with your parent
block 0+3 <= scrub_point 3. No parent check
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 5 < scrub_point 3. Check with your parent
block 6 < scrub_point 3. Check with your parent
block 7 < scrub_point 3. Check with your parent
block 0+3 <= scrub_point 4. No parent check
block 1+3 <= scrub_point 4. No parent check
block 2 < scrub_point 4. Check with your parent
block 3 < scrub_point 4. Check with your parent
block 4 < scrub_point 4. Check with your parent
block 5 < scrub_point 4. Check with your parent
block 6 < scrub_point 4. Check with your parent
block 7 < scrub_point 4. Check with your parent
block 0+3 <= scrub_point 5. No parent check
block 1+3 <= scrub_point 5. No parent check
block 2+3 <= scrub_point 5. No parent check
block 3 < scrub_point 5. Check with your parent
block 4 < scrub_point 5. Check with your parent
block 5 < scrub_point 5. Check with your parent
block 6 < scrub_point 5. Check with your parent
block 7 < scrub_point 5. Check with your parent
block 0+3 <= scrub_point 6. No parent check
block 1+3 <= scrub_point 6. No parent check
block 2+3 <= scrub_point 6. No parent check
block 3+3 <= scrub_point 6. No parent check
block 4 < scrub_point 6. Check with your parent
block 5 < scrub_point 6. Check with your parent
block 6 < scrub_point 6. Check with your parent
block 7 < scrub_point 6. Check with your parent
block 0+3 <= scrub_point 7. No parent check
block 1+3 <= scrub_point 7. No parent check
block 2+3 <= scrub_point 7. No parent check
block 3+3 <= scrub_point 7. No parent check
block 4+3 <= scrub_point 7. No parent check
block 5 < scrub_point 7. Check with your parent
block 6 < scrub_point 7. Check with your parent
block 7 < scrub_point 7. Check with your parent
block 0+3 <= scrub_point 8. No parent check
block 1+3 <= scrub_point 8. No parent check
block 2+3 <= scrub_point 8. No parent check
block 3+3 <= scrub_point 8. No parent check
block 4+3 <= scrub_point 8. No parent check
block 5+3 <= scrub_point 8. No parent check
block 6 < scrub_point 8. Check with your parent
block 7 < scrub_point 8. Check with your parent
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 5 < scrub_point 0. Check with your parent
block 6 < scrub_point 0. Check with your parent
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 5 < scrub_point 1. Check with your parent
block 6 < scrub_point 1. Check with your parent
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 5 < scrub_point 2. Check with your parent
block 6 < scrub_point 2. Check with your parent
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 5 < scrub_point 3. Check with your parent
block 6 < scrub_point 3. Check with your parent
block 0+4 <= scrub_point 4. No parent check
block 1 < scrub_point 4. Check with your parent
block 2 < scrub_point 4. Check with your parent
block 3 < scrub_point 4. Check with your parent
block 4 < scrub_point 4. Check with your parent
block 5 < scrub_point 4. Check with your parent
block 6 < scrub_point 4. Check with your parent
block 0+4 <= scrub_point 5. No parent check
block 1+4 <= scrub_point 5. No parent check
block 2 < scrub_point 5. Check with your parent
block 3 < scrub_point 5. Check with your parent
block 4 < scrub_point 5. Check with your parent
block 5 < scrub_point 5. Check with your parent
block 6 < scrub_point 5. Check with your parent
block 0+4 <= scrub_point 6. No parent check
block 1+4 <= scrub_point 6. No parent check
block 2+4 <= scrub_point 6. No parent check
block 3 < scrub_point 6. Check with your parent
block 4 < scrub_point 6. Check with your parent
block 5 < scrub_point 6. Check with your parent
block 6 < scrub_point 6. Check with your parent
block 0+4 <= scrub_point 7. No parent check
block 1+4 <= scrub_point 7. No parent check
block 2+4 <= scrub_point 7. No parent check
block 3+4 <= scrub_point 7. No parent check
block 4 < scrub_point 7. Check with your parent
block 5 < scrub_point 7. Check with your parent
block 6 < scrub_point 7. Check with your parent
block 0+4 <= scrub_point 8. No parent check
block 1+4 <= scrub_point 8. No parent check
block 2+4 <= scrub_point 8. No parent check
block 3+4 <= scrub_point 8. No parent check
block 4+4 <= scrub_point 8. No parent check
block 5 < scrub_point 8. Check with your parent
block 6 < scrub_point 8. Check with your parent
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 5 < scrub_point 0. Check with your parent
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 5 < scrub_point 1. Check with your parent
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 5 < scrub_point 2. Check with your parent
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 5 < scrub_point 3. Check with your parent
block 0 < scrub_point 4. Check with your parent
block 1 < scrub_point 4. Check with your parent
block 2 < scrub_point 4. Check with your parent
block 3 < scrub_point 4. Check with your parent
block 4 < scrub_point 4. Check with your parent
block 5 < scrub_point 4. Check with your parent
block 0+5 <= scrub_point 5. No parent check
block 1 < scrub_point 5. Check with your parent
block 2 < scrub_point 5. Check with your parent
block 3 < scrub_point 5. Check with your parent
block 4 < scrub_point 5. Check with your parent
block 5 < scrub_point 5. Check with your parent
block 0+5 <= scrub_point 6. No parent check
block 1+5 <= scrub_point 6. No parent check
block 2 < scrub_point 6. Check with your parent
block 3 < scrub_point 6. Check with your parent
block 4 < scrub_point 6. Check with your parent
block 5 < scrub_point 6. Check with your parent
block 0+5 <= scrub_point 7. No parent check
block 1+5 <= scrub_point 7. No parent check
block 2+5 <= scrub_point 7. No parent check
block 3 < scrub_point 7. Check with your parent
block 4 < scrub_point 7. Check with your parent
block 5 < scrub_point 7. Check with your parent
block 0+5 <= scrub_point 8. No parent check
block 1+5 <= scrub_point 8. No parent check
block 2+5 <= scrub_point 8. No parent check
block 3+5 <= scrub_point 8. No parent check
block 4 < scrub_point 8. Check with your parent
block 5 < scrub_point 8. Check with your parent
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 0 < scrub_point 4. Check with your parent
block 1 < scrub_point 4. Check with your parent
block 2 < scrub_point 4. Check with your parent
block 3 < scrub_point 4. Check with your parent
block 4 < scrub_point 4. Check with your parent
block 0 < scrub_point 5. Check with your parent
block 1 < scrub_point 5. Check with your parent
block 2 < scrub_point 5. Check with your parent
block 3 < scrub_point 5. Check with your parent
block 4 < scrub_point 5. Check with your parent
block 0+6 <= scrub_point 6. No parent check
block 1 < scrub_point 6. Check with your parent
block 2 < scrub_point 6. Check with your parent
block 3 < scrub_point 6. Check with your parent
block 4 < scrub_point 6. Check with your parent
block 0+6 <= scrub_point 7. No parent check
block 1+6 <= scrub_point 7. No parent check
block 2 < scrub_point 7. Check with your parent
block 3 < scrub_point 7. Check with your parent
block 4 < scrub_point 7. Check with your parent
block 0+6 <= scrub_point 8. No parent check
block 1+6 <= scrub_point 8. No parent check
block 2+6 <= scrub_point 8. No parent check
block 3 < scrub_point 8. Check with your parent
block 4 < scrub_point 8. Check with your parent
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 0 < scrub_point 4. Check with your parent
block 1 < scrub_point 4. Check with your parent
block 2 < scrub_point 4. Check with your parent
block 3 < scrub_point 4. Check with your parent
block 0 < scrub_point 5. Check with your parent
block 1 < scrub_point 5. Check with your parent
block 2 < scrub_point 5. Check with your parent
block 3 < scrub_point 5. Check with your parent
block 0 < scrub_point 6. Check with your parent
block 1 < scrub_point 6. Check with your parent
block 2 < scrub_point 6. Check with your parent
block 3 < scrub_point 6. Check with your parent
block 0+7 <= scrub_point 7. No parent check
block 1 < scrub_point 7. Check with your parent
block 2 < scrub_point 7. Check with your parent
block 3 < scrub_point 7. Check with your parent
block 0+7 <= scrub_point 8. No parent check
block 1+7 <= scrub_point 8. No parent check
block 2 < scrub_point 8. Check with your parent
block 3 < scrub_point 8. Check with your parent
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 0 < scrub_point 4. Check with your parent
block 1 < scrub_point 4. Check with your parent
block 2 < scrub_point 4. Check with your parent
block 0 < scrub_point 5. Check with your parent
block 1 < scrub_point 5. Check with your parent
block 2 < scrub_point 5. Check with your parent
block 0 < scrub_point 6. Check with your parent
block 1 < scrub_point 6. Check with your parent
block 2 < scrub_point 6. Check with your parent
block 0 < scrub_point 7. Check with your parent
block 1 < scrub_point 7. Check with your parent
block 2 < scrub_point 7. Check with your parent
block 0+8 <= scrub_point 8. No parent check
block 1 < scrub_point 8. Check with your parent
block 2 < scrub_point 8. Check with your parent
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 0 < scrub_point 4. Check with your parent
block 1 < scrub_point 4. Check with your parent
block 0 < scrub_point 5. Check with your parent
block 1 < scrub_point 5. Check with your parent
block 0 < scrub_point 6. Check with your parent
block 1 < scrub_point 6. Check with your parent
block 0 < scrub_point 7. Check with your parent
block 1 < scrub_point 7. Check with your parent
block 0 < scrub_point 8. Check with your parent
block 1 < scrub_point 8. Check with your parent
test volume::test::test_scrub_point_two_subvolume_smaller_3 ... ok
test volume::test::test_single_block ... ok
test volume::test::test_single_sub_volume_lba_coverage ... ok
test volume::test::test_single_sub_volume_lba_coverage_with_offset ... ok
test volume::test::test_three_layers ... ok
test volume::test::test_volume_size ... ok
test volume::test::test_volume_with_only_read_only_parent ... ok
test volume::test::test_write_unwritten_to_volume_with_only_read_only_parent ... ok
test volume::test::test_writing_to_volume_with_only_read_only_parent ... ok
Sep 22 23:08:16.243 INFO Test replacement of CID 0
Sep 22 23:08:16.243 INFO replace 127.0.0.1:5555 with 127.0.0.1:8888
Sep 22 23:08:16.243 INFO Test replacement of CID 1
Sep 22 23:08:16.243 INFO replace 127.0.0.1:6666 with 127.0.0.1:8888
Sep 22 23:08:16.243 INFO Test replacement of CID 2
Sep 22 23:08:16.243 INFO replace 127.0.0.1:7777 with 127.0.0.1:8888
test volume::test::volume_replace_basic ... ok
test volume::test::volume_replace_drop_rop ... ok
test volume::test::volume_replace_mismatch_opts_cert_pem ... ok
test volume::test::volume_replace_mismatch_opts_control ... ok
test volume::test::volume_replace_mismatch_opts_flush_timeout ... ok
test volume::test::volume_replace_mismatch_opts_id ... ok
test volume::test::volume_replace_mismatch_opts_key ... ok
test volume::test::volume_replace_mismatch_opts_key_pem ... ok
test volume::test::volume_replace_mismatch_opts_lossy ... ok
test volume::test::volume_replace_mismatch_opts_read_only ... ok
test volume::test::volume_replace_mismatch_opts_root_cert ... ok
test volume::test::volume_replace_mismatch_sv_bpe ... ok
test volume::test::volume_replace_mismatch_sv_bs ... ok
test volume::test::volume_replace_mismatch_sv_ec ... ok
test volume::test::volume_replace_mismatch_vblock ... ok
test volume::test::volume_replace_mismatch_vid ... ok
test volume::test::volume_replace_mismatch_vrop ... ok
test volume::test::volume_replace_rop ... ok
test volume::test::volume_replace_self ... ok
test volume::test::volume_vcr_no_target ... ok
{"msg":"Waiting for 3 jobs (currently 2)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:16.44651369Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"No repair needed for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:16.446558924Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:16.446581728Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:16.576110859Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Extent 0 close id:1001 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:16.576164542Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:16.576199011Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:16.580324988Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:16.580363259Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] DS Reports error Err(GenericError(\"bad\")) on job 1001, DownstairsIO { ds_id: JobId(1001), guest_id: 2, work: ExtentLiveNoOp { dependencies: [JobId(1000)] }, state: ClientData([InProgress, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:16.693386981Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[0] Reports error GenericError(\"bad\") on job 1001, DownstairsIO { ds_id: JobId(1001), guest_id: 2, work: ExtentLiveNoOp { dependencies: [JobId(1000)] }, state: ClientData([InProgress, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:16.693459007Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[0] client skip 3 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:16.693504306Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[0] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:16.693546092Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[0] 678cfcb8-a5e0-4920-b486-0e81743672c3 (d9d086f6-80b5-4805-9193-71fbf1abd250) Active LiveRepair Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:16.693591277Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:16.693634726Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:16.693657376Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Extent 0 close id:1001 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:16.693675519Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] client skip 3 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:16.69369364Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:16.693712995Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] 678cfcb8-a5e0-4920-b486-0e81743672c3 (d9d086f6-80b5-4805-9193-71fbf1abd250) Faulted LiveRepair Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:16.693734407Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from LiveRepair to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:16.693752271Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:16.693770239Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.447877777Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.447935081Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Finally, move the ReOpen job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.575359415Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now ACK the reopen job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.575416966Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Bailing with error","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:17.57545918Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"err:2 or:0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.575556804Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible stats registered with UUID: 0c47db24-e385-42f3-95d6-fa49ef7706df","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.57600123Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 0c47db24-e385-42f3-95d6-fa49ef7706df has session id: 88c0379c-a848-42d6-8062-1cd30b9aef3c","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.57603551Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 0c47db24-e385-42f3-95d6-fa49ef7706df (927b0333-54f5-4011-a1f3-3f18a06b5db1) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.576057828Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.576076617Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 0c47db24-e385-42f3-95d6-fa49ef7706df (927b0333-54f5-4011-a1f3-3f18a06b5db1) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.576097526Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.576127231Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 0c47db24-e385-42f3-95d6-fa49ef7706df (927b0333-54f5-4011-a1f3-3f18a06b5db1) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.576148993Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.576167305Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 0c47db24-e385-42f3-95d6-fa49ef7706df (927b0333-54f5-4011-a1f3-3f18a06b5db1) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.576187278Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.576205178Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 0c47db24-e385-42f3-95d6-fa49ef7706df (927b0333-54f5-4011-a1f3-3f18a06b5db1) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.57622562Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.576260178Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 0c47db24-e385-42f3-95d6-fa49ef7706df (927b0333-54f5-4011-a1f3-3f18a06b5db1) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.576283456Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.576301584Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 0c47db24-e385-42f3-95d6-fa49ef7706df (927b0333-54f5-4011-a1f3-3f18a06b5db1) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.576321572Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.576339371Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 0c47db24-e385-42f3-95d6-fa49ef7706df (927b0333-54f5-4011-a1f3-3f18a06b5db1) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30{"msg",:""time":"Now move the NoOp job forward"2023-09-22T23:08:17.576368834Z",,""v"hostname:":"0,"name":"ip-10-150-1-74.us-west-2.compute.internal"crucible,""pid,"":4291level":}30
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.576389778Z","hostname",:""time":"ip-10-150-1-74.us-west-2.compute.internal"2023-09-22T23:08:17.57640229Z",,""pidhostname""::"4291}
ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{{"msg":""msg":"Now ACK the NoOp job","v":0,"name":"crucible","level":[2] 0c47db24-e385-42f3-95d6-fa49ef7706df (927b0333-54f5-4011-a1f3-3f18a06b5db1) Active Active WaitQuorum ds_transition to Active30","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.57643434Z",,""time"hostname:"":"2023-09-22T23:08:17.576439056Z",ip-10-150-1-74.us-west-2.compute.internal"","hostname"pid:"":4291}
ip-10-150-1-74.us-west-2.compute.internal","pid":{4291}
"msg":"{"Finally, move the ReOpen job forwardmsg"":","v":0,"[2] Transition from WaitQuorum to Active"name":,""v"crucible:"0,,""levelname""::"30crucible","level":30,"time":","time":"2023-09-22T23:08:17.576469356Z","2023-09-22T23:08:17.576471339Zhostname"":","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":ip-10-150-1-74.us-west-2.compute.internal4291","}pid
":4291}
{"{msg":""msg":"Now ACK the Reopen job","v":0,"name":"crucible","0c47db24-e385-42f3-95d6-fa49ef7706df is now active with session: 927b0333-54f5-4011-a1f3-3f18a06b5db1"level":,"30v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.576500933Z",","hostname":time"":"ip-10-150-1-74.us-west-2.compute.internal2023-09-22T23:08:17.576505107Z"",","pid":hostname"4291:"}
ip-10-150-1-74.us-west-2.compute.internal","pid":4291{}
"msg":"{"Extent 0 close id:1002 Failed: Error: badmsg"":","v":0,"name":"crucible","level":50[1] 0c47db24-e385-42f3-95d6-fa49ef7706df (927b0333-54f5-4011-a1f3-3f18a06b5db1) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.576541068Z","hostname",:""time":"ip-10-150-1-74.us-west-2.compute.internal"2023-09-22T23:08:17.576546531Z,"",pid"":hostname4291":"}
ip-10-150-1-74.us-west-2.compute.internal","pid":{4291}
"msg":"{"msg"RE:0 Wait for result from reopen command 1003:4:""----------------------------------------------------------------
[1] Transition from Active to Faulted,""v,"":v"0:,0",name""name:"" Crucible gen:0 GIO:true work queues:  Upstairs:0  downstairs:4
:crucible"",crucible"",level""GW_ID      ACK  DSID    TYPE  BLOCKS  DS:0  DS:1  DS:2  REPLAY
:level"30:30    1    Acked  1000  FClose       0  Done   Err  Done  false
    2    Acked  1001    NoOp       0,","time  Donetime""::  Skip""  Done  false
2023-09-22T23:08:17.576589689Z2023-09-22T23:08:17.576589378Z""    3    Acked  1002    NoOp       0,,""  Donehostnamehostname"  Skip"::""  Done  false
ip-10-150-1-74.us-west-2.compute.internalip-10-150-1-74.us-west-2.compute.internal    4    Acked  1003  Reopen       0"",,"  Donepid""  Skip:pid4291"  Done}:
4291  false
{}   STATES      DS:0   DS:1   DS:2   TOTAL
"
msg"    New        :{"   0      0   "   0   msgExtent 0 close id:1003 Failed: Error: bad"":   0
,    Sent       ""v"   0   :   0   0   0   ,   0
    Done       "[1] 0c47db24-e385-42f3-95d6-fa49ef7706df (927b0333-54f5-4011-a1f3-3f18a06b5db1) Active Faulted Active ds_transition to LiveRepairReadyname"":,""   4   cruciblev"",:"0level,"":   0   50name"   4      8
    Skipped    :   0   "   3      0   crucible"   3
,"    Error         0   ,level""time:"30:   1   "   0      1
2023-09-22T23:08:17.576658929Z"Last Flush: ,0 "0 0 
hostname":","time"Downstairs last five completed::ip-10-150-1-74.us-west-2.compute.internal"",
2023-09-22T23:08:17.576670246Z""pid",:"4291hostname":}"
ip-10-150-1-74.us-west-2.compute.internal","pid":{4291}
"Upstairs last five completed:  msg":{"    4"RE:0 Bailing with error"msg",:""v"    3:    2[1] Transition from Faulted to LiveRepairReady0",,"    1
name"":v"":0crucible",","name"level:"":40crucible","level":30,"time":,""time":"2023-09-22T23:08:17.576706389Z","2023-09-22T23:08:17.576708222Z"hostname",:""hostname":"ip-10-150-1-74.us-west-2.compute.internal","ip-10-150-1-74.us-west-2.compute.internalpid"",:"4291pid":}4291
}
{"msg":"[1] 0c47db24-e385-42f3-95d6-fa49ef7706df (927b0333-54f5-4011-a1f3-3f18a06b5db1) Active LiveRepairReady Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.576743138Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.576775997Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for Close + ReOpen jobs","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.576808728Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.576841023Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 close id:1000 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.576861229Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from close command 1000:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.576879248Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible stats registered with UUID: 179ddd3d-9699-42f1-8c1f-c406c6f24702","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.577036743Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 179ddd3d-9699-42f1-8c1f-c406c6f24702 has session id: bfa3e4d2-5ff8-44fe-9d46-bee60ad15c9f","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.57706701Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 179ddd3d-9699-42f1-8c1f-c406c6f24702 (0798fb5a-0860-4b6f-8556-63835c5b08b6) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.577088615Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.577107238Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 179ddd3d-9699-42f1-8c1f-c406c6f24702 (0798fb5a-0860-4b6f-8556-63835c5b08b6) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.577130751Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.577149362Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 179ddd3d-9699-42f1-8c1f-c406c6f24702 (0798fb5a-0860-4b6f-8556-63835c5b08b6) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.577169239Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.577187049Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 179ddd3d-9699-42f1-8c1f-c406c6f24702 (0798fb5a-0860-4b6f-8556-63835c5b08b6) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.577206937Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.577224719Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 179ddd3d-9699-42f1-8c1f-c406c6f24702 (0798fb5a-0860-4b6f-8556-63835c5b08b6) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.577259874Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.577281624Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 179ddd3d-9699-42f1-8c1f-c406c6f24702 (0798fb5a-0860-4b6f-8556-63835c5b08b6) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.577302222Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.577320161Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 179ddd3d-9699-42f1-8c1f-c406c6f24702 (0798fb5a-0860-4b6f-8556-63835c5b08b6) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.577340356Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.57735824Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 179ddd3d-9699-42f1-8c1f-c406c6f24702 (0798fb5a-0860-4b6f-8556-63835c5b08b6) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.5773784Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.57739666Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 179ddd3d-9699-42f1-8c1f-c406c6f24702 (0798fb5a-0860-4b6f-8556-63835c5b08b6) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.577423032Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.577442294Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"179ddd3d-9699-42f1-8c1f-c406c6f24702 is now active with session: 0798fb5a-0860-4b6f-8556-63835c5b08b6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.577461698Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 179ddd3d-9699-42f1-8c1f-c406c6f24702 (0798fb5a-0860-4b6f-8556-63835c5b08b6) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.577481619Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.577499262Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 179ddd3d-9699-42f1-8c1f-c406c6f24702 (0798fb5a-0860-4b6f-8556-63835c5b08b6) Active Faulted Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.577519357Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.57753725Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 179ddd3d-9699-42f1-8c1f-c406c6f24702 (0798fb5a-0860-4b6f-8556-63835c5b08b6) Active LiveRepairReady Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.577557428Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.577575451Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for Close + ReOpen jobs","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.577593299Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.577611836Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 close id:1000 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.577629634Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from close command 1000:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.57764775Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now move the NoOp job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.581581588Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] DS Reports error Err(GenericError(\"bad\")) on job 1003, DownstairsIO { ds_id: JobId(1003), guest_id: 4, work: ExtentLiveReopen { dependencies: [JobId(1000), JobId(1001), JobId(1002)], extent: 0 }, state: ClientData([Done, InProgress, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:17.58169947Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] Reports error GenericError(\"bad\") on job 1003, DownstairsIO { ds_id: JobId(1003), guest_id: 4, work: ExtentLiveReopen { dependencies: [JobId(1000), JobId(1001), JobId(1002)], extent: 0 }, state: ClientData([Done, InProgress, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:17.581738522Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] client skip 4 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.581765525Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] changed 0 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.581785871Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] d53fef88-17c8-441f-9cbe-e9f976239c5e (36f4bb72-2a32-45e5-94a6-6cee1f536fe0) Active LiveRepair Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.581807551Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from LiveRepair to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.581825567Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from reopen command 1003:4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.581843806Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Extent 0 close id:1003 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:17.581861674Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Bailing with error","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:17.581879091Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible stats registered with UUID: edd7a265-5253-4482-a9cb-a825dc1347c4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.582151393Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible edd7a265-5253-4482-a9cb-a825dc1347c4 has session id: cb7be0f7-fb1c-4503-846e-658817fd52af","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.582183201Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] edd7a265-5253-4482-a9cb-a825dc1347c4 (1dd29b58-71e1-40af-b106-613dfa5b70bd) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.582205072Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.582223574Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] edd7a265-5253-4482-a9cb-a825dc1347c4 (1dd29b58-71e1-40af-b106-613dfa5b70bd) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.582258327Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.582280074Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] edd7a265-5253-4482-a9cb-a825dc1347c4 (1dd29b58-71e1-40af-b106-613dfa5b70bd) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.582300569Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.582318533Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] edd7a265-5253-4482-a9cb-a825dc1347c4 (1dd29b58-71e1-40af-b106-613dfa5b70bd) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.582338431Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.582356214Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] edd7a265-5253-4482-a9cb-a825dc1347c4 (1dd29b58-71e1-40af-b106-613dfa5b70bd) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.582381968Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.582401346Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] edd7a265-5253-4482-a9cb-a825dc1347c4 (1dd29b58-71e1-40af-b106-613dfa5b70bd) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.582421437Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.582439475Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] edd7a265-5253-4482-a9cb-a825dc1347c4 (1dd29b58-71e1-40af-b106-613dfa5b70bd) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.582459774Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.582477757Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] edd7a265-5253-4482-a9cb-a825dc1347c4 (1dd29b58-71e1-40af-b106-613dfa5b70bd) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.582497998Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.582516283Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] edd7a265-5253-4482-a9cb-a825dc1347c4 (1dd29b58-71e1-40af-b106-613dfa5b70bd) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.58253665Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.582554676Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"edd7a265-5253-4482-a9cb-a825dc1347c4 is now active with session: 1dd29b58-71e1-40af-b106-613dfa5b70bd","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.582574214Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] edd7a265-5253-4482-a9cb-a825dc1347c4 (1dd29b58-71e1-40af-b106-613dfa5b70bd) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.582594409Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.582612222Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] edd7a265-5253-4482-a9cb-a825dc1347c4 (1dd29b58-71e1-40af-b106-613dfa5b70bd) Active Faulted Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.582632612Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.582650632Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] edd7a265-5253-4482-a9cb-a825dc1347c4 (1dd29b58-71e1-40af-b106-613dfa5b70bd) Active LiveRepairReady Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.582670959Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.582689133Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for Close + ReOpen jobs","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.582711294Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.582730632Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 close id:1000 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.582748609Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from close command 1000:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.582766856Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now move the NoOp job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.694835559Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now ACK the NoOp job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.694892676Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Finally, move the ReOpen job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.694934461Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now ACK the Reopen job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.694972311Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Extent 0 close id:1002 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:17.695007655Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from reopen command 1003:4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.695040428Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Extent 0 close id:1003 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:17.69507568Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Bailing with error","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:17.695111844Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible stats registered with UUID: 6e81afb6-3f64-41f6-85cf-6a14c224e762","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.695483339Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 6e81afb6-3f64-41f6-85cf-6a14c224e762 has session id: af80e566-5757-4e37-a104-3652dbed167d","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.695516173Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 6e81afb6-3f64-41f6-85cf-6a14c224e762 (5769323c-5c76-4ebc-b603-7b2ed8f3d5cb) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.695537766Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.695556481Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 6e81afb6-3f64-41f6-85cf-6a14c224e762 (5769323c-5c76-4ebc-b603-7b2ed8f3d5cb) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.695576753Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.695594832Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 6e81afb6-3f64-41f6-85cf-6a14c224e762 (5769323c-5c76-4ebc-b603-7b2ed8f3d5cb) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.695614828Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.695632634Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 6e81afb6-3f64-41f6-85cf-6a14c224e762 (5769323c-5c76-4ebc-b603-7b2ed8f3d5cb) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.69565938Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.695678639Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 6e81afb6-3f64-41f6-85cf-6a14c224e762 (5769323c-5c76-4ebc-b603-7b2ed8f3d5cb) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.695698955Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.69571692Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 6e81afb6-3f64-41f6-85cf-6a14c224e762 (5769323c-5c76-4ebc-b603-7b2ed8f3d5cb) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.695736686Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.695754367Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 6e81afb6-3f64-41f6-85cf-6a14c224e762 (5769323c-5c76-4ebc-b603-7b2ed8f3d5cb) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.695774163Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.695791789Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 6e81afb6-3f64-41f6-85cf-6a14c224e762 (5769323c-5c76-4ebc-b603-7b2ed8f3d5cb) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.695811936Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.695829767Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 6e81afb6-3f64-41f6-85cf-6a14c224e762 (5769323c-5c76-4ebc-b603-7b2ed8f3d5cb) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.695849714Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.695867509Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"6e81afb6-3f64-41f6-85cf-6a14c224e762 is now active with session: 5769323c-5c76-4ebc-b603-7b2ed8f3d5cb","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.695887192Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 6e81afb6-3f64-41f6-85cf-6a14c224e762 (5769323c-5c76-4ebc-b603-7b2ed8f3d5cb) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.695907276Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.695924926Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 6e81afb6-3f64-41f6-85cf-6a14c224e762 (5769323c-5c76-4ebc-b603-7b2ed8f3d5cb) Active Faulted Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.69594497Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.695962853Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 6e81afb6-3f64-41f6-85cf-6a14c224e762 (5769323c-5c76-4ebc-b603-7b2ed8f3d5cb) Active LiveRepairReady Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.695987497Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.6960064Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for Close + ReOpen jobs","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.696024069Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.696042125Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 close id:1000 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.696059805Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from close command 1000:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:17.696077841Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for 3 jobs (currently 2)","v":0,"name":"crucible","level":30{"msg":"[2] DS Reports error Err(GenericError(\"bad\")) on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentFlushClose { dependencies: [], extent: 0, flush_number: 0, gen_number: 0, source_downstairs: ClientId(2), repair_downstairs: [ClientId(1)] }, state: ClientData([Done, Done, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,","time":name"":"crucible"2023-09-22T23:08:18.577587237Z","level":,"50hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
,"time":"{2023-09-22T23:08:18.577636961Z","hostname":""ip-10-150-1-74.us-west-2.compute.internal"msg",:""pid":4291No repair needed for extent 0",,""v"":":downstairs"0,"}
name":"crucible","level":30{"msg":"[2] Reports error GenericError(\"bad\") on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentFlushClose { dependencies: [], extent: 0, flush_number: 0, gen_number: 0, source_downstairs: ClientId(2), repair_downstairs: [ClientId(1)] }, state: ClientData([Done, Done, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }",,""v":time0",":name":""crucible","level":502023-09-22T23:08:18.577683037Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","}time
":"2023-09-22T23:08:18.57770613Z","hostname":"{ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":""downstairs"msg"}:"
RE:0 Wait for result from repair command 1001:2","v":0,"name{":"crucible"","msg":level"":30[2] client skip 2 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:18.577743486Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid,"":4291time",:""":"downstairs"}
2023-09-22T23:08:18.577739993Z"{,""hostnamemsg":"":"[2] changed 1 jobs to fault skipped","v":0,"ip-10-150-1-74.us-west-2.compute.internal"name",:""crucible"pid",:"4291level":30}
,"time":"2023-09-22T23:08:18.577773814Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] 179ddd3d-9699-42f1-8c1f-c406c6f24702 (0798fb5a-0860-4b6f-8556-63835c5b08b6) Active LiveRepair Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:18.577796581Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:18.577815044Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now ACK the close job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:18.577832575Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for 3 jobs (currently 2)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:18.577861648Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Extent 0 close id:1000 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:18.577881096Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] client skip 2 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:18.577899205Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:18.577918075Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] 179ddd3d-9699-42f1-8c1f-c406c6f24702 (0798fb5a-0860-4b6f-8556-63835c5b08b6) Active LiveRepair Faulted ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:18.577939541Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from LiveRepair to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:18.577957525Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:18.577975711Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for 3 jobs (currently 2)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:18.583659042Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"No repair needed for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:18.583690697Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:18.583711754Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now ACK the close job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:18.697023833Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for 3 jobs (currently 2)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:18.697053841Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"No repair needed for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:18.697077197Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:18.697105077Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:19.579004807Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:19.579088076Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:19.579132761Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Extent 0 close id:1001 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:19.579169382Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:19.57921585Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:19.585046873Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:19.585088862Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] DS Reports error Err(GenericError(\"bad\")) on job 1001, DownstairsIO { ds_id: JobId(1001), guest_id: 2, work: ExtentLiveNoOp { dependencies: [JobId(1000)] }, state: ClientData([Done, InProgress, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:19.698367573Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] Reports error GenericError(\"bad\") on job 1001, DownstairsIO { ds_id: JobId(1001), guest_id: 2, work: ExtentLiveNoOp { dependencies: [JobId(1000)] }, state: ClientData([Done, InProgress, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:19.698413672Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] client skip 3 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:19.69843499Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:19.698454399Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] 6e81afb6-3f64-41f6-85cf-6a14c224e762 (5769323c-5c76-4ebc-b603-7b2ed8f3d5cb) Active LiveRepair Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:19.69847646Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from LiveRepair to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:19.698495023Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:19.698512819Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Extent 0 close id:1001 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:19.698530723Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:19.698548997Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
----------------------------------------------------------------
 Crucible gen:0 GIO:true work queues:  Upstairs:2  downstairs:4
GW_ID      ACK  DSID    TYPE  BLOCKS  DS:0  DS:1  DS:2  REPLAY
    1    Acked  1000  FClose       0  Done  Done  Done  false
    2    Acked  1001    NoOp       0  Done  Done  Done  false
    3 NotAcked  1002    NoOp       0   New   New   New  false
    4 NotAcked  1003  Reopen       0   New   New   New  false
   STATES      DS:0   DS:1   DS:2   TOTAL
    New           2      2      2      6
    Sent          0      0      0      0
    Done          2      2      2      6
    Skipped       0      0      0      0
    Error         0      0      0      0
Last Flush: 0 0 0 
Downstairs last five completed:
Upstairs last five completed:      2    1
{"msg":"Now move the NoOp job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.448267245Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Finally, move the ReOpen job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.448318156Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now ACK the reopen job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.448337593Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from reopen command 1003:4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.448365119Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"jobs are: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.448427035Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test live_repair::repair_test::test_repair_extent_no_action_all ... ok
{"msg":"Now move the NoOp job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.579424242Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now ACK the NoOp job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.579456562Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Finally, move the ReOpen job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.579475457Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now ACK the Reopen job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.579493321Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Extent 0 close id:1002 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:20.579511545Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from reopen command 1003:4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.579530051Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Extent 0 close id:1003 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:20.579548326Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Bailing with error","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:20.57956605Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
----------------------------------------------------------------
 Crucible gen:0 GIO:true work queues:  Upstairs:0  downstairs:4
GW_ID      ACK  DSID    TYPE  BLOCKS  DS:0  DS:1  DS:2  REPLAY
    1    Acked  1000  FClose       0  Done  Done   Err  false
    2    Acked  1001    NoOp       0  Done  Skip  Skip  false
    3    Acked  1002    NoOp       0  Done  Skip  Skip  false
    4    Acked  1003  Reopen       0  Done  Skip  Skip  false
   STATES      DS:0   DS:1   DS:2   TOTAL
    New           0      0      0      0
    Sent          0      0      0      0
    Done          4      1      0      5
    Skipped       0      3      3      6
    Error         0      0      1      1
Last Flush: 0 0 0 
Downstairs last five completed:
Upstairs last five completed:      4    3    2    1
{"msg":"Crucible stats registered with UUID: 447579bf-098c-4009-8361-0912b7f0245f","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.580174177Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 447579bf-098c-4009-8361-0912b7f0245f has session id: b510ee14-6aae-4b3a-bb93-47d3774c88ab","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.580209288Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 447579bf-098c-4009-8361-0912b7f0245f (bc569f92-47ce-43bf-a7ad-b0ac8e3af76e) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.580231409Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"{2023-09-22T23:08:20.580256604Z","hostname":""msg":"ip-10-150-1-74.us-west-2.compute.internal","pidNow move the NoOp job forward"":4291,"}v"
:0,"{name":""crucible"msg":","level":30[0] 447579bf-098c-4009-8361-0912b7f0245f (bc569f92-47ce-43bf-a7ad-b0ac8e3af76e) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,,""timetime""::""2023-09-22T23:08:20.580292073Z2023-09-22T23:08:20.580284253Z"",","hostname"hostname:"":"ip-10-150-1-74.us-west-2.compute.internal"ip-10-150-1-74.us-west-2.compute.internal,"",pid"":pid"4291:4291}
}
{"{msg":""[0] Transition from WaitActive to WaitQuorummsg"":","v":0,"name":[0] DS Reports error Err(GenericError("\"crucible"bad\","level":30,"time":")) on job 1002, DownstairsIO { ds_id: JobId(1002), guest_id: 3, work: ExtentLiveNoOp { dependencies: [JobId(1000), JobId(1001)] }, state: ClientData([InProgress, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }2023-09-22T23:08:20.58033424Z"",,""v":hostname"0:","name":"crucibleip-10-150-1-74.us-west-2.compute.internal"",,""levelpid""::504291}
{"msg":","time":"2023-09-22T23:08:20.580354514Z[0] 447579bf-098c-4009-8361-0912b7f0245f (bc569f92-47ce-43bf-a7ad-b0ac8e3af76e) WaitQuorum New New ds_transition to Active"",,""v"hostname:":0","name":"crucibleip-10-150-1-74.us-west-2.compute.internal"",,""pid"level:":429130,"":"downstairs"}
{,""msgtime""::""2023-09-22T23:08:20.580373313Z[0] Reports error GenericError("\"bad,\""hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,") on job 1002, DownstairsIO { ds_id: JobId(1002), guest_id: 3, work: ExtentLiveNoOp { dependencies: [JobId(1000), JobId(1001)] }, state: ClientData([InProgress, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }name"":","vcrucible"":0,","level"name:":30"crucible","level":50,"time":","2023-09-22T23:08:20.580405447Z"time":,""hostname":"2023-09-22T23:08:20.580407845Z","hostname"ip-10-150-1-74.us-west-2.compute.internal:"","pid":4291ip-10-150-1-74.us-west-2.compute.internal}"
,"pid":4291{,"":""msg":downstairs""}
{"[1] 447579bf-098c-4009-8361-0912b7f0245f (bc569f92-47ce-43bf-a7ad-b0ac8e3af76e) Active New New ds_transition to WaitActive"msg":,""v":0,"name":[0] client skip 4 in process jobs because fault"",crucible""v",:"0,level"":name30":"crucible","level":30,"time":","2023-09-22T23:08:20.580440636Ztime"":","hostname":"2023-09-22T23:08:20.58044317Z","hostname":ip-10-150-1-74.us-west-2.compute.internal"","pid":4291ip-10-150-1-74.us-west-2.compute.internal"},
"pid":4291{,"":""msgdownstairs"":"}
[1] Transition from New to WaitActive","v"{:0,""name"msg:"":"crucible","level"[0] changed 1 jobs to fault skipped:"30,"v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.58047223Z","hostname",:""time":"ip-10-150-1-74.us-west-2.compute.internal"2023-09-22T23:08:20.580476858Z,"",pid"":4291hostname":"}
ip-10-150-1-74.us-west-2.compute.internal","pid"{:4291,""":"msg":"downstairs"}
{"[1] 447579bf-098c-4009-8361-0912b7f0245f (bc569f92-47ce-43bf-a7ad-b0ac8e3af76e) Active WaitActive New ds_transition to WaitQuorummsg"":","v":0,"name":"crucible","level":30[0] 0c47db24-e385-42f3-95d6-fa49ef7706df (927b0333-54f5-4011-a1f3-3f18a06b5db1) Active LiveRepair Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.580507249Z","hostname":","timeip-10-150-1-74.us-west-2.compute.internal"":","pid":42912023-09-22T23:08:20.580513342Z"},
"hostname":"{"msg"ip-10-150-1-74.us-west-2.compute.internal:"","pid":4291}[1] Transition from WaitActive to WaitQuorum
","v":{0,""name":msg"":"crucible","level":[0] Transition from Active to Faulted30","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.580539894Z","hostname":","time":"ip-10-150-1-74.us-west-2.compute.internal","2023-09-22T23:08:20.580544931Z"pid":,"4291hostname"}:
"{ip-10-150-1-74.us-west-2.compute.internal",""pidmsg""::"4291}
{"[1] 447579bf-098c-4009-8361-0912b7f0245f (bc569f92-47ce-43bf-a7ad-b0ac8e3af76e) Active WaitQuorum New ds_transition to Activemsg"":","v":0,"Extent 0 close id:1002 Failed: Error: badname"":","vcrucible"":,0","level":name"30:"crucible","level":50,"time":","time2023-09-22T23:08:20.580574739Z"":","hostname":2023-09-22T23:08:20.580577214Z"","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291ip-10-150-1-74.us-west-2.compute.internal",}"
pid":4291}
{{"msg":""msg":"[1] Transition from WaitQuorum to Active","[1] client skip 4 in process jobs because fault"v":,"0v,"":0name",:""name"crucible:"","crucible"level",:"30level":30,,""timetime""::""2023-09-22T23:08:20.580613489Z2023-09-22T23:08:20.580614289Z"",,""hostnamehostname""::""ip-10-150-1-74.us-west-2.compute.internalip-10-150-1-74.us-west-2.compute.internal"",,""pidpid""::42914291,}"
":"downstairs"{}
"msg":"{"msg":"[1] changed 1 jobs to fault skipped","v"[2] 447579bf-098c-4009-8361-0912b7f0245f (bc569f92-47ce-43bf-a7ad-b0ac8e3af76e) Active Active New ds_transition to WaitActive:"0,,""v"name:":0","crucible"name",:""level"crucible:"30,"level":30,",time"":time"":"2023-09-22T23:08:20.580650799Z"2023-09-22T23:08:20.580652094Z",","hostname":hostname"":"ip-10-150-1-74.us-west-2.compute.internal"ip-10-150-1-74.us-west-2.compute.internal,"","pid"pid:":42914291,"}"
:"downstairs"{}
"msg":"{"[2] Transition from New to WaitActive"msg":,""v":0,"name":"crucible","level":30[1] 0c47db24-e385-42f3-95d6-fa49ef7706df (927b0333-54f5-4011-a1f3-3f18a06b5db1) Faulted LiveRepair Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.580683301Z","hostname":","ip-10-150-1-74.us-west-2.compute.internal"time,"":"pid":4291}2023-09-22T23:08:20.580689541Z
","hostname{":""msg":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
[2] 447579bf-098c-4009-8361-0912b7f0245f (bc569f92-47ce-43bf-a7ad-b0ac8e3af76e) Active Active WaitActive ds_transition to WaitQuorum{",""v"msg:":0","name":"crucible[1] Transition from LiveRepair to Faulted"",","levelv""::300,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.580718083Z",","time":hostname"":"2023-09-22T23:08:20.580721668Z","ip-10-150-1-74.us-west-2.compute.internal"hostname,"":"pid":4291}
ip-10-150-1-74.us-west-2.compute.internal","pid":{4291}"
msg":"{"[2] Transition from WaitActive to WaitQuorummsg"":","v":0,"nameRE:0 Wait for result from reopen command 1003:4"":","cruciblev"":,0","level"name:":30"crucible","level":30,"time":","time"2023-09-22T23:08:20.580751055Z:"","hostname"2023-09-22T23:08:20.580753324Z:"","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":ip-10-150-1-74.us-west-2.compute.internal4291","}pid
":4291}
{"msg":"[2] 447579bf-098c-4009-8361-0912b7f0245f (bc569f92-47ce-43bf-a7ad-b0ac8e3af76e) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.580786452Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.580818941Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"447579bf-098c-4009-8361-0912b7f0245f is now active with session: bc569f92-47ce-43bf-a7ad-b0ac8e3af76e","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.580845336Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 447579bf-098c-4009-8361-0912b7f0245f (bc569f92-47ce-43bf-a7ad-b0ac8e3af76e) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.580865797Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.580883621Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 447579bf-098c-4009-8361-0912b7f0245f (bc569f92-47ce-43bf-a7ad-b0ac8e3af76e) Active Active Faulted ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.580903968Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.58092195Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 447579bf-098c-4009-8361-0912b7f0245f (bc569f92-47ce-43bf-a7ad-b0ac8e3af76e) Active Active LiveRepairReady ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.580942262Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.580965369Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for Close + ReOpen jobs","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.58098417Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.581002596Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 close id:1000 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.58102023Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from close command 1000:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.581038274Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now move the NoOp job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.586315591Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] DS Reports error Err(GenericError(\"bad\")) on job 1003, DownstairsIO { ds_id: JobId(1003), guest_id: 4, work: ExtentLiveReopen { dependencies: [JobId(1000), JobId(1001), JobId(1002)], extent: 0 }, state: ClientData([Done, Done, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:20.586434674Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] Reports error GenericError(\"bad\") on job 1003, DownstairsIO { ds_id: JobId(1003), guest_id: 4, work: ExtentLiveReopen { dependencies: [JobId(1000), JobId(1001), JobId(1002)], extent: 0 }, state: ClientData([Done, Done, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:20.586475265Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] client skip 4 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.586496312Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] changed 0 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.586515336Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] edd7a265-5253-4482-a9cb-a825dc1347c4 (1dd29b58-71e1-40af-b106-613dfa5b70bd) Active LiveRepair Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.586537149Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.586555345Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from reopen command 1003:4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.586573671Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Extent 0 close id:1003 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:20.58659162Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] client skip 4 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.586610233Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] changed 0 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.586629051Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] edd7a265-5253-4482-a9cb-a825dc1347c4 (1dd29b58-71e1-40af-b106-613dfa5b70bd) Active LiveRepair Faulted ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.586659806Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from LiveRepair to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.58667959Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Bailing with error","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:20.586697619Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible stats registered with UUID: 3d1b0574-7dcd-4ea0-9e64-07ebc6e4e1b4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.587028558Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 3d1b0574-7dcd-4ea0-9e64-07ebc6e4e1b4 has session id: 92a4d028-35da-4409-8cd6-38bb5d06dda2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.587063072Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 3d1b0574-7dcd-4ea0-9e64-07ebc6e4e1b4 (74a0a767-aee3-41c9-a191-b12e771ab110) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.587085052Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.587103839Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 3d1b0574-7dcd-4ea0-9e64-07ebc6e4e1b4 (74a0a767-aee3-41c9-a191-b12e771ab110) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.587131098Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.587158313Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 3d1b0574-7dcd-4ea0-9e64-07ebc6e4e1b4 (74a0a767-aee3-41c9-a191-b12e771ab110) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.587179575Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.587197573Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 3d1b0574-7dcd-4ea0-9e64-07ebc6e4e1b4 (74a0a767-aee3-41c9-a191-b12e771ab110) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.587217518Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.587235328Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 3d1b0574-7dcd-4ea0-9e64-07ebc6e4e1b4 (74a0a767-aee3-41c9-a191-b12e771ab110) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.587255578Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.587273719Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 3d1b0574-7dcd-4ea0-9e64-07ebc6e4e1b4 (74a0a767-aee3-41c9-a191-b12e771ab110) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.58729373Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.587311679Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 3d1b0574-7dcd-4ea0-9e64-07ebc6e4e1b4 (74a0a767-aee3-41c9-a191-b12e771ab110) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.587331706Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.587357214Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 3d1b0574-7dcd-4ea0-9e64-07ebc6e4e1b4 (74a0a767-aee3-41c9-a191-b12e771ab110) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.587379018Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.587397142Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 3d1b0574-7dcd-4ea0-9e64-07ebc6e4e1b4 (74a0a767-aee3-41c9-a191-b12e771ab110) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.587417293Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.587435184Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"3d1b0574-7dcd-4ea0-9e64-07ebc6e4e1b4 is now active with session: 74a0a767-aee3-41c9-a191-b12e771ab110","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.587454771Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 3d1b0574-7dcd-4ea0-9e64-07ebc6e4e1b4 (74a0a767-aee3-41c9-a191-b12e771ab110) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.587474901Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.587492646Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 3d1b0574-7dcd-4ea0-9e64-07ebc6e4e1b4 (74a0a767-aee3-41c9-a191-b12e771ab110) Active Active Faulted ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.587512754Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.587530665Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 3d1b0574-7dcd-4ea0-9e64-07ebc6e4e1b4 (74a0a767-aee3-41c9-a191-b12e771ab110) Active Active LiveRepairReady ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.587550851Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.58756888Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for Close + ReOpen jobs","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.587586596Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.587604791Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 close id:1000 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.587622428Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from close command 1000:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.587640498Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now move the NoOp job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.699698307Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now ACK the NoOp job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.69973085Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Finally, move the ReOpen job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.699756261Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now ACK the Reopen job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.699775559Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Extent 0 close id:1002 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:20.699793965Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from reopen command 1003:4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.699812393Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Extent 0 close id:1003 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:20.699830662Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Bailing with error","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:20.699848579Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible stats registered with UUID: c43b617e-cb7a-45ad-bdb9-19a9531028df","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.700263317Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible c43b617e-cb7a-45ad-bdb9-19a9531028df has session id: 6b8693a3-f5cb-4325-aa1b-5eb99cfa65bb","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.700296515Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] c43b617e-cb7a-45ad-bdb9-19a9531028df (6036a6d9-4d85-440b-b090-7602e4205cf6) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.700318361Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.700337013Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] c43b617e-cb7a-45ad-bdb9-19a9531028df (6036a6d9-4d85-440b-b090-7602e4205cf6) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.70035746Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.700375892Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] c43b617e-cb7a-45ad-bdb9-19a9531028df (6036a6d9-4d85-440b-b090-7602e4205cf6) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.700395965Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.700413937Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] c43b617e-cb7a-45ad-bdb9-19a9531028df (6036a6d9-4d85-440b-b090-7602e4205cf6) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.700434069Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.7004518Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] c43b617e-cb7a-45ad-bdb9-19a9531028df (6036a6d9-4d85-440b-b090-7602e4205cf6) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.700472433Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.700490556Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] c43b617e-cb7a-45ad-bdb9-19a9531028df (6036a6d9-4d85-440b-b090-7602e4205cf6) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.700510468Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.700534046Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] c43b617e-cb7a-45ad-bdb9-19a9531028df (6036a6d9-4d85-440b-b090-7602e4205cf6) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.700555366Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.700573262Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] c43b617e-cb7a-45ad-bdb9-19a9531028df (6036a6d9-4d85-440b-b090-7602e4205cf6) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.700593506Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.700611534Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] c43b617e-cb7a-45ad-bdb9-19a9531028df (6036a6d9-4d85-440b-b090-7602e4205cf6) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.700631791Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.700649698Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"c43b617e-cb7a-45ad-bdb9-19a9531028df is now active with session: 6036a6d9-4d85-440b-b090-7602e4205cf6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.700669141Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] c43b617e-cb7a-45ad-bdb9-19a9531028df (6036a6d9-4d85-440b-b090-7602e4205cf6) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.700689135Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.700706899Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] c43b617e-cb7a-45ad-bdb9-19a9531028df (6036a6d9-4d85-440b-b090-7602e4205cf6) Active Faulted Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.700727065Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.700745023Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] c43b617e-cb7a-45ad-bdb9-19a9531028df (6036a6d9-4d85-440b-b090-7602e4205cf6) Active LiveRepairReady Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.700765265Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.700783421Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for Close + ReOpen jobs","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.700801106Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.700819417Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 close id:1000 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.700837117Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from close command 1000:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:20.700855177Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] DS Reports error Err(GenericError(\"bad\")) on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentFlushClose { dependencies: [], extent: 0, flush_number: 0, gen_number: 0, source_downstairs: ClientId(0), repair_downstairs: [ClientId(2)] }, state: ClientData([InProgress, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:21.581800705Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[0] Reports error GenericError(\"bad\") on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentFlushClose { dependencies: [], extent: 0, flush_number: 0, gen_number: 0, source_downstairs: ClientId(0), repair_downstairs: [ClientId(2)] }, state: ClientData([InProgress, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:21.581869637Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[0] client skip 2 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:21.581891583Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[0] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:21.581911135Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[0] 447579bf-098c-4009-8361-0912b7f0245f (bc569f92-47ce-43bf-a7ad-b0ac8e3af76e) Active Active LiveRepair ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:21.581933026Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:21.581951395Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now ACK the close job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:21.581969123Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for 3 jobs (currently 2)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:21.581986923Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Extent 0 close id:1000 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:21.582004937Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] client skip 2 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:21.582023079Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:21.582041983Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] 447579bf-098c-4009-8361-0912b7f0245f (bc569f92-47ce-43bf-a7ad-b0ac8e3af76e) Faulted Active LiveRepair ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:21.582063449Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from LiveRepair to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:21.582081441Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:21.582099993Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for 3 jobs (currently 2)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:21.588549592Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"No repair needed for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:21.588593512Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:21.588617028Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now ACK the close job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:21.701726856Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for 3 jobs (currently 2)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:21.701759902Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"No repair needed for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:21.701779066Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:21.701800255Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:22.426318238Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:22.427355808Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"Finally, move the ReOpen job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:22.581647968Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now ACK the reopen job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:22.581678228Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Bailing with error","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:22.581707636Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"err:0 or:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:22.5818791Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:22.582289508Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Extent 0 close id:1001 Failed: Error: bad","v":0,"name":"crucible","level":50{"msg,"":"time":"2023-09-22T23:08:22.582337682Z","Crucible stats registered with UUID: 9a454318-3f11-4b24-be6e-b1033eaa5db9"hostname":","v":ip-10-150-1-74.us-west-2.compute.internal"0,",pid"":name":4291"crucible"}
,"level":30{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,,""time":"time":"2023-09-22T23:08:22.58238439Z","hostname":2023-09-22T23:08:22.582373004Z""ip-10-150-1-74.us-west-2.compute.internal,"","hostnamepid":"4291:"}
ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 9a454318-3f11-4b24-be6e-b1033eaa5db9 has session id: 09f21dfe-f6bc-4f91-af12-e181495c6b25","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:22.58244037Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 9a454318-3f11-4b24-be6e-b1033eaa5db9 (78e805e3-7c37-4f81-915e-a0267c342496) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:22.582489414Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:22.582532874Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 9a454318-3f11-4b24-be6e-b1033eaa5db9 (78e805e3-7c37-4f81-915e-a0267c342496) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:22.582577397Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:22.58261962Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 9a454318-3f11-4b24-be6e-b1033eaa5db9 (78e805e3-7c37-4f81-915e-a0267c342496) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:22.582677697Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:22.582721516Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 9a454318-3f11-4b24-be6e-b1033eaa5db9 (78e805e3-7c37-4f81-915e-a0267c342496) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:22.582764596Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:22.582806294Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 9a454318-3f11-4b24-be6e-b1033eaa5db9 (78e805e3-7c37-4f81-915e-a0267c342496) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:22.582850596Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:22.582893087Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 9a454318-3f11-4b24-be6e-b1033eaa5db9 (78e805e3-7c37-4f81-915e-a0267c342496) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:22.582935714Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:22.582976023Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 9a454318-3f11-4b24-be6e-b1033eaa5db9 (78e805e3-7c37-4f81-915e-a0267c342496) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:22.583016288Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:22.583057309Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 9a454318-3f11-4b24-be6e-b1033eaa5db9 (78e805e3-7c37-4f81-915e-a0267c342496) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:22.583102003Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:22.583143502Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 9a454318-3f11-4b24-be6e-b1033eaa5db9 (78e805e3-7c37-4f81-915e-a0267c342496) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:22.583186965Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:22.583227218Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"9a454318-3f11-4b24-be6e-b1033eaa5db9 is now active with session: 78e805e3-7c37-4f81-915e-a0267c342496","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:22.583266681Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 9a454318-3f11-4b24-be6e-b1033eaa5db9 (78e805e3-7c37-4f81-915e-a0267c342496) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:22.583302902Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:22.5833424Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 9a454318-3f11-4b24-be6e-b1033eaa5db9 (78e805e3-7c37-4f81-915e-a0267c342496) Active Faulted Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:22.583395514Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:22.583438293Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 9a454318-3f11-4b24-be6e-b1033eaa5db9 (78e805e3-7c37-4f81-915e-a0267c342496) Active LiveRepairReady Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:22.583475484Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:22.583508118Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for Close + ReOpen jobs","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:22.583539696Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:22.583572282Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 close id:1000 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:22.583603939Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from close command 1000:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:22.583636001Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:22.589793756Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:22.589829657Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] DS Reports error Err(GenericError(\"bad\")) on job 1001, DownstairsIO { ds_id: JobId(1001), guest_id: 2, work: ExtentLiveNoOp { dependencies: [JobId(1000)] }, state: ClientData([Done, Done, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:22.703151669Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] Reports error GenericError(\"bad\") on job 1001, DownstairsIO { ds_id: JobId(1001), guest_id: 2, work: ExtentLiveNoOp { dependencies: [JobId(1000)] }, state: ClientData([Done, Done, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:22.703198963Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] client skip 3 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:22.703220477Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:22.703239555Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] c43b617e-cb7a-45ad-bdb9-19a9531028df (6036a6d9-4d85-440b-b090-7602e4205cf6) Active LiveRepair Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:22.70326119Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:22.703279525Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:22.703305207Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Extent 0 close id:1001 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:22.703324498Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] client skip 3 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:22.703342795Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:22.703361777Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] c43b617e-cb7a-45ad-bdb9-19a9531028df (6036a6d9-4d85-440b-b090-7602e4205cf6) Active LiveRepair Faulted ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:22.703383167Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from LiveRepair to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:22.703401234Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:22.703419374Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now move the NoOp job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.583704612Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now ACK the NoOp job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.583785572Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Finally, move the ReOpen job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.583825217Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now ACK the Reopen job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.583859334Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Extent 0 close id:1002 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:23.58389128Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from reopen command 1003:4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.583924534Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Extent 0 close id:1003 Failed: Error: bad","v":0,"name":"crucible","level":50{"msg":"Waiting for 3 jobs (currently 2)",",time"":"v":2023-09-22T23:08:23.583957399Z"0,","hostname":"name":"crucible"ip-10-150-1-74.us-west-2.compute.internal",","pid"level:"4291:}----------------------------------------------------------------

30 Crucible gen:0 GIO:true work queues:  Upstairs:0  downstairs:4
{"GW_ID      ACK  DSID    TYPE  BLOCKS  DS:0  DS:1  DS:2  REPLAY
msg":"RE:0 Bailing with error","v":0,"name":"crucible"    1    Acked  1000  FClose       0,"level":   Err40  Done  Done  false
,"time":    2    Acked  1001    NoOp       0"  Skip  Done  Skip  false2023-09-22T23:08:23.583993099Z"
,"    3    Acked  1002    NoOp       0hostname  Skip":  Done"  Skip  false
ip-10-150-1-74.us-west-2.compute.internal",    4    Acked  1003  Reopen       0  Skip  Done  Skip  false
   STATES      DS:0   DS:1   DS:2   TOTAL
,    New        "   0   time"   0   :   0   ""   0
2023-09-22T23:08:23.584012214Z    Sent       "   0   pid",   0   "   0   hostname"   0
:    Done       :   0   "   4   4291   1      5
ip-10-150-1-74.us-west-2.compute.internal    Skipped    "   3   ,   0   "   3   pid   6
"}    Error      :   1   4291   0   
}   0   
   1
{Last Flush: 0 0 0 "
msg":"No repair needed for extent 0"Downstairs last five completed:,"
v":0,"name":"crucible","level":30Upstairs last five completed:      4    3    2    1
,"time":"2023-09-22T23:08:23.584092974Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.584148456Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible stats registered with UUID: f5357d68-1826-4d13-a77d-b2335da9d6f2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.584650501Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible f5357d68-1826-4d13-a77d-b2335da9d6f2 has session id: cd0c0a27-6e82-4f52-b38f-03711c7df63b","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.584686857Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] f5357d68-1826-4d13-a77d-b2335da9d6f2 (c3b8b937-bdf2-428d-b207-29fe37e50ee9) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.584709364Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.584728157Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] f5357d68-1826-4d13-a77d-b2335da9d6f2 (c3b8b937-bdf2-428d-b207-29fe37e50ee9) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.584748668Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.584766922Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] f5357d68-1826-4d13-a77d-b2335da9d6f2 (c3b8b937-bdf2-428d-b207-29fe37e50ee9) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.584787029Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.584805042Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] f5357d68-1826-4d13-a77d-b2335da9d6f2 (c3b8b937-bdf2-428d-b207-29fe37e50ee9) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.584825131Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.58484288Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] f5357d68-1826-4d13-a77d-b2335da9d6f2 (c3b8b937-bdf2-428d-b207-29fe37e50ee9) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.584863433Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.584881379Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] f5357d68-1826-4d13-a77d-b2335da9d6f2 (c3b8b937-bdf2-428d-b207-29fe37e50ee9) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.584901357Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.584919082Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] f5357d68-1826-4d13-a77d-b2335da9d6f2 (c3b8b937-bdf2-428d-b207-29fe37e50ee9) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.584939235Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.584956979Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] f5357d68-1826-4d13-a77d-b2335da9d6f2 (c3b8b937-bdf2-428d-b207-29fe37e50ee9) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.584984102Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.585003641Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] f5357d68-1826-4d13-a77d-b2335da9d6f2 (c3b8b937-bdf2-428d-b207-29fe37e50ee9) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.5850238Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.585041902Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"f5357d68-1826-4d13-a77d-b2335da9d6f2 is now active with session: c3b8b937-bdf2-428d-b207-29fe37e50ee9","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.585061411Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] f5357d68-1826-4d13-a77d-b2335da9d6f2 (c3b8b937-bdf2-428d-b207-29fe37e50ee9) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.58508135Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.585099066Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] f5357d68-1826-4d13-a77d-b2335da9d6f2 (c3b8b937-bdf2-428d-b207-29fe37e50ee9) Active Active Faulted ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.585119276Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.585137325Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] f5357d68-1826-4d13-a77d-b2335da9d6f2 (c3b8b937-bdf2-428d-b207-29fe37e50ee9) Active Active LiveRepairReady ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.5851574Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.585175516Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for Close + ReOpen jobs","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.585193212Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.5852115Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 close id:1000 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.585229384Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from close command 1000:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.585247364Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now move the NoOp job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.591146773Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] DS Reports error Err(GenericError(\"bad\")) on job 1003, DownstairsIO { ds_id: JobId(1003), guest_id: 4, work: ExtentLiveReopen { dependencies: [JobId(1000), JobId(1001), JobId(1002)], extent: 0 }, state: ClientData([InProgress, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:23.59123253Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[0] Reports error GenericError(\"bad\") on job 1003, DownstairsIO { ds_id: JobId(1003), guest_id: 4, work: ExtentLiveReopen { dependencies: [JobId(1000), JobId(1001), JobId(1002)], extent: 0 }, state: ClientData([InProgress, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:23.591280232Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[0] client skip 4 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.591303398Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[0] changed 0 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.59132299Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[0] 3d1b0574-7dcd-4ea0-9e64-07ebc6e4e1b4 (74a0a767-aee3-41c9-a191-b12e771ab110) Active Active LiveRepair ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.591344833Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.591363229Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from reopen command 1003:4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.591381657Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Extent 0 close id:1003 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:23.591399829Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] client skip 4 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.591418048Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] changed 0 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.591436908Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] 3d1b0574-7dcd-4ea0-9e64-07ebc6e4e1b4 (74a0a767-aee3-41c9-a191-b12e771ab110) Faulted Active LiveRepair ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.591459338Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from LiveRepair to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.591477794Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Bailing with error","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:23.591495636Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible stats registered with UUID: 2f3bf994-5f9f-4b02-b326-33674e795003","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.591822991Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 2f3bf994-5f9f-4b02-b326-33674e795003 has session id: bbb5cadd-c4c2-462b-aeb7-82647c4128c6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.591857131Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 2f3bf994-5f9f-4b02-b326-33674e795003 (0a9fd518-e87c-445f-bf08-874431d889a8) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.591879273Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.591898435Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 2f3bf994-5f9f-4b02-b326-33674e795003 (0a9fd518-e87c-445f-bf08-874431d889a8) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.591918958Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.591943598Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 2f3bf994-5f9f-4b02-b326-33674e795003 (0a9fd518-e87c-445f-bf08-874431d889a8) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.591965094Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.591983223Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 2f3bf994-5f9f-4b02-b326-33674e795003 (0a9fd518-e87c-445f-bf08-874431d889a8) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.592015998Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.592038552Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 2f3bf994-5f9f-4b02-b326-33674e795003 (0a9fd518-e87c-445f-bf08-874431d889a8) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.592059335Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.592077392Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 2f3bf994-5f9f-4b02-b326-33674e795003 (0a9fd518-e87c-445f-bf08-874431d889a8) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.592097242Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.592115157Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 2f3bf994-5f9f-4b02-b326-33674e795003 (0a9fd518-e87c-445f-bf08-874431d889a8) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.592135082Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.592152772Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 2f3bf994-5f9f-4b02-b326-33674e795003 (0a9fd518-e87c-445f-bf08-874431d889a8) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.592172935Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.592191098Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 2f3bf994-5f9f-4b02-b326-33674e795003 (0a9fd518-e87c-445f-bf08-874431d889a8) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.592211141Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.592229013Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"2f3bf994-5f9f-4b02-b326-33674e795003 is now active with session: 0a9fd518-e87c-445f-bf08-874431d889a8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.592248471Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 2f3bf994-5f9f-4b02-b326-33674e795003 (0a9fd518-e87c-445f-bf08-874431d889a8) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.5922686Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.592291994Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 2f3bf994-5f9f-4b02-b326-33674e795003 (0a9fd518-e87c-445f-bf08-874431d889a8) Active Active Faulted ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.592313258Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.592331556Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 2f3bf994-5f9f-4b02-b326-33674e795003 (0a9fd518-e87c-445f-bf08-874431d889a8) Active Active LiveRepairReady ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.592351719Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.592369887Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for Close + ReOpen jobs","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.592387452Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.592405722Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 close id:1000 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.592423513Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from close command 1000:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.59244181Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now move the NoOp job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.704501766Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now ACK the NoOp job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.704538546Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Finally, move the ReOpen job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.704558137Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now ACK the Reopen job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.70457572Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Extent 0 close id:1002 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:23.704593939Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from reopen command 1003:4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.704612248Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Extent 0 close id:1003 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:23.704630212Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Bailing with error","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:23.704647814Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible stats registered with UUID: 550be307-1aa8-42ac-b59e-9ea2749e822e","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.705087418Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 550be307-1aa8-42ac-b59e-9ea2749e822e has session id: a598f884-43b9-4928-b779-fd14ffe82b81","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.705120945Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 550be307-1aa8-42ac-b59e-9ea2749e822e (5aed1069-eb98-435d-bbfb-e191ed7d789f) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.705142536Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.705168509Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 550be307-1aa8-42ac-b59e-9ea2749e822e (5aed1069-eb98-435d-bbfb-e191ed7d789f) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.70519026Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.705208568Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 550be307-1aa8-42ac-b59e-9ea2749e822e (5aed1069-eb98-435d-bbfb-e191ed7d789f) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.705228535Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.705246292Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 550be307-1aa8-42ac-b59e-9ea2749e822e (5aed1069-eb98-435d-bbfb-e191ed7d789f) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.705266208Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.705283986Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 550be307-1aa8-42ac-b59e-9ea2749e822e (5aed1069-eb98-435d-bbfb-e191ed7d789f) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.705304242Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.705322158Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 550be307-1aa8-42ac-b59e-9ea2749e822e (5aed1069-eb98-435d-bbfb-e191ed7d789f) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.705341987Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.705359709Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 550be307-1aa8-42ac-b59e-9ea2749e822e (5aed1069-eb98-435d-bbfb-e191ed7d789f) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.705379558Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.705397317Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 550be307-1aa8-42ac-b59e-9ea2749e822e (5aed1069-eb98-435d-bbfb-e191ed7d789f) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.705417281Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.705435151Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 550be307-1aa8-42ac-b59e-9ea2749e822e (5aed1069-eb98-435d-bbfb-e191ed7d789f) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.705455165Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.705472854Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"550be307-1aa8-42ac-b59e-9ea2749e822e is now active with session: 5aed1069-eb98-435d-bbfb-e191ed7d789f","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.70549703Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 550be307-1aa8-42ac-b59e-9ea2749e822e (5aed1069-eb98-435d-bbfb-e191ed7d789f) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.705518091Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.705535881Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 550be307-1aa8-42ac-b59e-9ea2749e822e (5aed1069-eb98-435d-bbfb-e191ed7d789f) Active Active Faulted ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.705556101Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.705574048Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 550be307-1aa8-42ac-b59e-9ea2749e822e (5aed1069-eb98-435d-bbfb-e191ed7d789f) Active Active LiveRepairReady ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.705594138Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.705612006Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for Close + ReOpen jobs","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.705629644Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.70564795Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 close id:1000 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.705665679Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from close command 1000:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:23.705683741Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] DS Reports error Err(GenericError(\"bad\")) on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentFlushClose { dependencies: [], extent: 0, flush_number: 0, gen_number: 0, source_downstairs: ClientId(0), repair_downstairs: [ClientId(2)] }, state: ClientData([Done, InProgress, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:24.5852195Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] Reports error GenericError(\"bad\") on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentFlushClose { dependencies: [], extent: 0, flush_number: 0, gen_number: 0, source_downstairs: ClientId(0), repair_downstairs: [ClientId(2)] }, state: ClientData([Done, InProgress, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:24.585292077Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] client skip 2 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:24.585314833Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:24.585334475Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{{"msg":""msg":"Waiting for 4 jobs (currently 3)",[1] f5357d68-1826-4d13-a77d-b2335da9d6f2 (c3b8b937-bdf2-428d-b207-29fe37e50ee9) Active Active LiveRepair ds_transition to Faulted""v":,"v0":0,","name"name"::""cruciblecrucible"",,""levellevel""::3030,"time":","time":"2023-09-22T23:08:24.585382998Z","2023-09-22T23:08:24.585383202Z"hostname":","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":ip-10-150-1-74.us-west-2.compute.internal4291","}pid"
:4291}
{"msg":"{[1] Transition from Active to Faulted",""v"msg:":0","name":"crucibleRE:0 Wait for result from NoOp command 1002:3"",","levelv""::300,"name":"crucible","level":30,"time":",2023-09-22T23:08:24.585424085Z""time",:""hostname":"2023-09-22T23:08:24.585427723Z","hostnameip-10-150-1-74.us-west-2.compute.internal"":","pid":4291ip-10-150-1-74.us-west-2.compute.internal"},
"pid":4291{}
"msg":"Now ACK the close job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:24.585456617Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for 3 jobs (currently 2)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:24.585488002Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Extent 0 close id:1000 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:24.585517133Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] client skip 2 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:24.585536791Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:24.585555774Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] f5357d68-1826-4d13-a77d-b2335da9d6f2 (c3b8b937-bdf2-428d-b207-29fe37e50ee9) Active Faulted LiveRepair ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:24.585577204Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from LiveRepair to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:24.585595156Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:24.585613296Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for 3 jobs (currently 2)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:24.593338062Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"No repair needed for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:24.59337479Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:24.59339667Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now ACK the close job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:24.706573102Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for 3 jobs (currently 2)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:24.706611247Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"No repair needed for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:24.706630314Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:24.706652727Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now move the NoOp job forward","v":0,"name":"crucible","level":30{"msg":"Waiting for 4 jobs (currently 3)","v":0,",time"":name"":"crucible"2023-09-22T23:08:25.586693211Z","level":,30"hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] DS Reports error Err(GenericError(\"bad\","time":")) on job 1002, DownstairsIO { ds_id: JobId(1002), guest_id: 3, work: ExtentLiveNoOp { dependencies: [JobId(1000), JobId(1001)] }, state: ClientData([Done, InProgress, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }"2023-09-22T23:08:25.5867399Z","v":0,","hostnamename""::""crucible","levelip-10-150-1-74.us-west-2.compute.internal"":,50"pid":4291}
{"msg":","Extent 0 close id:1001 Failed: Error: bad"time":","v":02023-09-22T23:08:25.586840291Z",",name"":"hostname":crucible"","level":ip-10-150-1-74.us-west-2.compute.internal50","pid":4291,"":"downstairs"}
{"msg":","time":"[1] Reports error GenericError(\"bad\"2023-09-22T23:08:25.586862343Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from NoOp command 1002:3") on job 1002, DownstairsIO { ds_id: JobId(1002), guest_id: 3, work: ExtentLiveNoOp { dependencies: [JobId(1000), JobId(1001)] }, state: ClientData([Done, InProgress, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) },""v,"":v"0:,0","name"name:"":"crucible"crucible",","level"level:"30:50,,""timetime""::""2023-09-22T23:08:25.586898551Z2023-09-22T23:08:25.586898295Z"",,""hostnamehostname""::""ip-10-150-1-74.us-west-2.compute.internalip-10-150-1-74.us-west-2.compute.internal"",,""pidpid""::42914291,}"
":"downstairs"}
{"msg":"[1] client skip 4 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:25.58700462Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:25.587036291Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] 9a454318-3f11-4b24-be6e-b1033eaa5db9 (78e805e3-7c37-4f81-915e-a0267c342496) Active LiveRepair Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:25.58705911Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from LiveRepair to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:25.587077153Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Extent 0 close id:1002 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:25.587095119Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from reopen command 1003:4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:25.587113181Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:25.594677711Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:25.594718124Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] DS Reports error Err(GenericError(\"bad\")) on job 1001, DownstairsIO { ds_id: JobId(1001), guest_id: 2, work: ExtentLiveNoOp { dependencies: [JobId(1000)] }, state: ClientData([InProgress, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:25.707844809Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[0] Reports error GenericError(\"bad\") on job 1001, DownstairsIO { ds_id: JobId(1001), guest_id: 2, work: ExtentLiveNoOp { dependencies: [JobId(1000)] }, state: ClientData([InProgress, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:25.707912025Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[0] client skip 3 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:25.707945426Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[0] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:25.707966454Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[0] 550be307-1aa8-42ac-b59e-9ea2749e822e (5aed1069-eb98-435d-bbfb-e191ed7d789f) Active Active LiveRepair ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:25.70798859Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:25.708007087Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:25.708024949Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Extent 0 close id:1001 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:25.708043091Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] client skip 3 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:25.708061152Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:25.708080209Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] 550be307-1aa8-42ac-b59e-9ea2749e822e (5aed1069-eb98-435d-bbfb-e191ed7d789f) Faulted Active LiveRepair ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:25.708101571Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from LiveRepair to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:25.708119611Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:25.708137777Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now move the NoOp job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.587086452Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now ACK the NoOp job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.587142775Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Finally, move the ReOpen job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.587162255Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now ACK the Reopen job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.587180206Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Extent 0 close id:1002 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:26.587198363Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from reopen command 1003:4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.58721684Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Extent 0 close id:1003 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:26.587235366Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Bailing with error","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:26.587264794Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
----------------------------------------------------------------
 Crucible gen:0 GIO:true work queues:  Upstairs:0  downstairs:4
GW_ID      ACK  DSID    TYPE  BLOCKS  DS:0  DS:1  DS:2  REPLAY
    1    Acked  1000  FClose       0  Done   Err  Done  false
    2    Acked  1001    NoOp       0  Done  Skip  Skip  false
    3    Acked  1002    NoOp       0  Done  Skip  Skip  false
    4    Acked  1003  Reopen       0  Done  Skip  Skip  false
   STATES      DS:0   DS:1   DS:2   TOTAL
    New           0      0      0      0
    Sent          0      0      0      0
    Done          4      0      1      5
    Skipped       0      3      3      6
    Error         0      1      0      1
Last Flush: 0 0 0 
Downstairs last five completed:
Upstairs last five completed:      4    3    2    1
{"msg":"Crucible stats registered with UUID: 322c5d16-5794-4781-8bc0-a7be748f5fb6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.587879458Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 322c5d16-5794-4781-8bc0-a7be748f5fb6 has session id: f9f78e0a-83f0-444c-baec-2ff3c6918482","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.58791473Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 322c5d16-5794-4781-8bc0-a7be748f5fb6 (aa3992cb-83bf-4a9e-ae74-60b531d1ace4) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.587937086Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.587955739Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 322c5d16-5794-4781-8bc0-a7be748f5fb6 (aa3992cb-83bf-4a9e-ae74-60b531d1ace4) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.587976138Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.587994393Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 322c5d16-5794-4781-8bc0-a7be748f5fb6 (aa3992cb-83bf-4a9e-ae74-60b531d1ace4) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.588014357Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.58803225Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 322c5d16-5794-4781-8bc0-a7be748f5fb6 (aa3992cb-83bf-4a9e-ae74-60b531d1ace4) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.588052222Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.588070054Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 322c5d16-5794-4781-8bc0-a7be748f5fb6 (aa3992cb-83bf-4a9e-ae74-60b531d1ace4) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.588090265Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.588108224Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 322c5d16-5794-4781-8bc0-a7be748f5fb6 (aa3992cb-83bf-4a9e-ae74-60b531d1ace4) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.588128151Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.588145831Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 322c5d16-5794-4781-8bc0-a7be748f5fb6 (aa3992cb-83bf-4a9e-ae74-60b531d1ace4) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.588173475Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.588192732Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 322c5d16-5794-4781-8bc0-a7be748f5fb6 (aa3992cb-83bf-4a9e-ae74-60b531d1ace4) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.588212953Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.588230771Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 322c5d16-5794-4781-8bc0-a7be748f5fb6 (aa3992cb-83bf-4a9e-ae74-60b531d1ace4) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.588250768Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.588268553Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"322c5d16-5794-4781-8bc0-a7be748f5fb6 is now active with session: aa3992cb-83bf-4a9e-ae74-60b531d1ace4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.588288031Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 322c5d16-5794-4781-8bc0-a7be748f5fb6 (aa3992cb-83bf-4a9e-ae74-60b531d1ace4) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.588307962Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.588325643Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 322c5d16-5794-4781-8bc0-a7be748f5fb6 (aa3992cb-83bf-4a9e-ae74-60b531d1ace4) Active Active Faulted ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.588345946Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.588363838Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 322c5d16-5794-4781-8bc0-a7be748f5fb6 (aa3992cb-83bf-4a9e-ae74-60b531d1ace4) Active Active LiveRepairReady ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.588384078Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.588402122Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for Close + ReOpen jobs","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.58841977Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.588437997Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 close id:1000 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.588455682Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from close command 1000:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.588473688Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now move the NoOp job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.595951202Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] DS Reports error Err(GenericError(\"bad\")) on job 1003, DownstairsIO { ds_id: JobId(1003), guest_id: 4, work: ExtentLiveReopen { dependencies: [JobId(1000), JobId(1001), JobId(1002)], extent: 0 }, state: ClientData([Done, InProgress, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:26.596071269Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] Reports error GenericError(\"bad\") on job 1003, DownstairsIO { ds_id: JobId(1003), guest_id: 4, work: ExtentLiveReopen { dependencies: [JobId(1000), JobId(1001), JobId(1002)], extent: 0 }, state: ClientData([Done, InProgress, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:26.596111538Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] client skip 4 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.596132534Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] changed 0 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.596151611Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] 2f3bf994-5f9f-4b02-b326-33674e795003 (0a9fd518-e87c-445f-bf08-874431d889a8) Active Active LiveRepair ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.596173101Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.596191161Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from reopen command 1003:4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.596209564Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Extent 0 close id:1003 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:26.596227747Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] client skip 4 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.596246139Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] changed 0 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.596264942Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] 2f3bf994-5f9f-4b02-b326-33674e795003 (0a9fd518-e87c-445f-bf08-874431d889a8) Active Faulted LiveRepair ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.59628716Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from LiveRepair to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.596305426Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Bailing with error","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:26.596323001Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible stats registered with UUID: 9f99bbef-a6b4-4677-9b80-a44b3f26f5ad","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.596609644Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 9f99bbef-a6b4-4677-9b80-a44b3f26f5ad has session id: 96273c4d-251b-433b-b209-311dd73d94f3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.596642525Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 9f99bbef-a6b4-4677-9b80-a44b3f26f5ad (05a7d3c1-8da7-4ab0-8e59-ef61aff4b565) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.596670271Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.5966904Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 9f99bbef-a6b4-4677-9b80-a44b3f26f5ad (05a7d3c1-8da7-4ab0-8e59-ef61aff4b565) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.59671124Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.596729541Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 9f99bbef-a6b4-4677-9b80-a44b3f26f5ad (05a7d3c1-8da7-4ab0-8e59-ef61aff4b565) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.596752726Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.596771674Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 9f99bbef-a6b4-4677-9b80-a44b3f26f5ad (05a7d3c1-8da7-4ab0-8e59-ef61aff4b565) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.596791618Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.596809469Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 9f99bbef-a6b4-4677-9b80-a44b3f26f5ad (05a7d3c1-8da7-4ab0-8e59-ef61aff4b565) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.596829669Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.596847586Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 9f99bbef-a6b4-4677-9b80-a44b3f26f5ad (05a7d3c1-8da7-4ab0-8e59-ef61aff4b565) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.596874986Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.596901297Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 9f99bbef-a6b4-4677-9b80-a44b3f26f5ad (05a7d3c1-8da7-4ab0-8e59-ef61aff4b565) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.596922693Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.596940712Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 9f99bbef-a6b4-4677-9b80-a44b3f26f5ad (05a7d3c1-8da7-4ab0-8e59-ef61aff4b565) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.596961224Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.596979361Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 9f99bbef-a6b4-4677-9b80-a44b3f26f5ad (05a7d3c1-8da7-4ab0-8e59-ef61aff4b565) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.596999548Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.597017448Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"9f99bbef-a6b4-4677-9b80-a44b3f26f5ad is now active with session: 05a7d3c1-8da7-4ab0-8e59-ef61aff4b565","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.597041718Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 9f99bbef-a6b4-4677-9b80-a44b3f26f5ad (05a7d3c1-8da7-4ab0-8e59-ef61aff4b565) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.597062947Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.597080792Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 9f99bbef-a6b4-4677-9b80-a44b3f26f5ad (05a7d3c1-8da7-4ab0-8e59-ef61aff4b565) Active Active Faulted ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.597101134Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.597119176Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 9f99bbef-a6b4-4677-9b80-a44b3f26f5ad (05a7d3c1-8da7-4ab0-8e59-ef61aff4b565) Active Active LiveRepairReady ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.597139422Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.597157654Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for Close + ReOpen jobs","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.597175281Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.597193583Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 close id:1000 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.597211351Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from close command 1000:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.597229463Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now move the NoOp job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.708295399Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now ACK the NoOp job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.708329042Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Finally, move the ReOpen job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.708348164Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now ACK the Reopen job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.708365749Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Extent 0 close id:1002 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:26.70838374Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from reopen command 1003:4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.708402381Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Extent 0 close id:1003 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:26.708420342Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Bailing with error","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:26.708437942Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible stats registered with UUID: b90a2e95-f256-40ff-a6c3-b578ad79bb91","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.708867365Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible b90a2e95-f256-40ff-a6c3-b578ad79bb91 has session id: b423ec84-3a14-4a06-b44f-c9ac8aa2686b","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.708901423Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] b90a2e95-f256-40ff-a6c3-b578ad79bb91 (70f32c98-85ac-4a49-8e97-4e66c551292d) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.708923463Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.708941942Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] b90a2e95-f256-40ff-a6c3-b578ad79bb91 (70f32c98-85ac-4a49-8e97-4e66c551292d) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.708962269Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.708980406Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] b90a2e95-f256-40ff-a6c3-b578ad79bb91 (70f32c98-85ac-4a49-8e97-4e66c551292d) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.70900035Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.709018409Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] b90a2e95-f256-40ff-a6c3-b578ad79bb91 (70f32c98-85ac-4a49-8e97-4e66c551292d) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.709038506Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.709056263Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] b90a2e95-f256-40ff-a6c3-b578ad79bb91 (70f32c98-85ac-4a49-8e97-4e66c551292d) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.709076443Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.709094325Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] b90a2e95-f256-40ff-a6c3-b578ad79bb91 (70f32c98-85ac-4a49-8e97-4e66c551292d) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.709114239Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.709131931Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] b90a2e95-f256-40ff-a6c3-b578ad79bb91 (70f32c98-85ac-4a49-8e97-4e66c551292d) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.709152099Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.709169951Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] b90a2e95-f256-40ff-a6c3-b578ad79bb91 (70f32c98-85ac-4a49-8e97-4e66c551292d) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.70919003Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.709208024Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] b90a2e95-f256-40ff-a6c3-b578ad79bb91 (70f32c98-85ac-4a49-8e97-4e66c551292d) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.709233997Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.709253378Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"b90a2e95-f256-40ff-a6c3-b578ad79bb91 is now active with session: 70f32c98-85ac-4a49-8e97-4e66c551292d","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.709272712Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] b90a2e95-f256-40ff-a6c3-b578ad79bb91 (70f32c98-85ac-4a49-8e97-4e66c551292d) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.709292662Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.709310312Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] b90a2e95-f256-40ff-a6c3-b578ad79bb91 (70f32c98-85ac-4a49-8e97-4e66c551292d) Active Active Faulted ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.709330571Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.709348419Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] b90a2e95-f256-40ff-a6c3-b578ad79bb91 (70f32c98-85ac-4a49-8e97-4e66c551292d) Active Active LiveRepairReady ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.709368489Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.709386556Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for Close + ReOpen jobs","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.709404286Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.709422703Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 close id:1000 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.709440379Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from close command 1000:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:26.709458391Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Finally, move the ReOpen job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:27.588116073Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now ACK the reopen job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:27.58817006Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Bailing with error","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:27.588190072Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible stats registered with UUID: b122e61b-d947-42d8-bfa7-060e8c7d400f","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:27.588779913Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible b122e61b-d947-42d8-bfa7-060e8c7d400f has session id: 30c9ce5b-de1a-48b2-8413-140dc7077875","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:27.58881371Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] b122e61b-d947-42d8-bfa7-060e8c7d400f (bacc12da-7d6f-49e4-a766-3a01fd8288ee) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:27.588845398Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:27.588865911Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] b122e61b-d947-42d8-bfa7-060e8c7d400f (bacc12da-7d6f-49e4-a766-3a01fd8288ee) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:27.588886551Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:27.588904616Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] b122e61b-d947-42d8-bfa7-060e8c7d400f (bacc12da-7d6f-49e4-a766-3a01fd8288ee) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:27.588924754Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:27.588942635Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] b122e61b-d947-42d8-bfa7-060e8c7d400f (bacc12da-7d6f-49e4-a766-3a01fd8288ee) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:27.588962818Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:27.58898063Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] b122e61b-d947-42d8-bfa7-060e8c7d400f (bacc12da-7d6f-49e4-a766-3a01fd8288ee) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:27.589001016Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:27.589018958Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] b122e61b-d947-42d8-bfa7-060e8c7d400f (bacc12da-7d6f-49e4-a766-3a01fd8288ee) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:27.589039092Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:27.589057029Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] b122e61b-d947-42d8-bfa7-060e8c7d400f (bacc12da-7d6f-49e4-a766-3a01fd8288ee) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:27.589077118Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:27.589094937Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] b122e61b-d947-42d8-bfa7-060e8c7d400f (bacc12da-7d6f-49e4-a766-3a01fd8288ee) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:27.589115081Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:27.58913302Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] b122e61b-d947-42d8-bfa7-060e8c7d400f (bacc12da-7d6f-49e4-a766-3a01fd8288ee) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:27.589153132Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:27.589170977Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"b122e61b-d947-42d8-bfa7-060e8c7d400f is now active with session: bacc12da-7d6f-49e4-a766-3a01fd8288ee","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:27.589195495Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] b122e61b-d947-42d8-bfa7-060e8c7d400f (bacc12da-7d6f-49e4-a766-3a01fd8288ee) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:27.589216623Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:27.589234698Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] b122e61b-d947-42d8-bfa7-060e8c7d400f (bacc12da-7d6f-49e4-a766-3a01fd8288ee) Active Faulted Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:27.589254999Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:27.589272869Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] b122e61b-d947-42d8-bfa7-060e8c7d400f (bacc12da-7d6f-49e4-a766-3a01fd8288ee) Active LiveRepairReady Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:27.589292974Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:27.589310915Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for Close + ReOpen jobs","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:27.589339026Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:27.589374981Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 close id:1000 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:27.589411396Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from close command 1000:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:27.589447002Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] DS Reports error Err(GenericError(\"bad\")) on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentFlushClose { dependencies: [], extent: 0, flush_number: 0, gen_number: 0, source_downstairs: ClientId(0), repair_downstairs: [ClientId(2)] }, state: ClientData([Done, Done, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:27.589573085Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] Reports error GenericError(\"bad\") on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentFlushClose { dependencies: [], extent: 0, flush_number: 0, gen_number: 0, source_downstairs: ClientId(0), repair_downstairs: [ClientId(2)] }, state: ClientData([Done, Done, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:27.589624619Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] client skip 2 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:27.589647793Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:27.589674278Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] 322c5d16-5794-4781-8bc0-a7be748f5fb6 (aa3992cb-83bf-4a9e-ae74-60b531d1ace4) Active Active LiveRepair ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:27.589697948Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from LiveRepair to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:27.589716631Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now ACK the close job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:27.589734182Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for 3 jobs (currently 2)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:27.589752052Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Extent 0 close id:1000 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:27.589770084Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:27.589788069Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for 3 jobs (currently 2)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:27.598217088Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"No repair needed for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:27.598249086Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:27.598270344Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now ACK the close job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:27.710325764Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for 3 jobs (currently 2)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:27.710360947Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"No repair needed for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:27.710379814Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:27.710399972Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for 3 jobs (currently 2)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:28.589369735Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"No repair needed for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:28.58944961Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:28.589478212Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:28.589948731Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Extent 0 close id:1001 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:28.589981129Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:28.59000067Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:28.59953214Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:28.599572407Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] DS Reports error Err(GenericError(\"bad\")) on job 1001, DownstairsIO { ds_id: JobId(1001), guest_id: 2, work: ExtentLiveNoOp { dependencies: [JobId(1000)] }, state: ClientData([Done, InProgress, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:28.711658026Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] Reports error GenericError(\"bad\") on job 1001, DownstairsIO { ds_id: JobId(1001), guest_id: 2, work: ExtentLiveNoOp { dependencies: [JobId(1000)] }, state: ClientData([Done, InProgress, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:28.711700652Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] client skip 3 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:28.711721466Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:28.711740703Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] b90a2e95-f256-40ff-a6c3-b578ad79bb91 (70f32c98-85ac-4a49-8e97-4e66c551292d) Active Active LiveRepair ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:28.711762354Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:28.711797373Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:28.71181579Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Extent 0 close id:1001 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:28.711833848Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] client skip 3 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:28.711851783Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:28.711870652Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] b90a2e95-f256-40ff-a6c3-b578ad79bb91 (70f32c98-85ac-4a49-8e97-4e66c551292d) Active Faulted LiveRepair ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:28.711891901Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from LiveRepair to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:28.711909862Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:28.711928034Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:29.590724662Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:29.590790643Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now move the NoOp job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:29.591224322Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now ACK the NoOp job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:29.591259865Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Finally, move the ReOpen job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:29.591280044Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now ACK the Reopen job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:29.591298058Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Extent 0 close id:1002 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:29.591316342Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from reopen command 1003:4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:29.591334805Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Extent 0 close id:1003 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:29.59135295Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Bailing with error","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:29.591371239Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
----------------------------------------------------------------
 Crucible gen:0 GIO:true work queues:  Upstairs:0  downstairs:4
GW_ID      ACK  DSID    TYPE  BLOCKS  DS:0  DS:1  DS:2  REPLAY
    1    Acked  1000  FClose       0  Done  Done   Err  false
    2    Acked  1001    NoOp       0  Done  Done  Skip  false
    3    Acked  1002    NoOp       0  Done  Done  Skip  false
    4    Acked  1003  Reopen       0  Done  Done  Skip  false
   STATES      DS:0   DS:1   DS:2   TOTAL
    New           0      0      0      0
    Sent          0      0      0      0
    Done          4      4      0      8
    Skipped       0      0      3      3
    Error         0      0      1      1
Last Flush: 0 0 0 
Downstairs last five completed:
Upstairs last five completed:      4    3    2    1
test live_repair::repair_test::test_repair_extent_close_fails_all ... ok
{"msg":"Now move the NoOp job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:29.600800884Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] DS Reports error Err(GenericError(\"bad\")) on job 1003, DownstairsIO { ds_id: JobId(1003), guest_id: 4, work: ExtentLiveReopen { dependencies: [JobId(1000), JobId(1001), JobId(1002)], extent: 0 }, state: ClientData([Done, Done, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:29.600922156Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] Reports error GenericError(\"bad\") on job 1003, DownstairsIO { ds_id: JobId(1003), guest_id: 4, work: ExtentLiveReopen { dependencies: [JobId(1000), JobId(1001), JobId(1002)], extent: 0 }, state: ClientData([Done, Done, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:29.60096208Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] client skip 4 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:29.600983203Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] changed 0 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:29.60100256Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] 9f99bbef-a6b4-4677-9b80-a44b3f26f5ad (05a7d3c1-8da7-4ab0-8e59-ef61aff4b565) Active Active LiveRepair ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:29.601032878Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from LiveRepair to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:29.601052852Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from reopen command 1003:4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:29.601071267Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Extent 0 close id:1003 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:29.601089383Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Bailing with error","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:29.601107041Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test live_repair::repair_test::test_repair_extent_fail_reopen_all ... ok
{"msg":"Now move the NoOp job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:29.712010529Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now ACK the NoOp job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:29.712045564Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Finally, move the ReOpen job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:29.712064826Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now ACK the Reopen job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:29.712082378Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Extent 0 close id:1002 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:29.7121006Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from reopen command 1003:4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:29.712119254Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Extent 0 close id:1003 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:29.712137208Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Bailing with error","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:29.712154829Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible stats registered with UUID: bd11c632-367b-41e3-be1c-5e1fdf25ae23","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:29.712638031Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible bd11c632-367b-41e3-be1c-5e1fdf25ae23 has session id: 4f378124-a171-4d44-b978-e414771e8790","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:29.712671449Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] bd11c632-367b-41e3-be1c-5e1fdf25ae23 (a74b0c5b-941d-45d5-babc-e7b7f16307d0) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:29.712693589Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:29.712712311Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] bd11c632-367b-41e3-be1c-5e1fdf25ae23 (a74b0c5b-941d-45d5-babc-e7b7f16307d0) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:29.71277366Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:29.712795981Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] bd11c632-367b-41e3-be1c-5e1fdf25ae23 (a74b0c5b-941d-45d5-babc-e7b7f16307d0) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:29.712816016Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:29.712841569Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] bd11c632-367b-41e3-be1c-5e1fdf25ae23 (a74b0c5b-941d-45d5-babc-e7b7f16307d0) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:29.712863052Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:29.712881082Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] bd11c632-367b-41e3-be1c-5e1fdf25ae23 (a74b0c5b-941d-45d5-babc-e7b7f16307d0) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:29.712901617Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:29.712919552Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] bd11c632-367b-41e3-be1c-5e1fdf25ae23 (a74b0c5b-941d-45d5-babc-e7b7f16307d0) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:29.712939492Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:29.712957299Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] bd11c632-367b-41e3-be1c-5e1fdf25ae23 (a74b0c5b-941d-45d5-babc-e7b7f16307d0) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:29.712977319Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:29.712995054Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] bd11c632-367b-41e3-be1c-5e1fdf25ae23 (a74b0c5b-941d-45d5-babc-e7b7f16307d0) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:29.713015022Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:29.713032963Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] bd11c632-367b-41e3-be1c-5e1fdf25ae23 (a74b0c5b-941d-45d5-babc-e7b7f16307d0) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:29.71305294Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:29.713070673Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"bd11c632-367b-41e3-be1c-5e1fdf25ae23 is now active with session: a74b0c5b-941d-45d5-babc-e7b7f16307d0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:29.713090104Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] bd11c632-367b-41e3-be1c-5e1fdf25ae23 (a74b0c5b-941d-45d5-babc-e7b7f16307d0) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:29.713110107Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:29.713127786Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] bd11c632-367b-41e3-be1c-5e1fdf25ae23 (a74b0c5b-941d-45d5-babc-e7b7f16307d0) Active Active Faulted ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:29.713147982Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:29.7131714Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] bd11c632-367b-41e3-be1c-5e1fdf25ae23 (a74b0c5b-941d-45d5-babc-e7b7f16307d0) Active Active LiveRepairReady ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:29.713192905Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:29.71321106Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for Close + ReOpen jobs","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:29.713228807Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:29.713247125Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 close id:1000 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:29.713264894Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from close command 1000:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:29.713282746Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now move the NoOp job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:30.592052272Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] DS Reports error Err(GenericError(\"bad\")) on job 1002, DownstairsIO { ds_id: JobId(1002), guest_id: 3, work: ExtentLiveNoOp { dependencies: [JobId(1000), JobId(1001)] }, state: ClientData([Done, Done, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:30.59212328Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] Reports error GenericError(\"bad\") on job 1002, DownstairsIO { ds_id: JobId(1002), guest_id: 3, work: ExtentLiveNoOp { dependencies: [JobId(1000), JobId(1001)] }, state: ClientData([Done, Done, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:30.592156896Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] client skip 4 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:30.592177446Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:30.592197042Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] b122e61b-d947-42d8-bfa7-060e8c7d400f (bacc12da-7d6f-49e4-a766-3a01fd8288ee) Active LiveRepair Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:30.59221881Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:30.592237006Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Extent 0 close id:1002 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:30.592255104Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] client skip 4 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:30.592273222Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:30.592301937Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] b122e61b-d947-42d8-bfa7-060e8c7d400f (bacc12da-7d6f-49e4-a766-3a01fd8288ee) Active LiveRepair Faulted ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:30.592324903Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from LiveRepair to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:30.592342944Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from reopen command 1003:4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:30.592361206Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now ACK the close job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:30.713249424Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for 3 jobs (currently 2)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:30.713286347Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"No repair needed for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:30.713305551Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:30.713330312Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] DS Reports error Err(GenericError(\"bad\")) on job 1001, DownstairsIO { ds_id: JobId(1001), guest_id: 2, work: ExtentLiveNoOp { dependencies: [JobId(1000)] }, state: ClientData([Done, Done, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:31.714696694Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] Reports error GenericError(\"bad\") on job 1001, DownstairsIO { ds_id: JobId(1001), guest_id: 2, work: ExtentLiveNoOp { dependencies: [JobId(1000)] }, state: ClientData([Done, Done, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:31.714766232Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] client skip 3 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:31.714788486Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:31.714807736Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] bd11c632-367b-41e3-be1c-5e1fdf25ae23 (a74b0c5b-941d-45d5-babc-e7b7f16307d0) Active Active LiveRepair ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:31.71482934Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from LiveRepair to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:31.714848134Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:31.714865761Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Extent 0 close id:1001 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:31.714883824Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:31.714911878Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:32.42805873Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:32.429526239Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"Finally, move the ReOpen job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:32.593479723Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now ACK the reopen job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:32.593510975Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Bailing with error","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:32.593540406Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"err:2 or:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:32.593728373Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible stats registered with UUID: 5c83a6ac-c2a3-4701-911d-c42a2b3e2a6d","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:32.594178557Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 5c83a6ac-c2a3-4701-911d-c42a2b3e2a6d has session id: 760323dd-3f10-4c50-b9bd-60d87e8dbf17","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:32.594214312Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 5c83a6ac-c2a3-4701-911d-c42a2b3e2a6d (e3e27907-0d47-47a5-8211-bf45df96075d) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:32.59423699Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:32.594255757Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 5c83a6ac-c2a3-4701-911d-c42a2b3e2a6d (e3e27907-0d47-47a5-8211-bf45df96075d) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:32.59427621Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:32.594294564Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 5c83a6ac-c2a3-4701-911d-c42a2b3e2a6d (e3e27907-0d47-47a5-8211-bf45df96075d) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:32.594314512Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:32.594332461Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 5c83a6ac-c2a3-4701-911d-c42a2b3e2a6d (e3e27907-0d47-47a5-8211-bf45df96075d) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:32.59435244Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:32.594370276Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 5c83a6ac-c2a3-4701-911d-c42a2b3e2a6d (e3e27907-0d47-47a5-8211-bf45df96075d) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:32.594390403Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:32.594408415Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 5c83a6ac-c2a3-4701-911d-c42a2b3e2a6d (e3e27907-0d47-47a5-8211-bf45df96075d) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:32.594440581Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:32.59446023Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 5c83a6ac-c2a3-4701-911d-c42a2b3e2a6d (e3e27907-0d47-47a5-8211-bf45df96075d) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:32.594480404Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:32.594498178Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 5c83a6ac-c2a3-4701-911d-c42a2b3e2a6d (e3e27907-0d47-47a5-8211-bf45df96075d) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:32.594518439Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:32.594536366Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 5c83a6ac-c2a3-4701-911d-c42a2b3e2a6d (e3e27907-0d47-47a5-8211-bf45df96075d) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:32.594556356Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:32.594574233Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"5c83a6ac-c2a3-4701-911d-c42a2b3e2a6d is now active with session: e3e27907-0d47-47a5-8211-bf45df96075d","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:32.594593624Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 5c83a6ac-c2a3-4701-911d-c42a2b3e2a6d (e3e27907-0d47-47a5-8211-bf45df96075d) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:32.594613619Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:32.594633095Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 5c83a6ac-c2a3-4701-911d-c42a2b3e2a6d (e3e27907-0d47-47a5-8211-bf45df96075d) Active Active Faulted ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:32.594666383Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:32.594687023Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 5c83a6ac-c2a3-4701-911d-c42a2b3e2a6d (e3e27907-0d47-47a5-8211-bf45df96075d) Active Active LiveRepairReady ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:32.594707178Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:32.594725094Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for Close + ReOpen jobs","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:32.594742738Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:32.594761098Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 close id:1000 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:32.594778797Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from close command 1000:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:32.59480248Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now move the NoOp job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:32.715036423Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now ACK the NoOp job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:32.715072873Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Finally, move the ReOpen job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:32.715092949Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now ACK the Reopen job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:32.71511089Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Extent 0 close id:1002 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:32.715129656Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from reopen command 1003:4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:32.715148384Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Extent 0 close id:1003 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:32.715166662Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Bailing with error","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:32.715184346Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test live_repair::repair_test::test_repair_extent_repair_fails_all ... ok
{"msg":"Waiting for 3 jobs (currently 2)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:33.595760757Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"No repair needed for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:33.59581707Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:33.59583947Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:34.596114407Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:34.596170891Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now move the NoOp job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:35.59747985Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] DS Reports error Err(GenericError(\"bad\")) on job 1002, DownstairsIO { ds_id: JobId(1002), guest_id: 3, work: ExtentLiveNoOp { dependencies: [JobId(1000), JobId(1001)] }, state: ClientData([InProgress, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:35.597547906Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[0] Reports error GenericError(\"bad\") on job 1002, DownstairsIO { ds_id: JobId(1002), guest_id: 3, work: ExtentLiveNoOp { dependencies: [JobId(1000), JobId(1001)] }, state: ClientData([InProgress, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:35.597598349Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[0] client skip 4 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:35.597631571Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[0] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:35.597652677Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[0] 5c83a6ac-c2a3-4701-911d-c42a2b3e2a6d (e3e27907-0d47-47a5-8211-bf45df96075d) Active Active LiveRepair ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:35.597674616Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:35.597692722Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Extent 0 close id:1002 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:35.597710692Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] client skip 4 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:35.59772902Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:35.597747977Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] 5c83a6ac-c2a3-4701-911d-c42a2b3e2a6d (e3e27907-0d47-47a5-8211-bf45df96075d) Faulted Active LiveRepair ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:35.597769308Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from LiveRepair to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:35.597787291Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from reopen command 1003:4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:35.59780521Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Finally, move the ReOpen job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:37.597881117Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now ACK the reopen job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:37.597937406Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Bailing with error","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:37.597956955Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"err:0 or:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:37.598095409Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible stats registered with UUID: fe831f8d-4eb5-44ca-a62d-c61c3e0540f9","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:37.598728487Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible fe831f8d-4eb5-44ca-a62d-c61c3e0540f9 has session id: b65f4444-6904-40f8-8cb2-293cdf26dc74","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:37.598768661Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] fe831f8d-4eb5-44ca-a62d-c61c3e0540f9 (826d5ec6-dae1-4ee1-a079-b1a0132403d0) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:37.598792306Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:37.598811359Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] fe831f8d-4eb5-44ca-a62d-c61c3e0540f9 (826d5ec6-dae1-4ee1-a079-b1a0132403d0) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:37.598832073Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:37.598861052Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] fe831f8d-4eb5-44ca-a62d-c61c3e0540f9 (826d5ec6-dae1-4ee1-a079-b1a0132403d0) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:37.598882807Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:37.598900867Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] fe831f8d-4eb5-44ca-a62d-c61c3e0540f9 (826d5ec6-dae1-4ee1-a079-b1a0132403d0) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:37.598920912Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:37.598938858Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] fe831f8d-4eb5-44ca-a62d-c61c3e0540f9 (826d5ec6-dae1-4ee1-a079-b1a0132403d0) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:37.5989592Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:37.598977586Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] fe831f8d-4eb5-44ca-a62d-c61c3e0540f9 (826d5ec6-dae1-4ee1-a079-b1a0132403d0) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:37.598997677Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:37.59901557Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] fe831f8d-4eb5-44ca-a62d-c61c3e0540f9 (826d5ec6-dae1-4ee1-a079-b1a0132403d0) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:37.59903575Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:37.599053693Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] fe831f8d-4eb5-44ca-a62d-c61c3e0540f9 (826d5ec6-dae1-4ee1-a079-b1a0132403d0) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:37.599073939Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:37.599091975Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] fe831f8d-4eb5-44ca-a62d-c61c3e0540f9 (826d5ec6-dae1-4ee1-a079-b1a0132403d0) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:37.599112058Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:37.599129822Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"fe831f8d-4eb5-44ca-a62d-c61c3e0540f9 is now active with session: 826d5ec6-dae1-4ee1-a079-b1a0132403d0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:37.599149176Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] fe831f8d-4eb5-44ca-a62d-c61c3e0540f9 (826d5ec6-dae1-4ee1-a079-b1a0132403d0) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:37.599169236Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:37.599187231Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] fe831f8d-4eb5-44ca-a62d-c61c3e0540f9 (826d5ec6-dae1-4ee1-a079-b1a0132403d0) Active Active Faulted ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:37.5992123Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:37.599231536Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] fe831f8d-4eb5-44ca-a62d-c61c3e0540f9 (826d5ec6-dae1-4ee1-a079-b1a0132403d0) Active Active LiveRepairReady ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:37.599251937Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:37.599270217Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for Close + ReOpen jobs","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:37.599287917Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:37.599306274Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 close id:1000 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:37.599324069Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from close command 1000:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:37.599342384Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for 3 jobs (currently 2)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:38.600315847Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"No repair needed for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:38.600371435Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:38.60039319Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:39.601752599Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:39.60180804Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now move the NoOp job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:40.603049459Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] DS Reports error Err(GenericError(\"bad\")) on job 1002, DownstairsIO { ds_id: JobId(1002), guest_id: 3, work: ExtentLiveNoOp { dependencies: [JobId(1000), JobId(1001)] }, state: ClientData([Done, InProgress, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:40.603118754Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] Reports error GenericError(\"bad\") on job 1002, DownstairsIO { ds_id: JobId(1002), guest_id: 3, work: ExtentLiveNoOp { dependencies: [JobId(1000), JobId(1001)] }, state: ClientData([Done, InProgress, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:40.603151193Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] client skip 4 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:40.603182756Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:40.603203458Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[1] fe831f8d-4eb5-44ca-a62d-c61c3e0540f9 (826d5ec6-dae1-4ee1-a079-b1a0132403d0) Active Active LiveRepair ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:40.603225339Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:40.603243273Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Extent 0 close id:1002 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:40.603261303Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] client skip 4 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:40.603279644Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:40.603298618Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] fe831f8d-4eb5-44ca-a62d-c61c3e0540f9 (826d5ec6-dae1-4ee1-a079-b1a0132403d0) Active Faulted LiveRepair ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:40.603319924Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from LiveRepair to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:40.603337942Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from reopen command 1003:4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:40.60339978Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:42.430667666Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:42.433585042Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"Finally, move the ReOpen job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:42.604410414Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now ACK the reopen job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:42.604451236Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Bailing with error","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:42.604472204Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"err:1 or:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:42.604650275Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible stats registered with UUID: 67cad85b-0cae-4869-903a-ed4f571a48bf","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:42.605098436Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Crucible 67cad85b-0cae-4869-903a-ed4f571a48bf has session id: 8b541110-c8e0-4fbc-af9b-0181a0db9f0d","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:42.605132998Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 67cad85b-0cae-4869-903a-ed4f571a48bf (33ea95ab-5a5e-42bf-8c5f-88a32110e72d) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:42.605155389Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:42.605174004Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 67cad85b-0cae-4869-903a-ed4f571a48bf (33ea95ab-5a5e-42bf-8c5f-88a32110e72d) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:42.605207176Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:42.605226959Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] 67cad85b-0cae-4869-903a-ed4f571a48bf (33ea95ab-5a5e-42bf-8c5f-88a32110e72d) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:42.605246914Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:42.605264759Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 67cad85b-0cae-4869-903a-ed4f571a48bf (33ea95ab-5a5e-42bf-8c5f-88a32110e72d) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:42.605305222Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:42.605323457Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 67cad85b-0cae-4869-903a-ed4f571a48bf (33ea95ab-5a5e-42bf-8c5f-88a32110e72d) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:42.605343647Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:42.605361605Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] 67cad85b-0cae-4869-903a-ed4f571a48bf (33ea95ab-5a5e-42bf-8c5f-88a32110e72d) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:42.605381589Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:42.605399537Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 67cad85b-0cae-4869-903a-ed4f571a48bf (33ea95ab-5a5e-42bf-8c5f-88a32110e72d) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:42.605419622Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:42.605437292Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 67cad85b-0cae-4869-903a-ed4f571a48bf (33ea95ab-5a5e-42bf-8c5f-88a32110e72d) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:42.605457557Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:42.605475456Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 67cad85b-0cae-4869-903a-ed4f571a48bf (33ea95ab-5a5e-42bf-8c5f-88a32110e72d) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:42.605495567Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:42.605513333Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"67cad85b-0cae-4869-903a-ed4f571a48bf is now active with session: 33ea95ab-5a5e-42bf-8c5f-88a32110e72d","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:42.605532595Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 67cad85b-0cae-4869-903a-ed4f571a48bf (33ea95ab-5a5e-42bf-8c5f-88a32110e72d) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:42.60555798Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:42.605576879Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 67cad85b-0cae-4869-903a-ed4f571a48bf (33ea95ab-5a5e-42bf-8c5f-88a32110e72d) Active Active Faulted ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:42.605597128Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:42.6056152Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] 67cad85b-0cae-4869-903a-ed4f571a48bf (33ea95ab-5a5e-42bf-8c5f-88a32110e72d) Active Active LiveRepairReady ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:42.60563589Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:42.605654051Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for Close + ReOpen jobs","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:42.605671771Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:42.605690221Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 close id:1000 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:42.605708095Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from close command 1000:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:42.605726246Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for 3 jobs (currently 2)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:43.605757825Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"No repair needed for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:43.605813737Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:43.605835136Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:44.607117709Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:44.607174001Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now move the NoOp job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:45.608410458Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] DS Reports error Err(GenericError(\"bad\")) on job 1002, DownstairsIO { ds_id: JobId(1002), guest_id: 3, work: ExtentLiveNoOp { dependencies: [JobId(1000), JobId(1001)] }, state: ClientData([Done, Done, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:45.608477403Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] Reports error GenericError(\"bad\") on job 1002, DownstairsIO { ds_id: JobId(1002), guest_id: 3, work: ExtentLiveNoOp { dependencies: [JobId(1000), JobId(1001)] }, state: ClientData([Done, Done, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:45.608511069Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] client skip 4 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:45.608540203Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:45.608561158Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs"}
{"msg":"[2] 67cad85b-0cae-4869-903a-ed4f571a48bf (33ea95ab-5a5e-42bf-8c5f-88a32110e72d) Active Active LiveRepair ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:45.608582861Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[2] Transition from LiveRepair to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:45.60860098Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Extent 0 close id:1002 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:08:45.60861934Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Wait for result from reopen command 1003:4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:45.608637639Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Finally, move the ReOpen job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:47.609758736Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"Now ACK the reopen job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:47.609816442Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"RE:0 Bailing with error","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:08:47.609835961Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
test live_repair::repair_test::test_repair_extent_fail_noop_all ... ok
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:52.433942348Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:08:52.436669352Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
test dummy_downstairs_tests::protocol_test::test_error_during_live_repair_no_halt has been running for over 60 seconds
test dummy_downstairs_tests::protocol_test::test_successful_live_repair has been running for over 60 seconds
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:09:02.438130403Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:09:02.439887001Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible"{,"level":30"msg":"responded to ping","v":0,"name":"crucible","level":30,",time"":"time":"2023-09-22T23:09:12.440106039Z"2023-09-22T23:09:12.440096784Z",,""hostname":"hostname":"ip-10-150-1-74.us-west-2.compute.internal"ip-10-150-1-74.us-west-2.compute.internal",",pid":"4291pid":4291,"downstairs":,"downstairs":11}}

{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:09:22.44128292Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:09:22.443253552Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:09:32.44472766Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:09:32.449662755Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:09:42.450351755Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:09:42.45603341Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:09:52.453072296Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:09:52.463489641Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:10:02.453834668Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:10:02.470553522Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:10:12.456049838Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:10:12.471140238Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:10:22.457821177Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:10:22.471708314Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:10:32.463979352Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:10:32.477792438Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:10:42.470380686Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:10:42.485251425Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:10:52.477709367Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:10:52.485754799Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:11:02.478378716Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:11:02.495275161Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:11:12.479471887Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:11:12.501640624Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:11:22.487408066Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:11:22.506295072Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:11:32.495260776Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:11:32.506508824Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:11:42.504916727Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:11:42.515789374Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:11:52.515056891Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:11:52.524422778Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:12:02.523822232Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:12:02.534203295Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:12:12.525257357Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:12:12.541700905Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:12:22.525847855Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:12:22.549231986Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:12:32.536524336Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:12:32.555561006Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:12:42.540867903Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:12:42.564607868Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:12:52.552213333Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:12:52.572135492Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:13:02.560503503Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:13:02.582962602Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:13:12.562508388Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:13:12.583828876Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:13:22.565175291Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:13:22.586404983Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:13:32.57101666Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:13:32.592717721Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:13:42.573090033Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:13:42.597868754Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:13:52.586564231Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:13:52.600154005Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:02.600067606Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:02.602976077Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"[up] downstairs 0 failed, too many outstanding jobs 57001","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:03.999904019Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] client skip 57001 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:03.999964581Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] changed 57001 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:04.069859544Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 024c3783-b9b4-4453-823f-769dad90d4f2 (db076280-2529-45a1-a093-a3cd3f0799dc) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:04.158309084Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:04.158350638Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] flow control end ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:04.253517258Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[up] downstairs 0 failed, too many outstanding jobs 57001","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:11.221534538Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] client skip 57001 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:11.221596325Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] changed 57001 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:11.286809652Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] df6648fa-e480-4dd9-b7c4-7fec45dbddf8 (b793572f-19dc-423b-877e-0f0cfe9147d9) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:11.371940489Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:11.371981902Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] flow control end ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:11.372367Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:12.604276763Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"[0] will exit pm_task, this downstairs Faulted","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:12.604523851Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"127.0.0.1:39941: proc: [0] client work task ended, Ok(Err([0] This downstairs now in Faulted)), so we end too","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:14:12.604664608Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"looper":"0","upstairs":1}
{"msg":"[0] df6648fa-e480-4dd9-b7c4-7fec45dbddf8 Gone missing, transition from Faulted to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:12.604695747Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] df6648fa-e480-4dd9-b7c4-7fec45dbddf8 connection to 127.0.0.1:39941 closed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:12.604717029Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"looper":"0","upstairs":1}
{"msg":"[0] 127.0.0.1:39941 task reports connection:false","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:12.604737764Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"df6648fa-e480-4dd9-b7c4-7fec45dbddf8 Faulted Active Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:12.604757631Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] 127.0.0.1:39941 task reports offline","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:12.60477681Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"spawn_message_receiver saw disconnect, bailing","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:14:12.604927816Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:16.166109869Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"[0] Dropping job 1000, this downstairs is faulted","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:14:16.166170999Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"process_ds_operation error: No longer active","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:14:16.166192653Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] will exit pm_task, this downstairs Faulted","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:16.166212212Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"127.0.0.1:52643: proc: [0] client work task ended, Ok(Err([0] This downstairs now in Faulted)), so we end too","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:14:16.166233565Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"looper":"0","upstairs":1}
{"msg":"[0] 024c3783-b9b4-4453-823f-769dad90d4f2 Gone missing, transition from Faulted to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:16.166255941Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] 024c3783-b9b4-4453-823f-769dad90d4f2 connection to 127.0.0.1:52643 closed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:16.166285752Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"looper":"0","upstairs":1}
{"msg":"[0] 127.0.0.1:52643 task reports connection:false","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:16.166308235Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"024c3783-b9b4-4453-823f-769dad90d4f2 Faulted Active Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:16.166328202Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] 127.0.0.1:52643 task reports offline","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:16.166347284Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"spawn_message_receiver saw disconnect, bailing","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:14:16.166366402Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"could not send read response for job 3 = 1003: Broken pipe (os error 32)","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:14:16.166719959Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:16.928216467Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":2}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:16.928276714Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":3}
{"msg":"[0] 024c3783-b9b4-4453-823f-769dad90d4f2 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:17.166841032Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"looper":"0","upstairs":1}
{"msg":"[0] Proc runs for 127.0.0.1:52643 in state Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:17.166893333Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"negotiate packet HereIAm { version: 4, upstairs_id: 024c3783-b9b4-4453-823f-769dad90d4f2, session_id: db076280-2529-45a1-a093-a3cd3f0799dc, gen: 1, read_only: false, encrypted: false, alternate_versions: [] }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:17.166992284Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"[0] upstairs guest_io_ready=TRUE, promote! session db076280-2529-45a1-a093-a3cd3f0799dc","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:17.167102495Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"negotiate packet PromoteToActive { upstairs_id: 024c3783-b9b4-4453-823f-769dad90d4f2, session_id: db076280-2529-45a1-a093-a3cd3f0799dc, gen: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:17.167204245Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"negotiate packet RegionInfoPlease","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:17.167346954Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"[0] downstairs client at 127.0.0.1:52643 has UUID af1eb908-dd0e-4704-a165-8346bf74d171","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:17.167452325Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 10, shift: 9 }, extent_count: 10, uuid: af1eb908-dd0e-4704-a165-8346bf74d171, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:17.167485333Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"Returning client:0 UUID:af1eb908-dd0e-4704-a165-8346bf74d171 matches","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:17.167506407Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"024c3783-b9b4-4453-823f-769dad90d4f2 Faulted Active Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:17.167538367Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"negotiate packet ExtentVersionsPlease","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:17.167559643Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"[0] 024c3783-b9b4-4453-823f-769dad90d4f2 (db076280-2529-45a1-a093-a3cd3f0799dc) Faulted Active Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:17.167709765Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:17.167737378Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] new RM replaced this: Some(RegionMetadata { generation: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], flush_numbers: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], dirty: [false, false, false, false, false, false, false, false, false, false] })","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:17.167761623Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] 024c3783-b9b4-4453-823f-769dad90d4f2 Enter Ready for LiveRepair mode","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:17.167781424Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:17.167799746Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] 127.0.0.1:52643 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:17.167818895Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"024c3783-b9b4-4453-823f-769dad90d4f2 LiveRepairReady Active Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:17.167838418Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:17.167856775Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.169212551Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] 024c3783-b9b4-4453-823f-769dad90d4f2 (db076280-2529-45a1-a093-a3cd3f0799dc) LiveRepairReady Active Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.169272066Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.169293484Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"Live Repair started","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.169312364Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"Live Repair main task begins.","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:18.169331271Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"task":"repair","upstairs":1}
{"msg":"Start Live Repair of extents 0 to 10","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.169352172Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"task":"repair","upstairs":1}
{"msg":"Start extent 0 repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.169372069Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"task":"repair","upstairs":1}
{"msg":"RE:0 repair extent with ids 58201,58202,58203,58204 deps:[]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.169392815Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"RE:0 close id:58201 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.169423466Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"RE:0 Wait for result from close command 58201:57202","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.169445098Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] 58201 final dependency list []","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.169484736Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58204 final dependency list [JobId(58201), JobId(58202), JobId(58203)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.169597206Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58205 final dependency list [JobId(58204), JobId(58201)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.17022591Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"Write to Extent 0:0:9 under repair","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:18.170677377Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] 58206 final dependency list [JobId(58205), JobId(58204), JobId(58201)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.170872815Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"Repair for extent 0 s:2 d:[ClientId(0)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.179836819Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"RE:0 Wait for result from repair command 58202:57203","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.179878502Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] 58202 final dependency list [JobId(58201)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.179899874Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"RE:0 Wait for result from NoOp command 58203:57204","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.180470507Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] 58203 final dependency list [JobId(58201), JobId(58202)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.180499506Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"RE:0 Wait for result from reopen command 58204:57205","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.183302553Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"Start extent 1 repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.183331259Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"task":"repair","upstairs":1}
{"msg":"RE:1 repair extent with ids 58225,58226,58227,58228 deps:[JobId(58208), JobId(58207)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.183353823Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"RE:1 close id:58225 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.183373086Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"RE:1 Wait for result from close command 58225:57226","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.183392636Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] 58225 final dependency list []","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.183525247Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58228 final dependency list [JobId(58225), JobId(58226), JobId(58227)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.183563139Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58229 final dependency list [JobId(58206), JobId(58204), JobId(58203), JobId(58202), JobId(58201)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.184107002Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58230 final dependency list [JobId(58229), JobId(58206), JobId(58205), JobId(58204), JobId(58203), JobId(58202), JobId(58201)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.184691767Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58231 final dependency list [JobId(58228), JobId(58225)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.185266525Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"Write to Extent 1:0:9 under repair","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:18.185697111Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] 58232 final dependency list [JobId(58231), JobId(58228), JobId(58225)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.185874992Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"Repair for extent 1 s:2 d:[ClientId(0)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.193921589Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"RE:1 Wait for result from repair command 58226:57227","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.193952008Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] 58226 final dependency list [JobId(58225)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.193978784Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"RE:1 Wait for result from NoOp command 58227:57228","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.194535133Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] 58227 final dependency list [JobId(58225), JobId(58226)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.194565393Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"RE:1 Wait for result from reopen command 58228:57229","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.197336887Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"Start extent 2 repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.197365682Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"task":"repair","upstairs":1}
{"msg":"RE:2 repair extent with ids 58249,58250,58251,58252 deps:[JobId(58234), JobId(58233), JobId(58210), JobId(58209)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.197389056Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"RE:2 close id:58249 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.197408349Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"RE:2 Wait for result from close command 58249:57250","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.197427859Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] 58249 final dependency list []","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.197562711Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58252 final dependency list [JobId(58249), JobId(58250), JobId(58251)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.197592622Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58253 final dependency list [JobId(58230), JobId(58206), JobId(58204), JobId(58203), JobId(58202), JobId(58201)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.198191115Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58254 final dependency list [JobId(58253), JobId(58230), JobId(58229), JobId(58206), JobId(58205), JobId(58204), JobId(58203), JobId(58202), JobId(58201)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.198786741Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58255 final dependency list [JobId(58232), JobId(58228), JobId(58227), JobId(58226), JobId(58225)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.199377221Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58256 final dependency list [JobId(58255), JobId(58232), JobId(58231), JobId(58228), JobId(58227), JobId(58226), JobId(58225)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.199972439Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58257 final dependency list [JobId(58252), JobId(58249)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.200553868Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"Write to Extent 2:0:9 under repair","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:18.200992603Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] 58258 final dependency list [JobId(58257), JobId(58252), JobId(58249)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.201173846Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"Repair for extent 2 s:2 d:[ClientId(0)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.208454074Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"RE:2 Wait for result from repair command 58250:57251","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.208495291Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] 58250 final dependency list [JobId(58249)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.208517308Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"RE:2 Wait for result from NoOp command 58251:57252","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.209073625Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] 58251 final dependency list [JobId(58249), JobId(58250)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.209107453Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"RE:2 Wait for result from reopen command 58252:57253","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.211970872Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"Start extent 3 repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.211998909Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"task":"repair","upstairs":1}
{"msg":"RE:3 repair extent with ids 58273,58274,58275,58276 deps:[JobId(58260), JobId(58259), JobId(58236), JobId(58235), JobId(58212), JobId(58211)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.212023103Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"RE:3 close id:58273 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.212050864Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"RE:3 Wait for result from close command 58273:57274","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.212077122Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] 58273 final dependency list []","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.212209461Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58276 final dependency list [JobId(58273), JobId(58274), JobId(58275)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.212238505Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58277 final dependency list [JobId(58254), JobId(58230), JobId(58206), JobId(58204), JobId(58203), JobId(58202), JobId(58201)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.212818846Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58278 final dependency list [JobId(58277), JobId(58254), JobId(58253), JobId(58230), JobId(58229), JobId(58206), JobId(58205), JobId(58204), JobId(58203), JobId(58202), JobId(58201)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.213418906Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58279 final dependency list [JobId(58256), JobId(58232), JobId(58228), JobId(58227), JobId(58226), JobId(58225)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.214011192Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58280 final dependency list [JobId(58279), JobId(58256), JobId(58255), JobId(58232), JobId(58231), JobId(58228), JobId(58227), JobId(58226), JobId(58225)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.214607159Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58281 final dependency list [JobId(58258), JobId(58252), JobId(58251), JobId(58250), JobId(58249)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.215201822Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58282 final dependency list [JobId(58281), JobId(58258), JobId(58257), JobId(58252), JobId(58251), JobId(58250), JobId(58249)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.215802171Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58283 final dependency list [JobId(58276), JobId(58273)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.216455057Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"Write to Extent 3:0:9 under repair","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:18.21685856Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] 58284 final dependency list [JobId(58283), JobId(58276), JobId(58273)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.217039087Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"Repair for extent 3 s:2 d:[ClientId(0)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.223478967Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"RE:3 Wait for result from repair command 58274:57275","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.223508299Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] 58274 final dependency list [JobId(58273)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.223542062Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"RE:3 Wait for result from NoOp command 58275:57276","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.224105672Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] 58275 final dependency list [JobId(58273), JobId(58274)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.224138796Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"RE:3 Wait for result from reopen command 58276:57277","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.227065622Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"Start extent 4 repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.227092803Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"task":"repair","upstairs":1}
{"msg":"RE:4 repair extent with ids 58297,58298,58299,58300 deps:[JobId(58286), JobId(58285), JobId(58262), JobId(58261), JobId(58238), JobId(58237), JobId(58214), JobId(58213)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.227117551Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"RE:4 close id:58297 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.227200435Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"RE:4 Wait for result from close command 58297:57298","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.2272272Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] 58297 final dependency list []","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.227346346Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58300 final dependency list [JobId(58297), JobId(58298), JobId(58299)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.227376917Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58301 final dependency list [JobId(58278), JobId(58254), JobId(58230), JobId(58206), JobId(58204), JobId(58203), JobId(58202), JobId(58201)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.227961865Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58302 final dependency list [JobId(58301), JobId(58278), JobId(58277), JobId(58254), JobId(58253), JobId(58230), JobId(58229), JobId(58206), JobId(58205), JobId(58204), JobId(58203), JobId(58202), JobId(58201)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.228571964Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58303 final dependency list [JobId(58280), JobId(58256), JobId(58232), JobId(58228), JobId(58227), JobId(58226), JobId(58225)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.229173605Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58304 final dependency list [JobId(58303), JobId(58280), JobId(58279), JobId(58256), JobId(58255), JobId(58232), JobId(58231), JobId(58228), JobId(58227), JobId(58226), JobId(58225)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.229790849Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58305 final dependency list [JobId(58282), JobId(58258), JobId(58252), JobId(58251), JobId(58250), JobId(58249)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.230396465Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58306 final dependency list [JobId(58305), JobId(58282), JobId(58281), JobId(58258), JobId(58257), JobId(58252), JobId(58251), JobId(58250), JobId(58249)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.231008428Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58307 final dependency list [JobId(58284), JobId(58276), JobId(58275), JobId(58274), JobId(58273)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.231609842Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58308 final dependency list [JobId(58307), JobId(58284), JobId(58283), JobId(58276), JobId(58275), JobId(58274), JobId(58273)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.232236136Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58309 final dependency list [JobId(58300), JobId(58297)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.232834396Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"Write to Extent 4:0:9 under repair","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:18.233290072Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] 58310 final dependency list [JobId(58309), JobId(58300), JobId(58297)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.233476606Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"Repair for extent 4 s:2 d:[ClientId(0)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.239027261Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"RE:4 Wait for result from repair command 58298:57299","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.239057417Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] 58298 final dependency list [JobId(58297)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.239090459Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"RE:4 Wait for result from NoOp command 58299:57300","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.239663108Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] 58299 final dependency list [JobId(58297), JobId(58298)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.239697847Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"RE:4 Wait for result from reopen command 58300:57301","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.2426891Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"Start extent 5 repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.242716352Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"task":"repair","upstairs":1}
{"msg":"RE:5 repair extent with ids 58321,58322,58323,58324 deps:[JobId(58312), JobId(58311), JobId(58288), JobId(58287), JobId(58264), JobId(58263), JobId(58240), JobId(58239), JobId(58216), JobId(58215)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.242743356Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"RE:5 close id:58321 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.242779195Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"RE:5 Wait for result from close command 58321:57322","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.242799831Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] 58321 final dependency list []","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.242936908Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58324 final dependency list [JobId(58321), JobId(58322), JobId(58323)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.242966342Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58325 final dependency list [JobId(58302), JobId(58278), JobId(58254), JobId(58230), JobId(58206), JobId(58204), JobId(58203), JobId(58202), JobId(58201)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.243560056Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58326 final dependency list [JobId(58325), JobId(58302), JobId(58301), JobId(58278), JobId(58277), JobId(58254), JobId(58253), JobId(58230), JobId(58229), JobId(58206), JobId(58205), JobId(58204), JobId(58203), JobId(58202), JobId(58201)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.244181078Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58327 final dependency list [JobId(58304), JobId(58280), JobId(58256), JobId(58232), JobId(58228), JobId(58227), JobId(58226), JobId(58225)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.244797281Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58328 final dependency list [JobId(58327), JobId(58304), JobId(58303), JobId(58280), JobId(58279), JobId(58256), JobId(58255), JobId(58232), JobId(58231), JobId(58228), JobId(58227), JobId(58226), JobId(58225)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.245413427Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58329 final dependency list [JobId(58306), JobId(58282), JobId(58258), JobId(58252), JobId(58251), JobId(58250), JobId(58249)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.246058203Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58330 final dependency list [JobId(58329), JobId(58306), JobId(58305), JobId(58282), JobId(58281), JobId(58258), JobId(58257), JobId(58252), JobId(58251), JobId(58250), JobId(58249)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.246672581Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58331 final dependency list [JobId(58308), JobId(58284), JobId(58276), JobId(58275), JobId(58274), JobId(58273)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.247285421Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58332 final dependency list [JobId(58331), JobId(58308), JobId(58307), JobId(58284), JobId(58283), JobId(58276), JobId(58275), JobId(58274), JobId(58273)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.247899002Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58333 final dependency list [JobId(58310), JobId(58300), JobId(58299), JobId(58298), JobId(58297)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.248511005Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58334 final dependency list [JobId(58333), JobId(58310), JobId(58309), JobId(58300), JobId(58299), JobId(58298), JobId(58297)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.249125279Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58335 final dependency list [JobId(58324), JobId(58321)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.249748191Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"Write to Extent 5:0:9 under repair","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:18.250202733Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] 58336 final dependency list [JobId(58335), JobId(58324), JobId(58321)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.250392558Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"Repair for extent 5 s:2 d:[ClientId(0)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.255028193Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"RE:5 Wait for result from repair command 58322:57323","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.255063162Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] 58322 final dependency list [JobId(58321)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.255091369Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"RE:5 Wait for result from NoOp command 58323:57324","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.255665808Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] 58323 final dependency list [JobId(58321), JobId(58322)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.25570062Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"RE:5 Wait for result from reopen command 58324:57325","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.258790903Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"Start extent 6 repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.258818763Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"task":"repair","upstairs":1}
{"msg":"RE:6 repair extent with ids 58345,58346,58347,58348 deps:[JobId(58338), JobId(58337), JobId(58314), JobId(58313), JobId(58290), JobId(58289), JobId(58266), JobId(58265), JobId(58242), JobId(58241), JobId(58218), JobId(58217)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.258851211Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"RE:6 close id:58345 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.258888068Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"RE:6 Wait for result from close command 58345:57346","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.258909553Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] 58345 final dependency list []","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.259046894Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58348 final dependency list [JobId(58345), JobId(58346), JobId(58347)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.259077371Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58349 final dependency list [JobId(58326), JobId(58302), JobId(58278), JobId(58254), JobId(58230), JobId(58206), JobId(58204), JobId(58203), JobId(58202), JobId(58201)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.259698799Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58350 final dependency list [JobId(58349), JobId(58326), JobId(58325), JobId(58302), JobId(58301), JobId(58278), JobId(58277), JobId(58254), JobId(58253), JobId(58230), JobId(58229), JobId(58206), JobId(58205), JobId(58204), JobId(58203), JobId(58202), JobId(58201)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.260328109Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58351 final dependency list [JobId(58328), JobId(58304), JobId(58280), JobId(58256), JobId(58232), JobId(58228), JobId(58227), JobId(58226), JobId(58225)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.260978606Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58352 final dependency list [JobId(58351), JobId(58328), JobId(58327), JobId(58304), JobId(58303), JobId(58280), JobId(58279), JobId(58256), JobId(58255), JobId(58232), JobId(58231), JobId(58228), JobId(58227), JobId(58226), JobId(58225)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.26160772Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58353 final dependency list [JobId(58330), JobId(58306), JobId(58282), JobId(58258), JobId(58252), JobId(58251), JobId(58250), JobId(58249)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.262230133Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58354 final dependency list [JobId(58353), JobId(58330), JobId(58329), JobId(58306), JobId(58305), JobId(58282), JobId(58281), JobId(58258), JobId(58257), JobId(58252), JobId(58251), JobId(58250), JobId(58249)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.262860118Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58355 final dependency list [JobId(58332), JobId(58308), JobId(58284), JobId(58276), JobId(58275), JobId(58274), JobId(58273)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.263479972Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58356 final dependency list [JobId(58355), JobId(58332), JobId(58331), JobId(58308), JobId(58307), JobId(58284), JobId(58283), JobId(58276), JobId(58275), JobId(58274), JobId(58273)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.264105296Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58357 final dependency list [JobId(58334), JobId(58310), JobId(58300), JobId(58299), JobId(58298), JobId(58297)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.264724458Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58358 final dependency list [JobId(58357), JobId(58334), JobId(58333), JobId(58310), JobId(58309), JobId(58300), JobId(58299), JobId(58298), JobId(58297)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.265348457Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58359 final dependency list [JobId(58336), JobId(58324), JobId(58323), JobId(58322), JobId(58321)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.265996931Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58360 final dependency list [JobId(58359), JobId(58336), JobId(58335), JobId(58324), JobId(58323), JobId(58322), JobId(58321)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.266612995Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58361 final dependency list [JobId(58348), JobId(58345)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.267226099Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"Write to Extent 6:0:9 under repair","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:18.267687203Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] 58362 final dependency list [JobId(58361), JobId(58348), JobId(58345)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.267872425Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"Repair for extent 6 s:2 d:[ClientId(0)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.27162582Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"RE:6 Wait for result from repair command 58346:57347","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.27165623Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] 58346 final dependency list [JobId(58345)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.271690237Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"RE:6 Wait for result from NoOp command 58347:57348","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.272265623Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] 58347 final dependency list [JobId(58345), JobId(58346)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.272298383Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"RE:6 Wait for result from reopen command 58348:57349","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.275408063Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"Start extent 7 repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.275438262Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"task":"repair","upstairs":1}
{"msg":"RE:7 repair extent with ids 58369,58370,58371,58372 deps:[JobId(58364), JobId(58363), JobId(58340), JobId(58339), JobId(58316), JobId(58315), JobId(58292), JobId(58291), JobId(58268), JobId(58267), JobId(58244), JobId(58243), JobId(58220), JobId(58219)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.275497085Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"RE:7 close id:58369 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.275520795Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"RE:7 Wait for result from close command 58369:57370","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.275540459Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] 58369 final dependency list []","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.275668218Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58372 final dependency list [JobId(58369), JobId(58370), JobId(58371)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.275698042Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58373 final dependency list [JobId(58350), JobId(58326), JobId(58302), JobId(58278), JobId(58254), JobId(58230), JobId(58206), JobId(58204), JobId(58203), JobId(58202), JobId(58201)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.276316457Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58374 final dependency list [JobId(58373), JobId(58350), JobId(58349), JobId(58326), JobId(58325), JobId(58302), JobId(58301), JobId(58278), JobId(58277), JobId(58254), JobId(58253), JobId(58230), JobId(58229), JobId(58206), JobId(58205), JobId(58204), JobId(58203), JobId(58202), JobId(58201)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.276952001Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58375 final dependency list [JobId(58352), JobId(58328), JobId(58304), JobId(58280), JobId(58256), JobId(58232), JobId(58228), JobId(58227), JobId(58226), JobId(58225)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.277587916Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58376 final dependency list [JobId(58375), JobId(58352), JobId(58351), JobId(58328), JobId(58327), JobId(58304), JobId(58303), JobId(58280), JobId(58279), JobId(58256), JobId(58255), JobId(58232), JobId(58231), JobId(58228), JobId(58227), JobId(58226), JobId(58225)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.278212601Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58377 final dependency list [JobId(58354), JobId(58330), JobId(58306), JobId(58282), JobId(58258), JobId(58252), JobId(58251), JobId(58250), JobId(58249)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.278843523Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58378 final dependency list [JobId(58377), JobId(58354), JobId(58353), JobId(58330), JobId(58329), JobId(58306), JobId(58305), JobId(58282), JobId(58281), JobId(58258), JobId(58257), JobId(58252), JobId(58251), JobId(58250), JobId(58249)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.279480876Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58379 final dependency list [JobId(58356), JobId(58332), JobId(58308), JobId(58284), JobId(58276), JobId(58275), JobId(58274), JobId(58273)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.28013352Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58380 final dependency list [JobId(58379), JobId(58356), JobId(58355), JobId(58332), JobId(58331), JobId(58308), JobId(58307), JobId(58284), JobId(58283), JobId(58276), JobId(58275), JobId(58274), JobId(58273)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.280762349Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58381 final dependency list [JobId(58358), JobId(58334), JobId(58310), JobId(58300), JobId(58299), JobId(58298), JobId(58297)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.281392375Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58382 final dependency list [JobId(58381), JobId(58358), JobId(58357), JobId(58334), JobId(58333), JobId(58310), JobId(58309), JobId(58300), JobId(58299), JobId(58298), JobId(58297)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.282033541Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58383 final dependency list [JobId(58360), JobId(58336), JobId(58324), JobId(58323), JobId(58322), JobId(58321)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.282658852Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58384 final dependency list [JobId(58383), JobId(58360), JobId(58359), JobId(58336), JobId(58335), JobId(58324), JobId(58323), JobId(58322), JobId(58321)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.283285842Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58385 final dependency list [JobId(58362), JobId(58348), JobId(58347), JobId(58346), JobId(58345)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.283918127Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58386 final dependency list [JobId(58385), JobId(58362), JobId(58361), JobId(58348), JobId(58347), JobId(58346), JobId(58345)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.284548201Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58387 final dependency list [JobId(58372), JobId(58369)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.285202164Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"Write to Extent 7:0:9 under repair","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:18.285674894Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] 58388 final dependency list [JobId(58387), JobId(58372), JobId(58369)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.28586464Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"Repair for extent 7 s:2 d:[ClientId(0)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.288569868Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"RE:7 Wait for result from repair command 58370:57371","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.288605254Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] 58370 final dependency list [JobId(58369)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.288631177Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"RE:7 Wait for result from NoOp command 58371:57372","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.28920568Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] 58371 final dependency list [JobId(58369), JobId(58370)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.289241618Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"RE:7 Wait for result from reopen command 58372:57373","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.292485929Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"Start extent 8 repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.292513929Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"task":"repair","upstairs":1}
{"msg":"RE:8 repair extent with ids 58393,58394,58395,58396 deps:[JobId(58390), JobId(58389), JobId(58366), JobId(58365), JobId(58342), JobId(58341), JobId(58318), JobId(58317), JobId(58294), JobId(58293), JobId(58270), JobId(58269), JobId(58246), JobId(58245), JobId(58222), JobId(58221)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.29256009Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"RE:8 close id:58393 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.292593607Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"RE:8 Wait for result from close command 58393:57394","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.292614061Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] 58393 final dependency list []","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.292755123Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58396 final dependency list [JobId(58393), JobId(58394), JobId(58395)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.292787621Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58397 final dependency list [JobId(58374), JobId(58350), JobId(58326), JobId(58302), JobId(58278), JobId(58254), JobId(58230), JobId(58206), JobId(58204), JobId(58203), JobId(58202), JobId(58201)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.293438967Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58398 final dependency list [JobId(58397), JobId(58374), JobId(58373), JobId(58350), JobId(58349), JobId(58326), JobId(58325), JobId(58302), JobId(58301), JobId(58278), JobId(58277), JobId(58254), JobId(58253), JobId(58230), JobId(58229), JobId(58206), JobId(58205), JobId(58204), JobId(58203), JobId(58202), JobId(58201)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.294097314Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58399 final dependency list [JobId(58376), JobId(58352), JobId(58328), JobId(58304), JobId(58280), JobId(58256), JobId(58232), JobId(58228), JobId(58227), JobId(58226), JobId(58225)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.294761854Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58400 final dependency list [JobId(58399), JobId(58376), JobId(58375), JobId(58352), JobId(58351), JobId(58328), JobId(58327), JobId(58304), JobId(58303), JobId(58280), JobId(58279), JobId(58256), JobId(58255), JobId(58232), JobId(58231), JobId(58228), JobId(58227), JobId(58226), JobId(58225)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.295398683Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58401 final dependency list [JobId(58378), JobId(58354), JobId(58330), JobId(58306), JobId(58282), JobId(58258), JobId(58252), JobId(58251), JobId(58250), JobId(58249)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.296041366Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58402 final dependency list [JobId(58401), JobId(58378), JobId(58377), JobId(58354), JobId(58353), JobId(58330), JobId(58329), JobId(58306), JobId(58305), JobId(58282), JobId(58281), JobId(58258), JobId(58257), JobId(58252), JobId(58251), JobId(58250), JobId(58249)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.296683299Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58403 final dependency list [JobId(58380), JobId(58356), JobId(58332), JobId(58308), JobId(58284), JobId(58276), JobId(58275), JobId(58274), JobId(58273)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.297317321Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58404 final dependency list [JobId(58403), JobId(58380), JobId(58379), JobId(58356), JobId(58355), JobId(58332), JobId(58331), JobId(58308), JobId(58307), JobId(58284), JobId(58283), JobId(58276), JobId(58275), JobId(58274), JobId(58273)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.297955523Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58405 final dependency list [JobId(58382), JobId(58358), JobId(58334), JobId(58310), JobId(58300), JobId(58299), JobId(58298), JobId(58297)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.298604555Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58406 final dependency list [JobId(58405), JobId(58382), JobId(58381), JobId(58358), JobId(58357), JobId(58334), JobId(58333), JobId(58310), JobId(58309), JobId(58300), JobId(58299), JobId(58298), JobId(58297)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.299242601Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58407 final dependency list [JobId(58384), JobId(58360), JobId(58336), JobId(58324), JobId(58323), JobId(58322), JobId(58321)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.299914589Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58408 final dependency list [JobId(58407), JobId(58384), JobId(58383), JobId(58360), JobId(58359), JobId(58336), JobId(58335), JobId(58324), JobId(58323), JobId(58322), JobId(58321)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.300553367Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58409 final dependency list [JobId(58386), JobId(58362), JobId(58348), JobId(58347), JobId(58346), JobId(58345)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.301194481Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58410 final dependency list [JobId(58409), JobId(58386), JobId(58385), JobId(58362), JobId(58361), JobId(58348), JobId(58347), JobId(58346), JobId(58345)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.301831552Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58411 final dependency list [JobId(58388), JobId(58372), JobId(58371), JobId(58370), JobId(58369)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.302467226Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58412 final dependency list [JobId(58411), JobId(58388), JobId(58387), JobId(58372), JobId(58371), JobId(58370), JobId(58369)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.303108892Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58413 final dependency list [JobId(58396), JobId(58393)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.303738141Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"Write to Extent 8:0:9 under repair","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:18.304223107Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] 58414 final dependency list [JobId(58413), JobId(58396), JobId(58393)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.304409996Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"Repair for extent 8 s:2 d:[ClientId(0)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.30614146Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"RE:8 Wait for result from repair command 58394:57395","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.30617074Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] 58394 final dependency list [JobId(58393)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.306208073Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"RE:8 Wait for result from NoOp command 58395:57396","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.306786758Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] 58395 final dependency list [JobId(58393), JobId(58394)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.306823957Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"RE:8 Wait for result from reopen command 58396:57397","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.310155035Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"Start extent 9 repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.310183123Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"task":"repair","upstairs":1}
{"msg":"RE:9 repair extent with ids 58417,58418,58419,58420 deps:[JobId(58416), JobId(58415), JobId(58392), JobId(58391), JobId(58368), JobId(58367), JobId(58344), JobId(58343), JobId(58320), JobId(58319), JobId(58296), JobId(58295), JobId(58272), JobId(58271), JobId(58248), JobId(58247), JobId(58224), JobId(58223)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.310247055Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"RE:9 close id:58417 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.310272791Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"RE:9 Wait for result from close command 58417:57418","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.310293255Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] 58417 final dependency list []","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.310438208Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58420 final dependency list [JobId(58417), JobId(58418), JobId(58419)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.310489795Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58421 final dependency list [JobId(58398), JobId(58374), JobId(58350), JobId(58326), JobId(58302), JobId(58278), JobId(58254), JobId(58230), JobId(58206), JobId(58204), JobId(58203), JobId(58202), JobId(58201)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.311123356Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58422 final dependency list [JobId(58421), JobId(58398), JobId(58397), JobId(58374), JobId(58373), JobId(58350), JobId(58349), JobId(58326), JobId(58325), JobId(58302), JobId(58301), JobId(58278), JobId(58277), JobId(58254), JobId(58253), JobId(58230), JobId(58229), JobId(58206), JobId(58205), JobId(58204), JobId(58203), JobId(58202), JobId(58201)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.311784745Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58423 final dependency list [JobId(58400), JobId(58376), JobId(58352), JobId(58328), JobId(58304), JobId(58280), JobId(58256), JobId(58232), JobId(58228), JobId(58227), JobId(58226), JobId(58225)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.312438908Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58424 final dependency list [JobId(58423), JobId(58400), JobId(58399), JobId(58376), JobId(58375), JobId(58352), JobId(58351), JobId(58328), JobId(58327), JobId(58304), JobId(58303), JobId(58280), JobId(58279), JobId(58256), JobId(58255), JobId(58232), JobId(58231), JobId(58228), JobId(58227), JobId(58226), JobId(58225)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.313081003Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58425 final dependency list [JobId(58402), JobId(58378), JobId(58354), JobId(58330), JobId(58306), JobId(58282), JobId(58258), JobId(58252), JobId(58251), JobId(58250), JobId(58249)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.313757846Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58426 final dependency list [JobId(58425), JobId(58402), JobId(58401), JobId(58378), JobId(58377), JobId(58354), JobId(58353), JobId(58330), JobId(58329), JobId(58306), JobId(58305), JobId(58282), JobId(58281), JobId(58258), JobId(58257), JobId(58252), JobId(58251), JobId(58250), JobId(58249)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.314413673Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58427 final dependency list [JobId(58404), JobId(58380), JobId(58356), JobId(58332), JobId(58308), JobId(58284), JobId(58276), JobId(58275), JobId(58274), JobId(58273)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.315057194Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58428 final dependency list [JobId(58427), JobId(58404), JobId(58403), JobId(58380), JobId(58379), JobId(58356), JobId(58355), JobId(58332), JobId(58331), JobId(58308), JobId(58307), JobId(58284), JobId(58283), JobId(58276), JobId(58275), JobId(58274), JobId(58273)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.315711016Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58429 final dependency list [JobId(58406), JobId(58382), JobId(58358), JobId(58334), JobId(58310), JobId(58300), JobId(58299), JobId(58298), JobId(58297)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.316360181Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58430 final dependency list [JobId(58429), JobId(58406), JobId(58405), JobId(58382), JobId(58381), JobId(58358), JobId(58357), JobId(58334), JobId(58333), JobId(58310), JobId(58309), JobId(58300), JobId(58299), JobId(58298), JobId(58297)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.317014967Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58431 final dependency list [JobId(58408), JobId(58384), JobId(58360), JobId(58336), JobId(58324), JobId(58323), JobId(58322), JobId(58321)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.317660234Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58432 final dependency list [JobId(58431), JobId(58408), JobId(58407), JobId(58384), JobId(58383), JobId(58360), JobId(58359), JobId(58336), JobId(58335), JobId(58324), JobId(58323), JobId(58322), JobId(58321)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.318307629Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58433 final dependency list [JobId(58410), JobId(58386), JobId(58362), JobId(58348), JobId(58347), JobId(58346), JobId(58345)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.318978187Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58434 final dependency list [JobId(58433), JobId(58410), JobId(58409), JobId(58386), JobId(58385), JobId(58362), JobId(58361), JobId(58348), JobId(58347), JobId(58346), JobId(58345)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.319625129Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58435 final dependency list [JobId(58412), JobId(58388), JobId(58372), JobId(58371), JobId(58370), JobId(58369)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.320277897Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58436 final dependency list [JobId(58435), JobId(58412), JobId(58411), JobId(58388), JobId(58387), JobId(58372), JobId(58371), JobId(58370), JobId(58369)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.320920669Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58437 final dependency list [JobId(58414), JobId(58396), JobId(58395), JobId(58394), JobId(58393)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.321565353Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58438 final dependency list [JobId(58437), JobId(58414), JobId(58413), JobId(58396), JobId(58395), JobId(58394), JobId(58393)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.322209491Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58439 final dependency list [JobId(58420), JobId(58417)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.322858033Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"Write to Extent 9:0:9 under repair","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:18.323343927Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] 58440 final dependency list [JobId(58439), JobId(58420), JobId(58417)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.323533184Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"Repair for extent 9 s:2 d:[ClientId(0)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.324232316Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"RE:9 Wait for result from repair command 58418:57419","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.324260312Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] 58418 final dependency list [JobId(58417)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.324299619Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"RE:9 Wait for result from NoOp command 58419:57420","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.324879785Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] 58419 final dependency list [JobId(58417), JobId(58418)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.324919465Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"RE:9 Wait for result from reopen command 58420:57421","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.328333681Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"LiveRepair final flush submitted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.328455417Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] 58441 final dependency list [JobId(58440), JobId(58439), JobId(58438), JobId(58437), JobId(58436), JobId(58435), JobId(58434), JobId(58433), JobId(58432), JobId(58431), JobId(58430), JobId(58429), JobId(58428), JobId(58427), JobId(58426), JobId(58425), JobId(58424), JobId(58423), JobId(58422), JobId(58421), JobId(58420), JobId(58419), JobId(58418), JobId(58417), JobId(58414), JobId(58413), JobId(58412), JobId(58411), JobId(58410), JobId(58409), JobId(58408), JobId(58407), JobId(58406), JobId(58405), JobId(58404), JobId(58403), JobId(58402), JobId(58401), JobId(58400), JobId(58399), JobId(58398), JobId(58397), JobId(58396), JobId(58395), JobId(58394), JobId(58393), JobId(58388), JobId(58387), JobId(58386), JobId(58385), JobId(58384), JobId(58383), JobId(58382), JobId(58381), JobId(58380), JobId(58379), JobId(58378), JobId(58377), JobId(58376), JobId(58375), JobId(58374), JobId(58373), JobId(58372), JobId(58371), JobId(58370), JobId(58369), JobId(58362), JobId(58361), JobId(58360), JobId(58359), JobId(58358), JobId(58357), JobId(58356), JobId(58355), JobId(58354), JobId(58353), JobId(58352), JobId(58351), JobId(58350), JobId(58349), JobId(58348), JobId(58347), JobId(58346), JobId(58345), JobId(58336), JobId(58335), JobId(58334), JobId(58333), JobId(58332), JobId(58331), JobId(58330), JobId(58329), JobId(58328), JobId(58327), JobId(58326), JobId(58325), JobId(58324), JobId(58323), JobId(58322), JobId(58321), JobId(58310), JobId(58309), JobId(58308), JobId(58307), JobId(58306), JobId(58305), JobId(58304), JobId(58303), JobId(58302), JobId(58301), JobId(58300), JobId(58299), JobId(58298), JobId(58297), JobId(58284), JobId(58283), JobId(58282), JobId(58281), JobId(58280), JobId(58279), JobId(58278), JobId(58277), JobId(58276), JobId(58275), JobId(58274), JobId(58273), JobId(58258), JobId(58257), JobId(58256), JobId(58255), JobId(58254), JobId(58253), JobId(58252), JobId(58251), JobId(58250), JobId(58249), JobId(58232), JobId(58231), JobId(58230), JobId(58229), JobId(58228), JobId(58227), JobId(58226), JobId(58225), JobId(58206), JobId(58205), JobId(58204), JobId(58203), JobId(58202), JobId(58201)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.32877448Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"LiveRepair final flush completed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.330020036Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] 024c3783-b9b4-4453-823f-769dad90d4f2 (db076280-2529-45a1-a093-a3cd3f0799dc) LiveRepair Active Active ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.330051056Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] Transition from LiveRepair to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:18.330071025Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"Live Repair returns Ok(())","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:18.33008984Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
test dummy_downstairs_tests::protocol_test::test_successful_live_repair ... ok
{"msg":"sent read response for job 0 = 1000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:22.825043058Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] df6648fa-e480-4dd9-b7c4-7fec45dbddf8 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:22.82511575Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"looper":"0","upstairs":1}
{"msg":"[0] Proc runs for 127.0.0.1:39941 in state Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:22.825143981Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"could not send read response for job 1 = 1001: Broken pipe (os error 32)","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:14:22.825276494Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"negotiate packet HereIAm { version: 4, upstairs_id: df6648fa-e480-4dd9-b7c4-7fec45dbddf8, session_id: b793572f-19dc-423b-877e-0f0cfe9147d9, gen: 1, read_only: false, encrypted: false, alternate_versions: [] }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:22.825450664Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"[0] upstairs guest_io_ready=TRUE, promote! session b793572f-19dc-423b-877e-0f0cfe9147d9","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:22.825560173Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"negotiate packet PromoteToActive { upstairs_id: df6648fa-e480-4dd9-b7c4-7fec45dbddf8, session_id: b793572f-19dc-423b-877e-0f0cfe9147d9, gen: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:22.825662983Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"negotiate packet RegionInfoPlease","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:22.825804544Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"[0] downstairs client at 127.0.0.1:39941 has UUID c7df82f0-e00e-410b-ac45-3ec316aefac5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:22.825906199Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 10, shift: 9 }, extent_count: 10, uuid: c7df82f0-e00e-410b-ac45-3ec316aefac5, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:22.825939496Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"Returning client:0 UUID:c7df82f0-e00e-410b-ac45-3ec316aefac5 matches","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:22.825960161Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"df6648fa-e480-4dd9-b7c4-7fec45dbddf8 Faulted Active Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:22.82597977Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"negotiate packet ExtentVersionsPlease","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:22.8259989Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"[0] df6648fa-e480-4dd9-b7c4-7fec45dbddf8 (b793572f-19dc-423b-877e-0f0cfe9147d9) Faulted Active Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:22.826165619Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:22.82619343Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] new RM replaced this: Some(RegionMetadata { generation: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], flush_numbers: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], dirty: [false, false, false, false, false, false, false, false, false, false] })","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:22.826216966Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] df6648fa-e480-4dd9-b7c4-7fec45dbddf8 Enter Ready for LiveRepair mode","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:22.826244189Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:22.826264647Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] 127.0.0.1:39941 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:22.82678903Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"df6648fa-e480-4dd9-b7c4-7fec45dbddf8 LiveRepairReady Active Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:22.826817611Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:22.826836911Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:23.58755371Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":2}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:23.587609857Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":3}
{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:23.828147559Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] df6648fa-e480-4dd9-b7c4-7fec45dbddf8 (b793572f-19dc-423b-877e-0f0cfe9147d9) LiveRepairReady Active Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:23.828208416Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:23.828229921Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"Live Repair started","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:23.828248767Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"Live Repair main task begins.","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:23.828267763Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"task":"repair","upstairs":1}
{"msg":"Start Live Repair of extents 0 to 10","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:23.828288907Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"task":"repair","upstairs":1}
{"msg":"Start extent 0 repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:23.828308681Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"task":"repair","upstairs":1}
{"msg":"RE:0 repair extent with ids 58201,58202,58203,58204 deps:[]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:23.828329294Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"RE:0 close id:58201 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:23.828348388Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"RE:0 Wait for result from close command 58201:57202","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:23.828368103Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] 58201 final dependency list []","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:23.828387001Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58204 final dependency list [JobId(58201), JobId(58202), JobId(58203)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:23.828489967Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"Repair for extent 0 s:2 d:[ClientId(0)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:23.829182134Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"RE:0 Wait for result from repair command 58202:57203","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:23.829212886Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] 58202 final dependency list [JobId(58201)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:23.829233514Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"dropping ds1 now!","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:14:23.829566409Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"reconnecting ds1 now!","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:14:23.829592051Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] job id 58202 saw error GenericError(\"bad news, networks are tricky\")","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:14:23.829768206Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] DS Reports error Err(GenericError(\"bad news, networks are tricky\")) on job 58202, DownstairsIO { ds_id: JobId(58202), guest_id: 57203, work: ExtentLiveRepair { dependencies: [JobId(58201)], extent: 0, source_downstairs: ClientId(2), source_repair_address: 127.0.0.1:54125, repair_downstairs: [ClientId(0)] }, state: ClientData([InProgress, InProgress, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 9 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:14:23.829811552Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] Reports error GenericError(\"bad news, networks are tricky\") on job 58202, DownstairsIO { ds_id: JobId(58202), guest_id: 57203, work: ExtentLiveRepair { dependencies: [JobId(58201)], extent: 0, source_downstairs: ClientId(2), source_repair_address: 127.0.0.1:54125, repair_downstairs: [ClientId(0)] }, state: ClientData([InProgress, InProgress, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 9 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:14:23.829849806Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] client skip 3 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:23.829871978Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:23.829892461Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] df6648fa-e480-4dd9-b7c4-7fec45dbddf8 (b793572f-19dc-423b-877e-0f0cfe9147d9) LiveRepair Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:23.829915124Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] Transition from LiveRepair to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:23.829934197Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] will exit pm_task, this downstairs Faulted","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:23.829953296Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"127.0.0.1:39941: proc: [0] client work task ended, Ok(Err([0] This downstairs now in Faulted)), so we end too","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:14:23.829974232Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"looper":"0","upstairs":1}
{"msg":"[0] df6648fa-e480-4dd9-b7c4-7fec45dbddf8 Gone missing, transition from Faulted to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:23.830002428Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] df6648fa-e480-4dd9-b7c4-7fec45dbddf8 connection to 127.0.0.1:39941 closed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:23.830024311Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"looper":"0","upstairs":1}
{"msg":"[0] 127.0.0.1:39941 task reports connection:false","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:23.83004484Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"df6648fa-e480-4dd9-b7c4-7fec45dbddf8 Faulted Active Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:23.830064821Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] 127.0.0.1:39941 task reports offline","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:23.830083682Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"Extent 0 close id:58202 Failed: IO Error: 1 out of 3 downstairs failed to complete this IO","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:14:23.830103928Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"RE:0 Wait for result from NoOp command 58203:57204","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:23.830123355Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"spawn_message_receiver saw disconnect, bailing","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:14:23.830312058Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"ds1 negotiate start now!","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:14:24.831092692Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] df6648fa-e480-4dd9-b7c4-7fec45dbddf8 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:24.831153835Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"looper":"0","upstairs":1}
{"msg":"[0] Proc runs for 127.0.0.1:39941 in state Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:24.831178988Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"negotiate packet HereIAm { version: 4, upstairs_id: df6648fa-e480-4dd9-b7c4-7fec45dbddf8, session_id: b793572f-19dc-423b-877e-0f0cfe9147d9, gen: 1, read_only: false, encrypted: false, alternate_versions: [] }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:24.831293027Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"[0] upstairs guest_io_ready=TRUE, promote! session b793572f-19dc-423b-877e-0f0cfe9147d9","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:24.831406386Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"negotiate packet PromoteToActive { upstairs_id: df6648fa-e480-4dd9-b7c4-7fec45dbddf8, session_id: b793572f-19dc-423b-877e-0f0cfe9147d9, gen: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:24.831509773Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"negotiate packet RegionInfoPlease","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:24.831653968Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"ds1 negotiate extent versions please now!","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:14:24.831681857Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] downstairs client at 127.0.0.1:39941 has UUID c7df82f0-e00e-410b-ac45-3ec316aefac5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:24.831766432Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 10, shift: 9 }, extent_count: 10, uuid: c7df82f0-e00e-410b-ac45-3ec316aefac5, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:24.83180933Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"Returning client:0 UUID:c7df82f0-e00e-410b-ac45-3ec316aefac5 matches","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:24.831832616Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"df6648fa-e480-4dd9-b7c4-7fec45dbddf8 Faulted Active Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:24.831852435Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"negotiate packet ExtentVersionsPlease","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:24.831871416Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"downstairs":1}
{"msg":"ds1 spawn message receiver now!","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:14:24.831892881Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291}
{"msg":"[0] df6648fa-e480-4dd9-b7c4-7fec45dbddf8 (b793572f-19dc-423b-877e-0f0cfe9147d9) Faulted Active Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:24.832110642Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:24.832138541Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] new RM replaced this: Some(RegionMetadata { generation: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], flush_numbers: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], dirty: [false, false, false, false, false, false, false, false, false, false] })","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:24.832162731Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] df6648fa-e480-4dd9-b7c4-7fec45dbddf8 Enter Ready for LiveRepair mode","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:24.832182855Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:24.832201291Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] 127.0.0.1:39941 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:24.83234141Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"df6648fa-e480-4dd9-b7c4-7fec45dbddf8 LiveRepairReady Active Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:24.832369371Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:24.832388715Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"RE:0 Wait for result from reopen command 58204:57205","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:24.832596612Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"RE:0 Bailing with error","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:24.832623494Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"Error After extent 0 repair","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:24.832643075Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"task":"repair","upstairs":1}
{"msg":"Start extent 1 repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:24.832663424Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"task":"repair","upstairs":1}
{"msg":"extent 1 repair has failed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:24.832683376Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"task":"repair","upstairs":1}
{"msg":"Exit repair at extent 1","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:24.832708776Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"task":"repair","upstairs":1}
{"msg":"LiveRepair final flush submitted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:24.832730157Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"LiveRepair final flush completed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:24.834514914Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"Live Repair returns Ok(())","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:24.834542103Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:25.833959899Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] df6648fa-e480-4dd9-b7c4-7fec45dbddf8 (b793572f-19dc-423b-877e-0f0cfe9147d9) LiveRepairReady Active Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:25.834023122Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:25.83404429Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"Live Repair started","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:25.834063342Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"Live Repair main task begins.","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:25.834082449Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"task":"repair","upstairs":1}
{"msg":"Start Live Repair of extents 0 to 10","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:25.834103337Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"task":"repair","upstairs":1}
{"msg":"Start extent 0 repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:25.834123004Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"task":"repair","upstairs":1}
{"msg":"RE:0 repair extent with ids 58206,58207,58208,58209 deps:[]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:25.834143625Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"RE:0 close id:58206 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:25.834162655Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"RE:0 Wait for result from close command 58206:57207","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:25.834182044Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"upstairs":1}
{"msg":"[0] 58206 final dependency list []","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:25.834210812Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
{"msg":"[0] 58209 final dependency list [JobId(58206), JobId(58207), JobId(58208)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:25.834314675Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4291,"":"downstairs","upstairs":1}
test dummy_downstairs_tests::protocol_test::test_error_during_live_repair_no_halt ... ok

test result: ok. 351 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 383.45s

     Running `/work/oxidecomputer/crucible/target/debug/deps/crucible_agent-acbf118b39db417b --nocapture`

running 11 tests
{"id":"abc","state":"requested","block_size":4096,"extent_size":4096,"extent_count":100,"encrypted":false,"port_number":1701,"cert_pem":null,"key_pem":null,"root_pem":null}
Sep 22 23:14:25.892 INFO region 4d83cb9a-5f4f-448d-919c-4d7ce8fc18b4 state: Requested, component: test_harness_datafile
Sep 22 23:14:25.892 INFO region 6ed761bc-0273-4c2f-b2aa-b5a77c461371 state: Requested, component: test_harness_datafile
Sep 22 23:14:25.892 INFO region cd2faf68-2161-44ae-80cd-734d2ef30327 state: Requested, component: test_harness_datafile
Sep 22 23:14:25.892 INFO region 7c0b8c8e-1d24-4cd7-97b0-47ad837e7f01 state: Requested, component: test_harness_datafile
Sep 22 23:14:25.892 INFO region bd788714-d366-4573-ac0f-64f52c45fa42 state: Requested, component: test_harness_datafile
Sep 22 23:14:25.894 INFO region cd2faf68-2161-44ae-80cd-734d2ef30327 state: Requested -> Tombstoned, component: test_harness_datafile
Sep 22 23:14:25.894 INFO region bd788714-d366-4573-ac0f-64f52c45fa42 state: Requested -> Created, component: test_harness_datafile
Sep 22 23:14:25.894 INFO region 4d83cb9a-5f4f-448d-919c-4d7ce8fc18b4 state: Requested -> Created, component: test_harness_datafile
test model::test::basic ... ok
test test::test_collect_behaviour ... ok
Sep 22 23:14:25.895 INFO region 7c0b8c8e-1d24-4cd7-97b0-47ad837e7f01 state: Requested -> Tombstoned, component: test_harness_datafile
Sep 22 23:14:25.895 INFO region 6ed761bc-0273-4c2f-b2aa-b5a77c461371 state: Requested -> Created, component: test_harness_datafile
Sep 22 23:14:25.895 INFO region c1564dbf-e201-4b70-845d-fcb34017f4f7 state: Requested, component: test_harness_datafile
Sep 22 23:14:25.895 INFO region c1564dbf-e201-4b70-845d-fcb34017f4f7 state: Requested -> Failed, component: test_harness_datafile
Sep 22 23:14:25.895 INFO region b8985630-b020-47f6-828d-9cc22ed12f56 state: Requested, component: test_harness_datafile
Sep 22 23:14:25.895 INFO creating missing downstairs instance downstairs-4d83cb9a-5f4f-448d-919c-4d7ce8fc18b4
Sep 22 23:14:25.895 INFO creating missing downstairs instance downstairs-6ed761bc-0273-4c2f-b2aa-b5a77c461371
Sep 22 23:14:25.895 INFO creating missing downstairs instance downstairs-bd788714-d366-4573-ac0f-64f52c45fa42
Sep 22 23:14:25.895 INFO region b8985630-b020-47f6-828d-9cc22ed12f56 state: Requested -> Created, component: test_harness_datafile
test datafile::test::test_stat_parsing ... ok
Sep 22 23:14:25.896 INFO ok, have oxide/crucible/downstairs:downstairs-6ed761bc-0273-4c2f-b2aa-b5a77c461371
Sep 22 23:14:25.896 INFO creating missing downstairs instance downstairs-b8985630-b020-47f6-828d-9cc22ed12f56
Sep 22 23:14:25.896 INFO ok, have oxide/crucible/downstairs:downstairs-4d83cb9a-5f4f-448d-919c-4d7ce8fc18b4
Sep 22 23:14:25.896 INFO reconfig required, no property group
Sep 22 23:14:25.896 INFO ok, have oxide/crucible/downstairs:downstairs-bd788714-d366-4573-ac0f-64f52c45fa42
Sep 22 23:14:25.896 INFO reconfig required, no property group
Sep 22 23:14:25.896 INFO ok, have oxide/crucible/downstairs:downstairs-b8985630-b020-47f6-828d-9cc22ed12f56
Sep 22 23:14:25.896 INFO reconfig required, no property group
Sep 22 23:14:25.896 INFO reconfig required, no property group
Sep 22 23:14:25.896 INFO creating config property group
Sep 22 23:14:25.896 INFO creating config property group
Sep 22 23:14:25.896 INFO creating config property group
Sep 22 23:14:25.896 INFO reconfiguring oxide/crucible/downstairs:downstairs-bd788714-d366-4573-ac0f-64f52c45fa42
Sep 22 23:14:25.896 INFO reconfiguring oxide/crucible/downstairs:downstairs-6ed761bc-0273-4c2f-b2aa-b5a77c461371
Sep 22 23:14:25.896 INFO ensure directory SCF_TYPE_ASTRING /tmp/.tmp95MCpd/regions/bd788714-d366-4573-ac0f-64f52c45fa42
Sep 22 23:14:25.896 INFO reconfiguring oxide/crucible/downstairs:downstairs-b8985630-b020-47f6-828d-9cc22ed12f56
Sep 22 23:14:25.896 INFO ensure port SCF_TYPE_COUNT 1000
Sep 22 23:14:25.896 INFO ensure directory SCF_TYPE_ASTRING /tmp/.tmpITQLSK/regions/b8985630-b020-47f6-828d-9cc22ed12f56
Sep 22 23:14:25.896 INFO ensure directory SCF_TYPE_ASTRING /tmp/.tmpm5Gb9w/regions/6ed761bc-0273-4c2f-b2aa-b5a77c461371
Sep 22 23:14:25.896 INFO creating config property group
Sep 22 23:14:25.896 INFO ensure address SCF_TYPE_ASTRING 127.0.0.1
Sep 22 23:14:25.896 INFO commit
Sep 22 23:14:25.896 INFO ensure port SCF_TYPE_COUNT 1000
Sep 22 23:14:25.896 INFO ok!
Sep 22 23:14:25.896 INFO ensure address SCF_TYPE_ASTRING 127.0.0.1
Sep 22 23:14:25.896 INFO commit
Sep 22 23:14:25.896 INFO ensure port SCF_TYPE_COUNT 1000
Sep 22 23:14:25.896 INFO ok!
Sep 22 23:14:25.896 INFO ensure address SCF_TYPE_ASTRING 127.0.0.1
Sep 22 23:14:25.896 INFO reconfiguring oxide/crucible/downstairs:downstairs-4d83cb9a-5f4f-448d-919c-4d7ce8fc18b4
Sep 22 23:14:25.896 INFO commit
Sep 22 23:14:25.896 INFO ensure directory SCF_TYPE_ASTRING /tmp/.tmpN2BXAr/regions/4d83cb9a-5f4f-448d-919c-4d7ce8fc18b4
Sep 22 23:14:25.896 INFO ok!
Sep 22 23:14:25.896 INFO ensure port SCF_TYPE_COUNT 1000
test test::test_smf_region_create_then_destroy ... ok
Sep 22 23:14:25.896 INFO ensure address SCF_TYPE_ASTRING 127.0.0.1
Sep 22 23:14:25.896 INFO commit
Sep 22 23:14:25.896 INFO region 6ed761bc-0273-4c2f-b2aa-b5a77c461371 state: Created -> Tombstoned, component: test_harness_datafile
Sep 22 23:14:25.896 INFO ok!
Sep 22 23:14:25.896 INFO creating region b8985630-b020-47f6-828d-9cc22ed12f56 snapshot d7527af5-48af-4f15-92ae-500ac492e369, component: test_snapshot_interface
Sep 22 23:14:25.896 INFO creating region bd788714-d366-4573-ac0f-64f52c45fa42 snapshot 3a99346d-25cc-4829-a850-fc77546058b5, component: test_snapshot_interface
test test::test_smf_region_failed ... ok
Sep 22 23:14:25.896 INFO creating missing downstairs instance downstairs-4d83cb9a-5f4f-448d-919c-4d7ce8fc18b4
Sep 22 23:14:25.896 INFO ok, have oxide/crucible/downstairs:downstairs-4d83cb9a-5f4f-448d-919c-4d7ce8fc18b4
Sep 22 23:14:25.896 INFO reconfig required, no property group
Sep 22 23:14:25.896 INFO creating config property group
Sep 22 23:14:25.896 INFO disabling downstairs instance: downstairs-6ed761bc-0273-4c2f-b2aa-b5a77c461371 (instance states: (Some(Online), None))
Sep 22 23:14:25.896 INFO reconfiguring oxide/crucible/downstairs:downstairs-4d83cb9a-5f4f-448d-919c-4d7ce8fc18b4
Sep 22 23:14:25.896 INFO ensure directory SCF_TYPE_ASTRING /tmp/.tmpN2BXAr/regions/4d83cb9a-5f4f-448d-919c-4d7ce8fc18b4
test test::test_smf_datafile_race_region ... ok
Sep 22 23:14:25.896 INFO creating region b8985630-b020-47f6-828d-9cc22ed12f56 snapshot d7527af5-48af-4f15-92ae-500ac492e369 dir "/tmp/.tmpITQLSK/regions/b8985630-b020-47f6-828d-9cc22ed12f56/.zfs/snapshot/d7527af5-48af-4f15-92ae-500ac492e369", component: test_snapshot_interface
Sep 22 23:14:25.896 INFO ensure port SCF_TYPE_COUNT 1000
Sep 22 23:14:25.896 INFO ensure address SCF_TYPE_ASTRING 127.0.0.1
Sep 22 23:14:25.896 INFO creating region bd788714-d366-4573-ac0f-64f52c45fa42 snapshot 3a99346d-25cc-4829-a850-fc77546058b5 dir "/tmp/.tmp95MCpd/regions/bd788714-d366-4573-ac0f-64f52c45fa42/.zfs/snapshot/3a99346d-25cc-4829-a850-fc77546058b5", component: test_snapshot_interface
Sep 22 23:14:25.896 INFO commit
Sep 22 23:14:25.896 INFO ok!
Sep 22 23:14:25.896 INFO requesting running snapshot b8985630-b020-47f6-828d-9cc22ed12f56-d7527af5-48af-4f15-92ae-500ac492e369 state: Requested, component: test_harness_datafile
Sep 22 23:14:25.896 INFO requesting running snapshot bd788714-d366-4573-ac0f-64f52c45fa42-3a99346d-25cc-4829-a850-fc77546058b5 state: Requested, component: test_harness_datafile
test test::test_smf_region ... ok
Sep 22 23:14:25.897 DEBG found expected downstairs instance: downstairs-b8985630-b020-47f6-828d-9cc22ed12f56
Sep 22 23:14:25.897 DEBG found expected downstairs instance: downstairs-bd788714-d366-4573-ac0f-64f52c45fa42
test test::test_smf_region_bounce_idempotent ... ok
Sep 22 23:14:25.897 DEBG do not need to reconfigure oxide/crucible/downstairs:downstairs-b8985630-b020-47f6-828d-9cc22ed12f56
Sep 22 23:14:25.897 DEBG do not need to reconfigure oxide/crucible/downstairs:downstairs-bd788714-d366-4573-ac0f-64f52c45fa42
Sep 22 23:14:25.897 INFO creating missing snapshot instance snapshot-b8985630-b020-47f6-828d-9cc22ed12f56-d7527af5-48af-4f15-92ae-500ac492e369
Sep 22 23:14:25.897 INFO creating missing snapshot instance snapshot-bd788714-d366-4573-ac0f-64f52c45fa42-3a99346d-25cc-4829-a850-fc77546058b5
Sep 22 23:14:25.897 INFO ok, have oxide/crucible/downstairs:snapshot-b8985630-b020-47f6-828d-9cc22ed12f56-d7527af5-48af-4f15-92ae-500ac492e369
Sep 22 23:14:25.897 INFO ok, have oxide/crucible/downstairs:snapshot-bd788714-d366-4573-ac0f-64f52c45fa42-3a99346d-25cc-4829-a850-fc77546058b5
Sep 22 23:14:25.897 INFO reconfig required, no property group
Sep 22 23:14:25.897 INFO reconfig required, no property group
Sep 22 23:14:25.897 INFO reconfiguring oxide/crucible/downstairs:snapshot-b8985630-b020-47f6-828d-9cc22ed12f56-d7527af5-48af-4f15-92ae-500ac492e369
Sep 22 23:14:25.897 INFO reconfiguring oxide/crucible/downstairs:snapshot-bd788714-d366-4573-ac0f-64f52c45fa42-3a99346d-25cc-4829-a850-fc77546058b5
Sep 22 23:14:25.897 INFO ensure directory SCF_TYPE_ASTRING /tmp/.tmpITQLSK/regions/b8985630-b020-47f6-828d-9cc22ed12f56/.zfs/snapshot/d7527af5-48af-4f15-92ae-500ac492e369
Sep 22 23:14:25.897 INFO ensure directory SCF_TYPE_ASTRING /tmp/.tmp95MCpd/regions/bd788714-d366-4573-ac0f-64f52c45fa42/.zfs/snapshot/3a99346d-25cc-4829-a850-fc77546058b5
Sep 22 23:14:25.897 INFO ensure port SCF_TYPE_COUNT 1001
Sep 22 23:14:25.897 INFO ensure port SCF_TYPE_COUNT 1001
Sep 22 23:14:25.897 INFO ensure mode SCF_TYPE_ASTRING ro
Sep 22 23:14:25.897 INFO ensure mode SCF_TYPE_ASTRING ro
Sep 22 23:14:25.897 INFO ensure address SCF_TYPE_ASTRING 127.0.0.1
Sep 22 23:14:25.897 INFO ensure address SCF_TYPE_ASTRING 127.0.0.1
Sep 22 23:14:25.897 INFO commit
Sep 22 23:14:25.897 INFO commit
Sep 22 23:14:25.897 INFO ok!
Sep 22 23:14:25.897 INFO ok!
Sep 22 23:14:25.897 INFO region b8985630-b020-47f6-828d-9cc22ed12f56 running snapshot d7527af5-48af-4f15-92ae-500ac492e369 state: Requested -> Created, component: test_harness_datafile
Sep 22 23:14:25.897 INFO removing running snapshot bd788714-d366-4573-ac0f-64f52c45fa42-3a99346d-25cc-4829-a850-fc77546058b5, component: test_harness_datafile
Sep 22 23:14:25.897 INFO creating missing downstairs instance downstairs-b8985630-b020-47f6-828d-9cc22ed12f56
Sep 22 23:14:25.897 DEBG found expected downstairs instance: downstairs-bd788714-d366-4573-ac0f-64f52c45fa42
Sep 22 23:14:25.897 INFO ok, have oxide/crucible/downstairs:downstairs-b8985630-b020-47f6-828d-9cc22ed12f56
Sep 22 23:14:25.897 INFO reconfig required, no property group
Sep 22 23:14:25.897 INFO disabling snapshot instance: snapshot-bd788714-d366-4573-ac0f-64f52c45fa42-3a99346d-25cc-4829-a850-fc77546058b5 (instance states: (Some(Online), None))
Sep 22 23:14:25.897 INFO creating config property group
Sep 22 23:14:25.897 INFO reconfiguring oxide/crucible/downstairs:downstairs-b8985630-b020-47f6-828d-9cc22ed12f56
Sep 22 23:14:25.897 INFO ensure directory SCF_TYPE_ASTRING /tmp/.tmpITQLSK/regions/b8985630-b020-47f6-828d-9cc22ed12f56
Sep 22 23:14:25.897 DEBG do not need to reconfigure oxide/crucible/downstairs:downstairs-bd788714-d366-4573-ac0f-64f52c45fa42
Sep 22 23:14:25.897 INFO ensure port SCF_TYPE_COUNT 1000
Sep 22 23:14:25.897 INFO ensure address SCF_TYPE_ASTRING 127.0.0.1
Sep 22 23:14:25.897 INFO commit
Sep 22 23:14:25.897 INFO creating missing downstairs instance downstairs-bd788714-d366-4573-ac0f-64f52c45fa42
Sep 22 23:14:25.897 INFO ok, have oxide/crucible/downstairs:downstairs-bd788714-d366-4573-ac0f-64f52c45fa42
Sep 22 23:14:25.897 INFO ok!
Sep 22 23:14:25.897 INFO reconfig required, no property group
Sep 22 23:14:25.897 INFO creating config property group
Sep 22 23:14:25.897 INFO creating missing snapshot instance snapshot-b8985630-b020-47f6-828d-9cc22ed12f56-d7527af5-48af-4f15-92ae-500ac492e369
Sep 22 23:14:25.897 INFO reconfiguring oxide/crucible/downstairs:downstairs-bd788714-d366-4573-ac0f-64f52c45fa42
Sep 22 23:14:25.897 INFO ensure directory SCF_TYPE_ASTRING /tmp/.tmp95MCpd/regions/bd788714-d366-4573-ac0f-64f52c45fa42
Sep 22 23:14:25.897 INFO ok, have oxide/crucible/downstairs:snapshot-b8985630-b020-47f6-828d-9cc22ed12f56-d7527af5-48af-4f15-92ae-500ac492e369
Sep 22 23:14:25.897 INFO reconfig required, no property group
Sep 22 23:14:25.897 INFO ensure port SCF_TYPE_COUNT 1000
Sep 22 23:14:25.897 INFO ensure address SCF_TYPE_ASTRING 127.0.0.1
Sep 22 23:14:25.897 INFO reconfiguring oxide/crucible/downstairs:snapshot-b8985630-b020-47f6-828d-9cc22ed12f56-d7527af5-48af-4f15-92ae-500ac492e369
Sep 22 23:14:25.897 INFO commit
Sep 22 23:14:25.897 INFO ensure directory SCF_TYPE_ASTRING /tmp/.tmpITQLSK/regions/b8985630-b020-47f6-828d-9cc22ed12f56/.zfs/snapshot/d7527af5-48af-4f15-92ae-500ac492e369
Sep 22 23:14:25.897 INFO ok!
Sep 22 23:14:25.897 INFO ensure port SCF_TYPE_COUNT 1001
Sep 22 23:14:25.897 INFO ensure mode SCF_TYPE_ASTRING ro
Sep 22 23:14:25.897 INFO ensure address SCF_TYPE_ASTRING 127.0.0.1
Sep 22 23:14:25.897 INFO commit
Sep 22 23:14:25.897 INFO ok!
test test::test_smf_datafile_race_running_snapshots ... ok
test test::test_smf_running_snapshot ... ok
test tests::test_crucible_agent_openapi ... ok

test result: ok. 11 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.02s

     Running `/work/oxidecomputer/crucible/target/debug/deps/crucible_agent_client-6f3900e8033b57ec --nocapture`

running 0 tests

test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s

     Running `/work/oxidecomputer/crucible/target/debug/deps/crucible_client_types-5ba9f9d411803900 --nocapture`

running 0 tests

test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s

     Running `/work/oxidecomputer/crucible/target/debug/deps/crucible_common-9a32809bdbdf85c4 --nocapture`

running 2 tests
test region::test::test_basic_region ... ok
test region::test::test_region_validate_io ... ok

test result: ok. 2 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s

     Running `/work/oxidecomputer/crucible/target/debug/deps/crucible_control_client-d0a58354872d46d9 --nocapture`

running 0 tests

test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s

     Running `/work/oxidecomputer/crucible/target/debug/deps/crucible_downstairs-dce67baac661a5f4 --nocapture`

running 156 tests
test dump::test::color_compare ... ok
test dump::test::color_compare_one ... ok
test dump::test::color_compare_red0 ... ok
test dump::test::color_compare_red01 ... ok
test dump::test::color_compare_red01_2 ... ok
test dump::test::color_compare_red02 ... ok
test dump::test::color_compare_red02_2 ... ok
test dump::test::color_compare_red1 ... ok
test dump::test::color_compare_red12 ... ok
test dump::test::color_compare_red12_2 ... ok
test dump::test::color_compare_red2 ... ok
test dump::test::color_compare_two ... ok
test dump::test::color_compare_two_red0 ... ok
test dump::test::color_compare_two_red1 ... ok
test region::test::copy_path_basic ... ok
thread 'region::test::bad_import_region' panicked at 'called `Result::unwrap()` on an `Err` value: Error open "/tmp/12345678-1111-2222-3333-123456789999/notadir/region.json": file not found opening region config "/tmp/12345678-1111-2222-3333-123456789999/notadir/region.json"', downstairs/src/region.rs:3466:10
note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace
{{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,,""time":"time":"2023-09-22T23:14:25.972128532Z"2023-09-22T23:14:25.972083262Z","hostname":","hostname":"ip-10-150-1-74.us-west-2.compute.internal"ip-10-150-1-74.us-west-2.compute.internal","pid":,"4298pid":4298}
}
{{,"time":"""msgmsg"2023-09-22T23:14:25.972249247Z:""":","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
Created new region file \"/tmp/.tmptl1G8F/region.json\"","{v":0,""name"msg"::""crucible",Created new region file ,\"""level"/tmp/.tmpUgERFY/region.json:\"time"30":","v":02023-09-22T23:14:25.972467365Z,""name":"crucible",",hostname""{,level"":":time"30ip-10-150-1-74.us-west-2.compute.internal"":",msg"2023-09-22T23:14:25.97206699Z",",pid"":time"::"",4298""hostname"current number of open files limit 65536 is already the maximum2023-09-22T23:14:25.972734886Ztime"",:""":hostname""2023-09-22T23:14:25.972749447Z:""ip-10-150-1-74.us-west-2.compute.internal",ip-10-150-1-74.us-west-2.compute.internal""hostname},""
pid:"":,4298ip-10-150-1-74.us-west-2.compute.internal",}"
pid""{:pid"4298:}4298
}"
msg":"Created new region file \"{/tmp/.tmpr5qBKq/region.json\"","v":0",msg"":"name":"crucible"Created new region file \","level":/tmp/.tmp34a3XL/region.json\"30","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:25.972829052Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":,4298"time":"}
2023-09-22T23:14:25.972836155Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
Created new region file \"/tmp/.tmpEqmWYa/region.json\"",,""v":v0",:"0name":","crucible"name,"":level"":crucible"30,"level":30,"time":"2023-09-22T23:14:25.972915198Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
,,""time"time":":"2023-09-22T23:14:25.972917321Z"2023-09-22T23:14:25.972222273Z","hostname":","hostnameip-10-150-1-74.us-west-2.compute.internal"",":"pid":4298}ip-10-150-1-74.us-west-2.compute.internal"
,"pid":4298}
{"msg":"Created new region file {\"/tmp/.tmpHNTfYZ/region.json\"","v":0,""name":"msg":"crucible","level":30Created new region file \"/tmp/.tmpbmowMo/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:25.972984691Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
,"time":"2023-09-22T23:14:25.972993254Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:25.97635864Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Created new region file \"/tmp/.tmpuVAiXy/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:25.976401415Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
test region::test::bad_import_region - should panic ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:25.978891399Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Created new region file \"/tmp/.tmpAYVwJc/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:25.97903037Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:25.990143615Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:25.990208272Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:25.99024686Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
test region::test::duplicate_context_insert ... ok
test region::test::encryption_context ... ok
test region::test::extent_dir_max ... ok
test region::test::extent_dir_basic ... ok
test region::test::extent_dir_min ... ok
thread 'region::test::extent_io_invalid_block_buf' panicked at 'called `Result::unwrap()` on an `Err` value: DataLenUnaligned', downstairs/src/region.rs:3516:52
thread 'region::test::extent_io_bad_block' panicked at 'called `Result::unwrap()` on an `Err` value: OffsetInvalid', downstairs/src/region.rs:3506:53
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:25.994903769Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
test region::test::extent_io_invalid_block_buf - should panic ... ok
{"msg":"Created new region file \"/tmp/.tmpzs8jke/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:25.995009972Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
test region::test::extent_io_bad_block - should panic ... ok
thread 'region::test::extent_io_non_aligned_large' panicked at 'called `Result::unwrap()` on an `Err` value: DataLenUnaligned', downstairs/src/region.rs:3486:51
thread 'region::test::extent_io_invalid_large' panicked at 'called `Result::unwrap()` on an `Err` value: DataLenUnaligned', downstairs/src/region.rs:3526:51
test region::test::extent_io_non_aligned_large - should panic ... ok
test region::test::extent_io_invalid_large - should panic ... ok
thread 'region::test::extent_io_non_aligned_small' panicked at 'called `Result::unwrap()` on an `Err` value: DataLenUnaligned', downstairs/src/region.rs:3496:51
test region::test::copy_extent_dir_twice ... ok
test region::test::extent_name_basic ... ok
test region::test::extent_io_non_aligned_small - should panic ... ok
test region::test::extent_io_valid ... ok
test region::test::copy_extent_dir ... ok
EXT  BLOCKS GEN0 test region::test::extent_name_basic_ext ... ok
 FL0  D0
  0 000-009test region::test::extent_name_basic_ext_shm ... ok
    0    0 test region::test::extent_name_basic_ext_wal ... ok
  F
  1 010-019    0    0   F
Max gen: 0,  Max flush: 0
test region::test::close_extent ... ok
test region::test::extent_name_basic_three ... ok
test region::test::extent_name_basic_two ... ok
test region::test::extent_name_max ... ok
test region::test::extent_name_min ... ok
test region::test::extent_path_mid_hi ... ok
test region::test::extent_path_mid_lo ... ok
test region::test::extent_path_max ... ok
test region::test::extent_path_min ... ok
test region::test::extent_path_three ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.001733999Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Created new region file \"/tmp/.tmpSzJQkz/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.001791418Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.002046859Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Created new region file \"/tmp/.tmp3IoV6E/region.json\"","v":0,"name":"crucible","level":30test region::test::dump_a_region ... ,"ok
time":"2023-09-22T23:14:26.002101511Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.002212931Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Created new region file \"/tmp/.tmpWC03QK/region.json\"","v":0,"name":"crucible","level":30{"msg",":time"":"2023-09-22T23:14:26.002269491Z"current number of open files limit 65536 is already the maximum","hostname",":v"":0,"ip-10-150-1-74.us-west-2.compute.internalname":"","crucible"pid",:"4298level":}30
,"time":"2023-09-22T23:14:26.002306475Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Opened existing region file \"/tmp/.tmpuVaodS/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.002362049Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.002442781Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
test region::test::new_region ... {ok
"{msg":"Opened existing region file \""/tmp/.tmpBbFuRN/region.json\""msg",:""v":0,"name":"current number of open files limit 65536 is already the maximum"crucible",",level"":v"30:0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.002507432Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
,"time":"2023-09-22T23:14:26.002513225Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.002572997Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.002612601Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30test region::test::region_bad_database_read_version_high ... ok
,"time":"2023-09-22T23:14:26.002659232Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.002709665Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.002747878Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
test region::test::region_bad_database_read_version_low ... ok
test region::test::new_existing_region ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.003291208Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"{msg":"Opened existing region file \""/tmp/.tmpU8e474/region.json\""msg",:""v":0,"name":"current number of open files limit 65536 is already the maximum"crucible",",level"":v30":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.003349992Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
,"time":"{2023-09-22T23:14:26.003356061Z",""hostname"msg":":"Database read version 1ip-10-150-1-74.us-west-2.compute.internal"",","pid"v"::42980,"name}"
:"crucible","level":30{"msg":"Opened existing region file \"/tmp/.tmpc0jGKx/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.003404304Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
,"time":"2023-09-22T23:14:26.003417495Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.003458766Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
test region::test::region_bad_database_write_version_high ... ok
test region::test::region_bad_database_write_version_low ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.003993244Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Created new region file \"/tmp/.tmpL9TBhh/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.004071049Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.004149715Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"{Created new region file \"/tmp/.tmp1rT7EC/region.json\"","v"":0msg,"":name"":"crucible","level":current number of open files limit 65536 is already the maximum30","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.004201177Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
,"time":"2023-09-22T23:14:26.004209413Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Created new region file \"/tmp/.tmpI32qSh/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.004266727Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.004467655Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.004548628Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{{"msg":""msg":"Created new region file \"Created new region file /tmp/.tmpvhyZdk/region.json\"\""/tmp/.tmpfJBJTP/region.json\",""v":,0","v"name:"0:","crucible"name":","crucible"level",":level":3030,"time":"2023-09-22T23:14:26.004661369Z","hostname":",ip-10-150-1-74.us-west-2.compute.internal""time":,""pid":2023-09-22T23:14:26.004661091Z4298","}hostname
":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.005318733Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.00537805Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.005412648Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.00685219Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.006903006Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.006943371Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
EXT  BLOCKS GEN0 GEN1  FL0 FL1  D0 D1
  0 000-009    0    0    0   0   F  F
  1 010-019    0    0    0   0   F  F
Max gen: 0,  Max flush: 0
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.013423279Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.013472145Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.013498377Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
test region::test::multiple_context ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.014843257Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Created new region file \"/tmp/.tmpF7g92b/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.014948678Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
cp "/tmp/.tmpfJBJTP/00/000/000" to "/tmp/.tmpfJBJTP/00/000/001.copy/001"
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.016681823Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Opened existing region file \"/tmp/.tmpL9TBhh/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.016737138Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30Recreate "/tmp/.tmpfJBJTP/00/000/001.db-shm"
,"time":"2023-09-22T23:14:26.016776455Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.016811348Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
test region::test::dump_two_region ... ok
Recreate "/tmp/.tmpfJBJTP/00/000/001.db-wal"
{"msg":"Extent 1 found replacement dir, finishing replacement","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.017148086Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Copy files from \"/tmp/.tmpfJBJTP/00/000/001.replace\" in \"/tmp/.tmpfJBJTP/00/000\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.017207141Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{{"msg"":"msg":Remove old file \""/tmp/.tmpfJBJTP/00/000/001.db-shm\"current number of open files limit 65536 is already the maximum" as there is no replacement",",v"":v":00,","name"name"::""crucible"crucible",",level"":level"30:30,"time":"2023-09-22T23:14:26.017510447Z","hostname":","time":"ip-10-150-1-74.us-west-2.compute.internal"2023-09-22T23:14:26.017511604Z",",pid"":hostname4298":"}
ip-10-150-1-74.us-west-2.compute.internal","pid":4298{}
"msg":"Remove old file \"{/tmp/.tmpfJBJTP/00/000/001.db-wal\""msg" as there is no replacement":","v"Created new region file \":0/tmp/.tmpbHZr8I/region.json\"",,""vname""::0","name":"cruciblecrucible"","level":,30"level":30,"time":"2023-09-22T23:14:26.017580597Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
,"time":"2023-09-22T23:14:26.017582411Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Move directory  \"/tmp/.tmpfJBJTP/00/000/001.replace\" to \"/tmp/.tmpfJBJTP/00/000/001.completed\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.017640245Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
test region::test::reopen_extent_cleanup_one ... ok
{"msg":"Extent 1 found replacement dir, finishing replacement","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.018768685Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Copy files from \"/tmp/.tmpvhyZdk/00/000/001.replace\" in \"/tmp/.tmpvhyZdk/00/000\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.01881309Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Move directory  \"/tmp/.tmpvhyZdk/00/000/001.replace\" to \"/tmp/.tmpvhyZdk/00/000/001.completed\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.018929836Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.019438724Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Created new region file \"/tmp/.tmpMyPH8a/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.019516476Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
test region::test::reopen_extent_cleanup_replay_short ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.02207878Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Created new region file \"/tmp/.tmpxFGvC6/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.02213245Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
test region::test::reopen_extent_cleanup_replay ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.023013937Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Created new region file \"/tmp/.tmpROqSgO/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.023076575Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
test region::test::region_create_drop_open ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.025399734Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.025449946Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30{","timemsg"":":"2023-09-22T23:14:26.025488848Z"Failed write hash validation","hostname":,""v":0,"ip-10-150-1-74.us-west-2.compute.internal"name",:""cruciblepid"":,"4298level":}50
,"time":"2023-09-22T23:14:26.025522244Z","           Extent 2
hostname":GEN      "ip-10-150-1-74.us-west-2.compute.internal","       0 pid":       0 4298
}FLUSH_ID 
{       0        0 
DIRTY    "msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.025587684Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Created new region file \"/tmp/.tmpn46IR3/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.025662398Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
                  

BLOCK  D0 D1  C0 C1 DIFF 
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.025921102Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.025959661Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.025980307Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
test region::test::test_bad_hash_bad ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.027553163Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Created new region file \"/tmp/.tmpLMYlj2/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.027676313Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
test region::test::reopen_extent_cleanup_two ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30test region::test::reopen_all_extents ... ok
,"time":"2023-09-22T23:14:26.029723851Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Created new region file \"/tmp/.tmpV4jWy1/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.029922075Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.030547955Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Created new region file \"/tmp/.tmpnFW2oG/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.030699753Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
test region::test::test_blank_block_read_ok ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.03217067Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.032246235Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.032274887Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.032668418Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Created new region file \"/tmp/.tmpYEI8V4/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.032704524Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
test region::test::reopen_extent_no_replay_readonly ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.033989388Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Created new region file \"/tmp/.tmpPVxrNK/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.03404737Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
test region::test::test_extent_write_flush_close ... ok
020   A  A   A  A     
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.037371368Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.037550668Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Created new region file \"/tmp/.tmpaig8rX/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.037606677Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.03774674Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.037794009Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
test region::test::test_big_write ... ok
test region::test::test_extent_close_reopen_flush_close ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.038745937Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Created new region file \"/tmp/.tmpp33hxJ/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.038821735Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.039088916Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Created new region file \"/tmp/.tmp0OWlpB/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.039165218Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
test region::test::test_flush_extent_limit_too_large ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.041652294Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Created new region file \"/tmp/.tmpGCIag7/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.0417218Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
test region::test::test_flush_extent_limit_base ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.042509141Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Created new region file \"/tmp/.tmpgHBd3w/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.042636269Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.043052852Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.043102212Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.043138051Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
test region::test::test_flush_extent_limit_end ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.045286815Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Created new region file \"/tmp/.tmp09t5Jb/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.045416509Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
test region::test::test_fully_rehash_and_clean_does_not_mark_blocks_as_written ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.047215838Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Created new region file \"/tmp/.tmp58F936/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.047371474Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
021   A  A   A  A     
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.047857508Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.047909278Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.047950461Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
test region::test::test_ok_hash_ok ... ok
test region::test::test_fully_rehash_marks_blocks_unwritten_if_data_never_hit_disk ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.048843097Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Created new region file \"/tmp/.tmpKdyj6e/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.049014463Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.049207482Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Created new region file \"/tmp/.tmprexlHr/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.049273557Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.053461401Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.053510767Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.053545674Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
022   A  A   A  A     
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.056311607Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.05636286Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.056400815Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
test region::test::test_read_multiple_disjoint_none_contiguous ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.059531851Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Created new region file \"/tmp/.tmpEB530l/region.json\"","v":0,"name":"crucible","level":30Send flush to extent limit 0
,"time":"2023-09-22T23:14:26.059611405Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
test region::test::test_read_multiple_disjoint_large_contiguous ... ok
extent 0 should not be dirty now
verify 1 still dirty
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.060279931Z","hostname":"{ip-10-150-1-74.us-west-2.compute.internal","pid":4298"}msg":
"current number of open files limit 65536 is already the maximum"{,"v"":msg":"0,"nameCreated new region file ":\""crucible"/tmp/.tmp0aekN1/region.json\",""level":,"30v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.060345116Z",","time":hostname"":"2023-09-22T23:14:26.06034024Z","ip-10-150-1-74.us-west-2.compute.internal"hostname":,""pid":4298ip-10-150-1-74.us-west-2.compute.internal"},
"pid":4298}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.060402374Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.060440904Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
verify 2 still dirty
verify 3 still dirty
verify 4 still dirty
verify 5 still dirty
test region::test::test_read_single_large_contiguous ... ok
verify 6 still dirty
verify 7 still dirty
verify 8 still dirty
verify 9 still dirty
Send flush to extent limit 1
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.062350441Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
extent 1 should not be dirty now
test region::test::test_read_single_large_contiguous_span_extents ... ok
verify 2 still dirty
{"msg":"Created new region file \"/tmp/.tmpYOFJTN/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.062505473Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
verify 3 still dirty
verify 4 still dirty
verify 5 still dirty
verify 6 still dirty
verify 7 still dirty
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.063086701Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Created new region file \"/tmp/.tmp1r2Mld/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.06321583Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
verify 8 still dirty
verify 9 still dirty
Send flush to extent limit 2
extent 2 should not be dirty now
verify 3 still dirty
verify 4 still dirty
verify 5 still dirty
verify 6 still dirty
verify 7 still dirty
verify 8 still dirty
verify 9 still dirty
Send flush to extent limit 3
extent 3 should not be dirty now
verify 4 still dirty
verify 5 still dirty
verify 6 still dirty
verify 7 still dirty
verify 8 still dirty
verify 9 still dirty
Send flush to extent limit 4
023   A  A   A  A     
extent 4 should not be dirty now
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","verify 5 still dirty
level":30,"time"verify 6 still dirty
:"2023-09-22T23:14:26.065713263Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
verify 7 still dirty
verify 8 still dirty
verify 9 still dirty
Send flush to extent limit 5
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.066084565Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.06616534Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
extent 5 should not be dirty now
verify 6 still dirty
verify 7 still dirty
verify 8 still dirty
verify 9 still dirty
Send flush to extent limit 6
test region::test::test_region_open_removes_partial_writes ... ok
extent 6 should not be dirty now
verify 7 still dirty
verify 8 still dirty
verify 9 still dirty
Send flush to extent limit 7
extent 7 should not be dirty now
verify 8 still dirty
{verify 9 still dirty
"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30Send flush to extent limit 8
,"time":"2023-09-22T23:14:26.067394229Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Created new region file \"/tmp/.tmpFxX9XQ/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.067473788Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
extent 8 should not be dirty now
verify 9 still dirty
Send flush to extent limit 9
extent 9 should not be dirty now
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.070354518Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.070405693Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.070501654Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
test region::test::test_flush_extent_limit_walk_it_off ... ok
024   A  A   A  A     
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.07315167Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Created new region file \"/tmp/.tmpdCShqT/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.073232169Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.073285592Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.073333713Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.073371937Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
test region::test::test_write_multiple_disjoint_large_contiguous ... ok
test region::test::test_write_multiple_disjoint_none_contiguous ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30test region::test::test_write_single_large_contiguous ... ok
,"time":"2023-09-22T23:14:26.076556839Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Created new region file \"/tmp/.tmpu2Fu9O/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.076714669Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.077055495Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.077130547Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Created new region file \"{/tmp/.tmphRuxRF/region.json\""",msg""v:"":0,"Created new region file name\"":"crucible"/tmp/.tmp2B4zKP/region.json\"",","level"v:":030,"name":"crucible","level":30,,""timetime""::""2023-09-22T23:14:26.077208922Z2023-09-22T23:14:26.077205881Z"",,""hostnamehostname""::""ip-10-150-1-74.us-west-2.compute.internalip-10-150-1-74.us-west-2.compute.internal"",,""pidpid""::42984298}}

{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.078021362Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.078069332Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.078108793Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
test region::test::test_write_single_large_contiguous_span_extents ... ok
Total size: 15360
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.081614625Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
025   A  A   A  A     
{"msg":"Created new region file \"/tmp/.tmp5OTYZn/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.081758341Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.081882654Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.081928924Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.081966592Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
test region::test::test_write_unwritten_big_write ... ok
buffer size:2048
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.084601655Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Created new region file \"/tmp/.tmpACQl0A/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.084654837Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
Read eid: 0, 0 offset: Block { value: 0, shift: 9 }
{Read eid: 0, 1 offset: Block { value: 1, shift: 9 }
Read eid: 0, 2 offset: Block { value: 2, shift: 9 }
"Read eid: 0, 3 offset: Block { value: 3, shift: 9 }
msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.084947044Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.084998137Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"Read a region, append
2023-09-22T23:14:26.085038808Z"Read a region, append
,"hostname":Read a region, append
"Read a region, append
ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
Total size: 20480
test region::test::test_write_unwritten_big_write_partial_final ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.087852986Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Created new region file \"/tmp/.tmpguYfqg/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.087908077Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
test region::test::test_write_unwritten_big_write_partial_0 ... ok
test region::test::test_write_unwritten_big_write_partial_1 ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.089227833Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Created new region file \"/tmp/.tmptWvoUN/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.089291335Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.089434242Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Created new region file \"/tmp/.tmpuhsTvB/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.089533952Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
026   A  A   A  A     
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.090632829Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.09068206Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.090716528Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.093016468Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.093065937Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.093103345Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
test region::test::test_write_unwritten_big_write_partial_sparse ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.096343914Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Created new region file \"/tmp/.tmpKj0abr/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.096532793Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
test region::test::test_write_unwritten_multiple_disjoint_large_contiguous ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.097478628Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Created new region file \"/tmp/.tmpsnGu6h/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.0975328Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
test region::test::test_write_unwritten_multiple_disjoint_none_contiguous ... ok
027   A  A   A  A     
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.097977173Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.098028398Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.098081105Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
test region::test::test_write_unwritten_single_large_contiguous ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.098273785Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Created new region file \"/tmp/.tmpNq9xEH/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.09839773Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
test region::test::validate_repair_files_also_good ... ok
test region::test::validate_repair_files_duplicate ... ok
test region::test::validate_repair_files_duplicate_pair ... ok
test region::test::validate_repair_files_empty ... ok
test region::test::validate_repair_files_good ... ok
test region::test::validate_repair_files_not_good_enough ... ok
test region::test::validate_repair_files_offbyon ... ok
test region::test::validate_repair_files_quad_duplicate ... ok
test region::test::validate_repair_files_too_good ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.101511766Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.101553615Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.101575722Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
test region::test::test_write_unwritten_when_empty ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.101843821Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Created new region file \"/tmp/.tmpiSwTUM/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.101975905Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
test region::test::test_write_unwritten_single_large_contiguous_span_extents ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.102502915Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Created new region file \"/tmp/.tmpgZFRJj/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.102560824Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
test region::test::test_write_unwritten_when_written ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.102903811Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Created new region file \"/tmp/.tmpplfrpC/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.10298642Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.103628894Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Created new region file \"/tmp/.tmpuu6Uye/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.103682937Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
test region::test::test_write_unwritten_when_written_flush ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.105133894Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Created new region file \"/tmp/.tmpChHAJv/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.105212462Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
028   A  A   A  A     
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.10608331Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.106133584Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.106167927Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
files: ["001", "001.db", "001.db-shm", "001.db-wal"]
test repair::test::extent_expected_files_fail ... ok
test repair::test::extent_expected_files ... ok
files: ["001", "001.db"]
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.11102439Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Created new region file \"/tmp/.tmp6LDZTO/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.111136129Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
test repair::test::extent_expected_files_fail_two ... ok
files: ["001", "001.db"]
test repair::test::extent_expected_files_short ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.112963279Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.113011982Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.113046612Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.113133911Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Created new region file \"/tmp/.tmp9ZmoLF/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.113186171Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.113476324Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Created new region file \"/tmp/.tmp3VBj3m/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.113551998Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
test repair::test::extent_expected_files_short_with_close ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.114442264Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Created new region file \"/tmp/.tmpl7sOI4/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.114505052Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
029   A  A   A  A     
test region::test::dump_extent ... ok
{"msg":"1002 job Flush for connection UpstairsConnection { upstairs_id: ef0699b7-a7d2-4976-b7ae-bba467e5372f, session_id: 7bbe9283-f318-4c62-98b0-765ed5da40e5, gen: 0 } waiting on 1 deps","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:26.122281576Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
test repair::test::test_crucible_repair_openapi ... ok
test test::job_dep_not_satisfied ... ok
test test::jobs_extent_flush_close ... ok
test test::jobs_extent_close ... ok
test test::jobs_extent_live_noop ... ok
test test::jobs_extent_live_reopen ... ok
test test::jobs_extent_live_repair ... ok
test test::jobs_independent ... ok
test test::jobs_write_unwritten ... ok
{"msg":"1003 job Read for connection UpstairsConnection { upstairs_id: 7cc10823-5dc1-45d3-b4e3-caebc1707232, session_id: 09a31877-3cc1-45a5-900a-ac6d12bbfc0d, gen: 0 } waiting on 2 deps","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:26.129170867Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"1003 job Read for connection UpstairsConnection { upstairs_id: da404d58-d3b5-45a9-8123-7d7ea17331ee, session_id: 470ece38-bc72-4a95-acd5-ab8a19a83232, gen: 0 } waiting on 2 deps","v":0,"name":"crucible","level":40{"msg",:""time":"2023-09-22T23:14:26.129688383Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","1003 job Read for connection UpstairsConnection { upstairs_id: 7cc10823-5dc1-45d3-b4e3-caebc1707232, session_id: 09a31877-3cc1-45a5-900a-ac6d12bbfc0d, gen: 0 } waiting on 1 depspid"":4298,"v":}
0,"name":"crucible","level":40,"time":"2023-09-22T23:14:26.129738173Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
Import file_size: 51200  Extent size: 5120  Needed extents: 10
Region already large enough for image
Importing "/tmp/.tmpU4tedt/random_data" to region
{"msg":"1003 job Read for connection UpstairsConnection { upstairs_id: da404d58-d3b5-45a9-8123-7d7ea17331ee, session_id: 470ece38-bc72-4a95-acd5-ab8a19a83232, gen: 0 } waiting on 1 deps","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:26.130684165Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
test test::out_of_order_arrives_after_first_do_work ... ok
test test::out_of_order_arrives_after_1001_completes ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.132192601Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
Import file_size: 51200  Extent size: 5120  Needed extents: 10
Region already large enough for image
Importing "/tmp/.tmpYqlDGl/random_data" to region
{"msg":"Created new region file \"/tmp/.tmpFWGEYo/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.13233155Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
Import file_size: 51300  Extent size: 5120  Needed extents: 11
Extending region to fit image
{"msg":"1003 job Read for connection UpstairsConnection { upstairs_id: e7023dfa-b4ee-47ff-86db-76391d641315, session_id: cff987a0-69f9-40a4-bed7-56a371434922, gen: 0 } waiting on 2 deps","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:26.134379625Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
Import file_size: 51100  Extent size: 5120  Needed extents: 10
Region already large enough for image
Importing "/tmp/.tmpCjoWLo/random_data" to region
{"msg":"1003 job Read for connection UpstairsConnection { upstairs_id: e7023dfa-b4ee-47ff-86db-76391d641315, session_id: cff987a0-69f9-40a4-bed7-56a371434922, gen: 0 } waiting on 1 deps","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:26.135453879Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
Importing "/tmp/.tmphz84Vr/random_data" to region
test test::out_of_order_arrives_after_first_push_next_jobs ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.137841284Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Created new region file \"/tmp/.tmpbuYlX2/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.137894612Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.13907102Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Opened existing region file \"/tmp/.tmpFWGEYo/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.139107903Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.139128476Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.139146926Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"UUID: 450d54ff-1683-4680-b6a5-24cc9ce0daf7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.141874587Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Blocks per extent:4 Total Extents: 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.142221537Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"UpstairsConnection { upstairs_id: d057f4b7-1b10-42a3-88cf-51785901d47d, session_id: 9afe45e4-1ca0-4807-99fc-8653b658d629, gen: 10 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.142269388Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Attempting RW takeover from UpstairsConnection { upstairs_id: d057f4b7-1b10-42a3-88cf-51785901d47d, session_id: 9afe45e4-1ca0-4807-99fc-8653b658d629, gen: 10 } to UpstairsConnection { upstairs_id: d057f4b7-1b10-42a3-88cf-51785901d47d, session_id: 9afe45e4-1ca0-4807-99fc-8653b658d629, gen: 10 }","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:26.142555358Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Signaling to UpstairsConnection { upstairs_id: d057f4b7-1b10-42a3-88cf-51785901d47d, session_id: 9afe45e4-1ca0-4807-99fc-8653b658d629, gen: 10 } thread that UpstairsConnection { upstairs_id: d057f4b7-1b10-42a3-88cf-51785901d47d, session_id: 9afe45e4-1ca0-4807-99fc-8653b658d629, gen: 10 } is being promoted (read-write)","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:26.142623106Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Crucible Downstairs promoting UpstairsConnection { upstairs_id: d057f4b7-1b10-42a3-88cf-51785901d47d, session_id: 9afe45e4-1ca0-4807-99fc-8653b658d629, gen: 10 } to active, discarding 1 jobs","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:26.142669582Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"UpstairsConnection { upstairs_id: d057f4b7-1b10-42a3-88cf-51785901d47d, session_id: 9afe45e4-1ca0-4807-99fc-8653b658d629, gen: 10 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.142710766Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.142945978Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Opened existing region file \"/tmp/.tmpbuYlX2/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.143000967Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.143037686Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.143071166Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
test test::test_complete_work_can_see_none ... ok
{"msg":"UUID: 83f98483-6271-40ff-97e5-08827ab14d83","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.1445917Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Blocks per extent:4 Total Extents: 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.144641319Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"UpstairsConnection { upstairs_id: 2a2fb5a4-15e7-480f-9a28-a07bb7742f73, session_id: e0427225-64a8-4195-a672-02c0760a2888, gen: 10 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.144680031Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Attempting RW takeover from UpstairsConnection { upstairs_id: 2a2fb5a4-15e7-480f-9a28-a07bb7742f73, session_id: e0427225-64a8-4195-a672-02c0760a2888, gen: 10 } to UpstairsConnection { upstairs_id: a1d15123-7067-4c3f-89e4-378e79794086, session_id: 1d8586e9-d692-46f3-84af-ff5f40a5cfdd, gen: 11 }","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:26.144947683Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Signaling to UpstairsConnection { upstairs_id: 2a2fb5a4-15e7-480f-9a28-a07bb7742f73, session_id: e0427225-64a8-4195-a672-02c0760a2888, gen: 10 } thread that UpstairsConnection { upstairs_id: a1d15123-7067-4c3f-89e4-378e79794086, session_id: 1d8586e9-d692-46f3-84af-ff5f40a5cfdd, gen: 11 } is being promoted (read-write)","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:26.145001289Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Crucible Downstairs promoting UpstairsConnection { upstairs_id: a1d15123-7067-4c3f-89e4-378e79794086, session_id: 1d8586e9-d692-46f3-84af-ff5f40a5cfdd, gen: 11 } to active, discarding 1 jobs","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:26.145043244Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"UpstairsConnection { upstairs_id: a1d15123-7067-4c3f-89e4-378e79794086, session_id: 1d8586e9-d692-46f3-84af-ff5f40a5cfdd, gen: 11 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.145081931Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"UpstairsConnection { upstairs_id: 2a2fb5a4-15e7-480f-9a28-a07bb7742f73, session_id: e0427225-64a8-4195-a672-02c0760a2888, gen: 10 } cannot grab work lock, 2a2fb5a4-15e7-480f-9a28-a07bb7742f73 is not active!","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:26.145128869Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.145302405Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Created new region file \"/tmp/.tmpuhw6gO/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.145403449Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
test test::test_complete_work_cannot_see_none_different_upstairs_id ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.148326206Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Created new region file \"/tmp/.tmpHEvATG/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.148391903Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.14907689Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Opened existing region file \"/tmp/.tmpuhw6gO/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.14911482Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.149145675Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.149171253Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"UUID: 72c34938-e54c-40f2-a16e-12d71ec544ba","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.150261575Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Blocks per extent:4 Total Extents: 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.150307918Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"UpstairsConnection { upstairs_id: 6dc1f99f-e171-42d7-963b-de7a3686232f, session_id: 24b68982-d2b2-4031-8e67-90612b3f5688, gen: 10 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.150349014Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Attempting RW takeover from UpstairsConnection { upstairs_id: 6dc1f99f-e171-42d7-963b-de7a3686232f, session_id: 24b68982-d2b2-4031-8e67-90612b3f5688, gen: 10 } to UpstairsConnection { upstairs_id: 6dc1f99f-e171-42d7-963b-de7a3686232f, session_id: d8c24f4b-68d5-42a8-9e9e-cc29439323c1, gen: 11 }","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:26.150568121Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Signaling to UpstairsConnection { upstairs_id: 6dc1f99f-e171-42d7-963b-de7a3686232f, session_id: 24b68982-d2b2-4031-8e67-90612b3f5688, gen: 10 } thread that UpstairsConnection { upstairs_id: 6dc1f99f-e171-42d7-963b-de7a3686232f, session_id: d8c24f4b-68d5-42a8-9e9e-cc29439323c1, gen: 11 } is being promoted (read-write)","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:26.150623741Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Crucible Downstairs promoting UpstairsConnection { upstairs_id: 6dc1f99f-e171-42d7-963b-de7a3686232f, session_id: d8c24f4b-68d5-42a8-9e9e-cc29439323c1, gen: 11 } to active, discarding 1 jobs","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:26.150666361Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"UpstairsConnection { upstairs_id: 6dc1f99f-e171-42d7-963b-de7a3686232f, session_id: d8c24f4b-68d5-42a8-9e9e-cc29439323c1, gen: 11 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.150712043Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"UpstairsConnection { upstairs_id: 6dc1f99f-e171-42d7-963b-de7a3686232f, session_id: 24b68982-d2b2-4031-8e67-90612b3f5688, gen: 10 } cannot grab lock, does not match UpstairsConnection { upstairs_id: 6dc1f99f-e171-42d7-963b-de7a3686232f, session_id: d8c24f4b-68d5-42a8-9e9e-cc29439323c1, gen: 11 }!","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:26.150757257Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
test test::test_complete_work_cannot_see_none_same_upstairs_id ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.153124179Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Created new region file \"/tmp/.tmp7Bkl4G/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.153177953Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.154067254Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Opened existing region file \"/tmp/.tmpHEvATG/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.154121508Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.154156951Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.154190102Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"UUID: 90179e5a-16a5-46d5-b74c-1dd8822109a4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.161512812Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Blocks per extent:4 Total Extents: 5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.161564063Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"UpstairsConnection { upstairs_id: 55411ce4-0884-4ba0-b98a-b2cd8a769c36, session_id: c062eae8-f235-4a59-ae71-0f843aaba046, gen: 10 } is now active (read-write)","v":0,"name":"crucible","level":30Active Upstairs connections: [UpstairsConnection { upstairs_id: 55411ce4-0884-4ba0-b98a-b2cd8a769c36, session_id: c062eae8-f235-4a59-ae71-0f843aaba046, gen: 10 }]
,"time":"2023-09-22T23:14:26.162478477Z","hostname":"Crucible Downstairs work queue:
ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
DSW:[1000]  EClose  New deps:[]
DSW:[1001] EFClose  New deps:[]
DSW:[1002]  ReOpen  New deps:[JobId(1000)]
DSW:[1003]  ReOpen  New deps:[JobId(1001)]
Done tasks []
last_flush: JobId(0)
--------------------------------------
Got new work: [JobId(1000), JobId(1001), JobId(1002), JobId(1003)]
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.16332699Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Opened existing region file \"/tmp/.tmp7Bkl4G/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.163365072Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.163387295Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.163421776Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"UUID: 84243b75-8316-472c-ba11-8b81676900ba","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.167513495Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Blocks per extent:4 Total Extents: 5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.167579086Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"UpstairsConnection { upstairs_id: 0cb5c48e-8289-45ce-9f41-b89f63ddb194, session_id: 27220b61-bec0-40c8-a3f7-47a239b4bea0, gen: 10 } is now active (read-write)","Before doing work we have:
v":0,"name":"crucible"Active Upstairs connections: [UpstairsConnection { upstairs_id: 0cb5c48e-8289-45ce-9f41-b89f63ddb194, session_id: 27220b61-bec0-40c8-a3f7-47a239b4bea0, gen: 10 }]
,"level":30Crucible Downstairs work queue:
DSW:[1000]  EClose  New deps:[]
DSW:[1001] EFClose  New deps:[]
DSW:[1002]    Read  New deps:[JobId(1000), JobId(1001)]
,"DSW:[1003]    NoOp  New deps:[JobId(1000), JobId(1001), JobId(1002)]
time":"DSW:[1004]  ReOpen  New deps:[JobId(1000), JobId(1001), JobId(1002), JobId(1003)]
2023-09-22T23:14:26.168476029ZDone tasks []
"last_flush: JobId(0)
--------------------------------------
,"hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
Got new work: [JobId(1000), JobId(1001), JobId(1002), JobId(1003), JobId(1004)]
Do IOop 1000
test test::test_extent_new_close_flush_close ... ok
Got m: ExtentLiveCloseAck { upstairs_id: 0cb5c48e-8289-45ce-9f41-b89f63ddb194, session_id: 27220b61-bec0-40c8-a3f7-47a239b4bea0, job_id: JobId(1000), result: Ok((0, 0, false)) }
Do IOop 1001
Got m: ExtentLiveCloseAck { upstairs_id: 0cb5c48e-8289-45ce-9f41-b89f63ddb194, session_id: 27220b61-bec0-40c8-a3f7-47a239b4bea0, job_id: JobId(1001), result: Ok((0, 0, false)) }
Do IOop 1002
{Got m: ReadResponse { upstairs_id: 0cb5c48e-8289-45ce-9f41-b89f63ddb194, session_id: 27220b61-bec0-40c8-a3f7-47a239b4bea0, job_id: JobId(1002), responses: Ok([ReadResponse { eid: 2, offset: Block { value: 1, shift: 9 }, data: b"\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", block_contexts: [] }]) }
"msg":"current number of open files limit 65536 is already the maximum","v":Do IOop 1003
0,"name":"crucible","level":30Got m: ExtentLiveAckId { upstairs_id: 0cb5c48e-8289-45ce-9f41-b89f63ddb194, session_id: 27220b61-bec0-40c8-a3f7-47a239b4bea0, job_id: JobId(1003), result: Ok(()) }
,"time":"2023-09-22T23:14:26.16976289Z","hostname"Do IOop 1004
:"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Created new region file \"/tmp/.tmpoEhsiG/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.169822164Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
Got m: ExtentLiveAckId { upstairs_id: 0cb5c48e-8289-45ce-9f41-b89f63ddb194, session_id: 27220b61-bec0-40c8-a3f7-47a239b4bea0, job_id: JobId(1004), result: Ok(()) }
test test::test_extent_simple_close_flush_close ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.173656182Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Created new region file \"/tmp/.tmp2IKeMN/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.17371271Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.17651627Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Opened existing region file \"/tmp/.tmpoEhsiG/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.176570048Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.176606838Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.176639789Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"UUID: 7b8a2fdd-e7fa-41fb-ba6d-29998f3b93da","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.179445329Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Blocks per extent:4 Total Extents: 5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.179496721Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
Active Upstairs connections: [UpstairsConnection { upstairs_id: e9c958d8-148a-44b9-9739-f11441835204, session_id: bf487ac1-bcd6-432a-ae01-13b6988c7e6a, gen: 10 }]
{"Crucible Downstairs work queue:
msg":"current number of open files limit 65536 is already the maximum"DSW:[1000]   Write  New deps:[]
,"v":DSW:[1001]  EClose  New deps:[JobId(1000)]
0Done tasks []
,"last_flush: JobId(0)
name":--------------------------------------
"crucible","level":30Got new work: [JobId(1000), JobId(1001)]
,"time":"2023-09-22T23:14:26.180039179Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Opened existing region file \"/tmp/.tmp2IKeMN/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.180092871Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.18013116Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.180166215Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"UpstairsConnection { upstairs_id: e9c958d8-148a-44b9-9739-f11441835204, session_id: bf487ac1-bcd6-432a-ae01-13b6988c7e6a, gen: 10 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.180248354Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
test test::test_extent_write_close ... ok
{"msg":"UUID: b58633dc-e777-4cbe-9ca4-68b1a51741e9","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.183806866Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Blocks per extent:4 Total Extents: 5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.183857971Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Active Upstairs connections: [UpstairsConnection { upstairs_id: d7339ef2-846e-45a9-97f5-26e0bae2d192, session_id: c28c0787-e835-4833-a63b-8e7c4ed91207, gen: 10 }]
UpstairsConnection { upstairs_id: d7339ef2-846e-45a9-97f5-26e0bae2d192, session_id: c28c0787-e835-4833-a63b-8e7c4ed91207, gen: 10 } is now active (read-write)","v":0,"name":"Crucible Downstairs work queue:
crucible","level":30DSW:[1000]   Write  New deps:[]
DSW:[1001] EFClose  New deps:[JobId(1000)]
Done tasks []
last_flush: JobId(0)
--------------------------------------
,"time":"2023-09-22T23:14:26.184537609Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pidGot new work: [JobId(1000), JobId(1001)]
":4298}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.184645205Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Created new region file \"/tmp/.tmpxzz7iE/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.184709004Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
test test::test_extent_write_flush_close ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.188717442Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Created new region file \"/tmp/.tmpse7v29/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.188771852Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.191613881Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Opened existing region file \"/tmp/.tmpxzz7iE/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.191668199Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.191705916Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.191740916Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"UUID: 28bb9eac-6aa0-42de-b72b-6604429b5dd2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.193774961Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Blocks per extent:4 Total Extents: 5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.193822683Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{Active Upstairs connections: [UpstairsConnection { upstairs_id: 77cb7963-731b-490c-9ce7-14fe9ddb89d1, session_id: c86ee0fc-1408-4020-bc81-8ca87eab267c, gen: 10 }]
"msg":"Crucible Downstairs work queue:
UpstairsConnection { upstairs_id: 77cb7963-731b-490c-9ce7-14fe9ddb89d1, session_id: c86ee0fc-1408-4020-bc81-8ca87eab267c, gen: 10 } is now active (read-write)","v":0,"name":"DSW:[1000]   Write  New deps:[]
crucible"DSW:[1001]   Flush  New deps:[]
,"level"DSW:[1002]   Write  New deps:[JobId(1000), JobId(1001)]
:30DSW:[1003]  EClose  New deps:[JobId(1000), JobId(1001), JobId(1002)]
Done tasks []
last_flush: JobId(0)
--------------------------------------
Got new work: [JobId(1000), JobId(1001), JobId(1002), JobId(1003)]
,"time":"2023-09-22T23:14:26.194515469Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.194873908Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Opened existing region file \"/tmp/.tmpse7v29/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.194929156Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.194968162Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.195003852Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"UUID: 808c8e16-3443-43e9-bf16-febf4fb4541f","v":0,"name":"crucible","level":30test test::test_extent_write_flush_write_close ... ok
,"time":"2023-09-22T23:14:26.198199678Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Blocks per extent:4 Total Extents: 5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.198250481Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.198831078Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
Active Upstairs connections: [UpstairsConnection { upstairs_id: 3037f5d2-30e9-4f18-b4e1-7161671a03e0, session_id: 1988a056-8d38-43bd-b8fa-9545cfa696cc, gen: 10 }]
Crucible Downstairs work queue:
DSW:[1000]   Write  New deps:[]
DSW:[1001]   Write  New deps:[]
DSW:[1002] EFClose  New deps:[JobId(1000)]
DSW:[1003]  EClose  New deps:[JobId(1001)]
Done tasks []
last_flush: JobId(0)
--------------------------------------
Got new work: [JobId(1000), JobId(1001), JobId(1002), JobId(1003)]
{"msg":"Created new region file \"/tmp/.tmpVilIEa/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.198978104Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"UpstairsConnection { upstairs_id: 3037f5d2-30e9-4f18-b4e1-7161671a03e0, session_id: 1988a056-8d38-43bd-b8fa-9545cfa696cc, gen: 10 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.199063612Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
test test::test_extent_write_write_flush_close ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.203786258Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Created new region file \"/tmp/.tmpxQvbkS/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.203838589Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.204123342Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Opened existing region file \"/tmp/.tmpVilIEa/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.204179172Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.204217068Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.204253771Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"UUID: b66f61a5-b14a-4ce4-ba9b-6a4c98e2915e","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.204892173Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Blocks per extent:4 Total Extents: 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.204939831Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
test test::test_multiple_read_only_no_job_id_collision ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.207734953Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Created new region file \"/tmp/.tmpQXuMCi/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.20778794Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.207979613Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Opened existing region file \"/tmp/.tmpxQvbkS/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.208026557Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.208061142Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.208095123Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"UUID: bcd6b1a8-f754-4069-b16b-256b9067e2d1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.208769287Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Blocks per extent:4 Total Extents: 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.208820241Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
test test::test_promote_to_active_multi_read_only_different_uuid ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.211286419Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Created new region file \"/tmp/.tmpIHYD46/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.211365797Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.211685564Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Opened existing region file \"/tmp/.tmpQXuMCi/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.211740076Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.211778726Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.211816963Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"UUID: 3adb2941-a761-4ec0-9c56-75f1fee90eab","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.212559781Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Blocks per extent:4 Total Extents: 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.21261141Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Signaling to UpstairsConnection { upstairs_id: 9ceb7609-1b50-43ef-b9d8-5723c7697fa1, session_id: d28eb097-f28a-423a-8e1e-3fae5a9e9ada, gen: 1 } thread that UpstairsConnection { upstairs_id: 9ceb7609-1b50-43ef-b9d8-5723c7697fa1, session_id: 9a2048e7-004c-4c26-8dd7-e00cb0be5dd2, gen: 1 } is being promoted (read-only)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.213660364Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
test test::test_promote_to_active_multi_read_only_same_uuid ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.215485645Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Created new region file \"/tmp/.tmpDI35aR/region.json\"","v":0,"name":"crucible","level":30{"msg":","timecurrent number of open files limit 65536 is already the maximum"":","v2023-09-22T23:14:26.215563331Z"":,"0hostname":","name":"crucibleip-10-150-1-74.us-west-2.compute.internal"",","pid"level"::429830}
,"time":"2023-09-22T23:14:26.21559619Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Opened existing region file \"/tmp/.tmpIHYD46/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.215653646Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.215695044Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.215732092Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"UUID: de931ed0-806c-4581-98d6-a0ba1b93aff8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.216798146Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Blocks per extent:4 Total Extents: 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.216848901Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
ds1: MutexGuard { value: Downstairs { region: Region { dir: "/tmp/.tmpIHYD46", def: RegionDefinition { block_size: 512, extent_size: Block { value: 4, shift: 9 }, extent_count: 2, uuid: de931ed0-806c-4581-98d6-a0ba1b93aff8, encrypted: false, database_read_version: 1, database_write_version: 1 }, extents: [Mutex { data: Opened(Extent { number: 0, read_only: false, block_size: 512, extent_size: Block { value: 4, shift: 9 }, iov_max: 1024, inner: Mutex { data: Inner { file: File { fd: 234 }, metadb: Connection { path: Some("/tmp/.tmpIHYD46/00/000/000.db") }, dirty_blocks: {} } } }) }, Mutex { data: Opened(Extent { number: 1, read_only: false, block_size: 512, extent_size: Block { value: 4, shift: 9 }, iov_max: 1024, inner: Mutex { data: Inner { file: File { fd: 238 }, metadb: Connection { path: Some("/tmp/.tmpIHYD46/00/000/001.db") }, dirty_blocks: {} } } }) }], dirty_extents: {}, read_only: false, log: Logger() }, lossy: false, read_errors: false, write_errors: false, flush_errors: false, active_upstairs: {}, dss: DsStatOuter { ds_stat_wrap: Mutex { is_locked: false, has_waiters: false } }, read_only: false, encrypted: false, address: None, repair_address: None, log: Logger() }, mutex: Mutex { is_locked: true, has_waiters: false } }

ds2: MutexGuard { value: Downstairs { region: Region { dir: "/tmp/.tmpIHYD46", def: RegionDefinition { block_size: 512, extent_size: Block { value: 4, shift: 9 }, extent_count: 2, uuid: de931ed0-806c-4581-98d6-a0ba1b93aff8, encrypted: false, database_read_version: 1, database_write_version: 1 }, extents: [Mutex { data: Opened(Extent { number: 0, read_only: false, block_size: 512, extent_size: Block { value: 4, shift: 9 }, iov_max: 1024, inner: Mutex { data: Inner { file: File { fd: 234 }, metadb: Connection { path: Some("/tmp/.tmpIHYD46/00/000/000.db") }, dirty_blocks: {} } } }) }, Mutex { data: Opened(Extent { number: 1, read_only: false, block_size: 512, extent_size: Block { value: 4, shift: 9 }, iov_max: 1024, inner: Mutex { data: Inner { file: File { fd: 238 }, metadb: Connection { path: Some("/tmp/.tmpIHYD46/00/000/001.db") }, dirty_blocks: {} } } }) }], dirty_extents: {}, read_only: false, log: Logger() }, lossy: false, read_errors: false, write_errors: false, flush_errors: false, active_upstairs: {19701549-8c58-40ed-bb17-7c74ded37f68: ActiveUpstairs { upstairs_connection: UpstairsConnection { upstairs_id: 19701549-8c58-40ed-bb17-7c74ded37f68, session_id: 9db573ae-8d0b-42da-a5b8-c3f109f81491, gen: 2 }, work: Mutex { is_locked: false, has_waiters: false }, terminate_sender: Sender { chan: Tx { inner: Chan { tx: Tx { block_tail: 0x57d0050, tail_position: 0 }, semaphore: Semaphore { semaphore: Semaphore { permits: 1 }, bound: 1 }, rx_waker: AtomicWaker, tx_count: 1, rx_fields: "..." } } } }}, dss: DsStatOuter { ds_stat_wrap: Mutex { is_locked: false, has_waiters: false } }, read_only: false, encrypted: false, address: None, repair_address: None, log: Logger() }, mutex: Mutex { is_locked: true, has_waiters: false } }

{"msg":"UpstairsConnection { upstairs_id: 19701549-8c58-40ed-bb17-7c74ded37f68, session_id: 9db573ae-8d0b-42da-a5b8-c3f109f81491, gen: 2 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.217792854Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Attempting RW takeover from UpstairsConnection { upstairs_id: 19701549-8c58-40ed-bb17-7c74ded37f68, session_id: 9db573ae-8d0b-42da-a5b8-c3f109f81491, gen: 2 } to UpstairsConnection { upstairs_id: 5e379a22-3c2f-4551-9fc9-ad3870755d7a, session_id: 97bba431-3401-427f-a7bd-ebb3b3269626, gen: 1 }","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:26.217855605Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
test test::test_promote_to_active_multi_read_write_different_uuid_lower_gen ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.21897165Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Opened existing region file \"/tmp/.tmpDI35aR/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.219025033Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.219078156Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.219114676Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.219412542Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Created new region file \"/tmp/.tmpfNTSSm/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.21949228Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"UUID: 52f7f69b-43c7-4f37-9439-1be23add535f","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.22038355Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Blocks per extent:4 Total Extents: 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.220435295Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"UpstairsConnection { upstairs_id: e908b04f-d364-4854-bb49-6f0d5be2289f, session_id: 3f4bb7da-417a-4c11-9b21-bcda36aee8d2, gen: 1 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.220836786Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Attempting RW takeover from UpstairsConnection { upstairs_id: e908b04f-d364-4854-bb49-6f0d5be2289f, session_id: 3f4bb7da-417a-4c11-9b21-bcda36aee8d2, gen: 1 } to UpstairsConnection { upstairs_id: 398953cd-7d91-4ad5-9bbc-a75405eababb, session_id: 81c7c98c-90e2-4ce3-a767-d7ce7df9ca4e, gen: 1 }","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:26.22089295Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
Populated 11 extents by copying 51712 bytes (101 blocks)
test test::test_promote_to_active_multi_read_write_different_uuid_same_gen ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.223063653Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Created new region file \"/tmp/.tmpcxko0X/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.223120712Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.224112428Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Opened existing region file \"/tmp/.tmpfNTSSm/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.224200848Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.224263863Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.224313961Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"UUID: 803d56cc-604b-4a1e-914d-3094ef59ab32","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.225367209Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Blocks per extent:4 Total Extents: 2Populated 10 extents by copying 51200 bytes (100 blocks)
","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.225438036Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
Export total_size: 56320  Extent size:5120  Total Extents:11
Exporting from start_block: 0  count:101
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.225967251Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Opened existing region file \"/tmp/.tmpcxko0X/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.22602287Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.226069793Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.226109482Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
Populated 10 extents by copying 51200 bytes (100 blocks)
{"msg":"UpstairsConnection { upstairs_id: 8b47ade1-a217-4690-9bae-35fa541c9998, session_id: 49b5d662-290a-41d8-a1f6-e053df67278f, gen: 1 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.226420954Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Attempting RW takeover from UpstairsConnection { upstairs_id: 8b47ade1-a217-4690-9bae-35fa541c9998, session_id: 49b5d662-290a-41d8-a1f6-e053df67278f, gen: 1 } to UpstairsConnection { upstairs_id: 8b47ade1-a217-4690-9bae-35fa541c9998, session_id: 42e0ccac-f200-45c4-84e8-d7825764fe9a, gen: 2 }","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:26.226498619Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Signaling to UpstairsConnection { upstairs_id: 8b47ade1-a217-4690-9bae-35fa541c9998, session_id: 49b5d662-290a-41d8-a1f6-e053df67278f, gen: 1 } thread that UpstairsConnection { upstairs_id: 8b47ade1-a217-4690-9bae-35fa541c9998, session_id: 42e0ccac-f200-45c4-84e8-d7825764fe9a, gen: 2 } is being promoted (read-write)","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:26.226621161Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"UpstairsConnection { upstairs_id: 8b47ade1-a217-4690-9bae-35fa541c9998, session_id: 42e0ccac-f200-45c4-84e8-d7825764fe9a, gen: 2 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.226662042Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
test test::test_promote_to_active_multi_read_write_same_uuid_larger_gen ... {ok
"msg":"UUID: 4892109d-aab0-4aea-b071-15726cbd6530","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.227387872Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Blocks per extent:4 Total Extents: 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.227440221Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"UpstairsConnection { upstairs_id: 73dae26e-1e29-42b5-9ed1-ce87c9338289, session_id: 782ed9a4-b5c3-4a6a-9a4e-ac45001d1939, gen: 1 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.227833783Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Attempting RW takeover from UpstairsConnection { upstairs_id: 73dae26e-1e29-42b5-9ed1-ce87c9338289, session_id: 782ed9a4-b5c3-4a6a-9a4e-ac45001d1939, gen: 1 } to UpstairsConnection { upstairs_id: 73dae26e-1e29-42b5-9ed1-ce87c9338289, session_id: 804a6b78-70a2-40e8-a86f-1ff5c54737d2, gen: 1 }","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:26.22789618Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.228046523Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Created new region file \"/tmp/.tmpar7POn/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.228168799Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
test test::test_promote_to_active_multi_read_write_same_uuid_same_gen ... ok
Export total_size: 51200  Extent size:5120  Total Extents:10
Exporting from start_block: 0  count:100
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.22963038Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Created new region file \"/tmp/.tmp24NmSA/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.229693997Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30Populated 10 extents by copying 51200 bytes (100 blocks)
,"time":"2023-09-22T23:14:26.231319321Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Opened existing region file \"/tmp/.tmpar7POn/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.231373768Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.23141143Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.231447561Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"UUID: 8a20b68b-256d-4dc5-a918-562bc0077aac","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.232780858Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Blocks per extent:4 Total Extents: 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.232832195Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.232887272Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Opened existing region file \"/tmp/.tmp24NmSA/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.232943129Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.232986993Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.233028181Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
test test::test_promote_to_active_one_read_only ... ok
{"msg":"UUID: 54bbf592-cabd-40af-9fe7-2224ae97382e","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.23444435Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Blocks per extent:4 Total Extents: 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.234491653Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
Export total_size: 51200  Extent size:5120  Total Extents:10
Exporting from start_block: 0  count:100
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.234698789Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Created new region file \"/tmp/.tmpZVPXLF/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.234765401Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"UpstairsConnection { upstairs_id: 6a624d39-d4f6-4cc6-9590-d28983cf34a0, session_id: 22876128-f147-4ab2-bf2d-9b227d57490d, gen: 1 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.234877443Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
test test::test_promote_to_active_one_read_write ... ok
Read and wrote out 101 blocks
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.236322431Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Created new region file \"/tmp/.tmp31s0dl/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.236378205Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.238942272Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Opened existing region file \"/tmp/.tmpZVPXLF/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.239015188Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.239047283Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.239070003Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"UUID: 74f443cd-4ab5-4618-8512-dc2ae037a0c8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.241224385Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Blocks per extent:4 Total Extents: 2","v":0,"name":"crucible","level":30Active Upstairs connections: [UpstairsConnection { upstairs_id: b2f7a8d9-e891-4b8a-9d71-3b7f09ce387c, session_id: 7c0d32a6-2649-495c-bbdb-ce1c249a507e, gen: 10 }]
,"time":"2023-09-22T23:14:26.241274163Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid"Crucible Downstairs work queue:
:4298}
{DSW:[1000]    Read  New deps:[]
"msg":"DSW:[1001]    Read  New deps:[JobId(1000)]
Done tasks []
UpstairsConnection { upstairs_id: b2f7a8d9-e891-4b8a-9d71-3b7f09ce387c, session_id: 7c0d32a6-2649-495c-bbdb-ce1c249a507e, gen: 10 } is now active (read-write)"last_flush: JobId(0)
,--------------------------------------
"v":0,"name":"crucible","level":30Got new work: [JobId(1000), JobId(1001)]
,"time":"2023-09-22T23:14:26.241328529Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal",Do IOop 1000
"pid":4298}
Got m: ReadResponse { upstairs_id: b2f7a8d9-e891-4b8a-9d71-3b7f09ce387c, session_id: 7c0d32a6-2649-495c-bbdb-ce1c249a507e, job_id: JobId(1000), responses: Ok([ReadResponse { eid: 0, offset: Block { value: 1, shift: 9 }, data: b"\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", block_contexts: [] }]) }
Read and wrote out 100 blocks
Do IOop 1001
Got m: ReadResponse { upstairs_id: b2f7a8d9-e891-4b8a-9d71-3b7f09ce387c, session_id: 7c0d32a6-2649-495c-bbdb-ce1c249a507e, job_id: JobId(1001), responses: Ok([ReadResponse { eid: 1, offset: Block { value: 1, shift: 9 }, data: b"\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", block_contexts: [] }]) }
Active Upstairs connections: [UpstairsConnection { upstairs_id: b2f7a8d9-e891-4b8a-9d71-3b7f09ce387c, session_id: 7c0d32a6-2649-495c-bbdb-ce1c249a507e, gen: 10 }]
Crucible Downstairs work queue:  Empty
Done tasks [JobId(1000), JobId(1001)]
last_flush: JobId(0)
--------------------------------------
test test::import_test_too_large ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.242666118Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Created new region file \"/tmp/.tmpZP1sMk/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.242824519Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
test test::test_simple_read ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.244652353Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Created new region file \"/tmp/.tmpqubsKp/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.244815755Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.245035445Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Opened existing region file \"/tmp/.tmp31s0dl/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.245069895Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.245089506Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.245107525Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
test test::import_test_basic ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.248789924Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Created new region file \"/tmp/.tmpFvOuef/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.248934775Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"UUID: 1607a550-5075-492a-b42f-150b765059f9","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.250136793Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Blocks per extent:4 Total Extents: 5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.250184161Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
test test::import_test_basic_read_blocks ... ok
{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: ed48f294784d46ea7d4bb99336918b74358eca46\nCommit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main\nrustc: 1.70.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: true Opt level: 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.251251991Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298,"task":"main"}
{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.251303619Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298,"task":"main"}
{"msg":"Using address: 127.0.0.1:5557","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.251343545Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298,"task":"main"}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.251495761Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Created new region file \"/tmp/.tmpFAaLjw/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.251578725Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Repair listens on 127.0.0.1:5558","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.251777199Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298,"task":"repair"}
{"msg":"listening","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.252108444Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298,"local_addr":"127.0.0.1:5558","task":"repair"}
{"msg":"Using repair address: 127.0.0.1:5558","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.252357869Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298,"task":"main"}
{"msg":"No SSL acceptor configured","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.252413869Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298,"task":"main"}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.252866559Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Opened existing region file \"/tmp/.tmpZP1sMk/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.252917336Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.252955086Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.252990163Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.253447464Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Opened existing region file \"/tmp/.tmpqubsKp/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.253501449Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":{4298}
"msg":"{listening on 127.0.0.1:5557"",msg":""v":Database read version 1"0,",v"":name0":,""crucible"name":","crucible"level",":level":3030,"time":"2023-09-22T23:14:26.253551767Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal",,""pid"time:"4298:"}
2023-09-22T23:14:26.253551923Z","hostname"{:""msg"ip-10-150-1-74.us-west-2.compute.internal":","pid":Database write version 1"4298,"v":,0","task"name":":"crucible"main","level"}:
30,"time":"2023-09-22T23:14:26.253605261Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"accepted connection from 127.0.0.1:49975","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.254060438Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298,"task":"main"}
{"msg":"Connection request from 935b1a66-7a6c-4fa1-8abe-908a1d7bfaac with version 3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.254399187Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298,"task":"proc"}
{"msg":"connection (127.0.0.1:49975) Exits with error: Required version 4, Or [3] got 3","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:14:26.254589893Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.254831371Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Opened existing region file \"/tmp/.tmpFvOuef/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.254881402Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.254918866Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.254961852Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
Read and wrote out 100 blocks
test test::test_version_downrev ... ok
{"msg":"UUID: e85e0099-05b3-4835-89a6-80c1c5211ec4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.257199191Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Blocks per extent:4 Total Extents: 5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.257243214Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.257655Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Opened existing region file \"/tmp/.tmpFAaLjw/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.257709618Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.25774797Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.257798309Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"UUID: d7e10717-6bc5-4ee2-8774-7145f63de841","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.258520172Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Blocks per extent:4 Total Extents: 5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.25858243Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: ed48f294784d46ea7d4bb99336918b74358eca46\nCommit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main\nrustc: 1.70.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: true Opt level: 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.258951015Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298,"task":"main"}
{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.259008384Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298,"task":"main"}
{"msg":"Using address: 127.0.0.1:5555","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.2590481Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298,"task":"main"}
{"msg":"Repair listens on 127.0.0.1:5556","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.259303649Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298,"task":"repair"}
{"msg":"listening","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.259376454Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298,"local_addr":"127.0.0.1:5556","task":"repair"}
{"msg":"Using repair address: 127.0.0.1:5556","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.259617206Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298,"task":"main"}
{"msg":"No SSL acceptor configured","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.259663042Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298,"task":"main"}
{"msg":"listening on 127.0.0.1:5555","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.259723179Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298,"task":"main"}
{"msg":"accepted connection from 127.0.0.1:40010","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.260064501Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298,"task":"main"}
{"msg":"Connection request from f5694906-48f3-4395-8bea-4227cf12f387 with version 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.260297344Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298,"task":"proc"}
{{"msg":""msg":upstairs UpstairsConnection { upstairs_id: f5694906-48f3-4395-8bea-4227cf12f387, session_id: 406accdd-25fa-4bb6-b7ec-24d088ab7351, gen: 1 } connected, version 4"","v":0UUID: db42931f-715c-43d7-abac-4e896436b1b0","name":",crucible"",v"":level":300,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.260362385Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298,"task":"proc"}
,"time":"2023-09-22T23:14:26.260368346Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Blocks per extent:4 Total Extents: 5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.260423607Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"1002 job Read for connection UpstairsConnection { upstairs_id: 4d29a5db-9e90-4c9c-922a-08d1aebbce46, session_id: b4ff52ee-dd5c-407f-acf1-59120c5095cb, gen: 0 } waiting on 1 deps","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:26.260648127Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: ed48f294784d46ea7d4bb99336918b74358eca46\nCommit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main\nrustc: 1.70.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: true Opt level: 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.261114904Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298,"task":"main"}
{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.261158802Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298,"task":"main"}
{"msg":"Using address: 127.0.0.1:5561","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.261182088Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298,"task":"main"}
{"msg":"Repair listens on 127.0.0.1:5562","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.261513509Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298,"task":"repair"}
{"msg":"listening","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.261664164Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298,"local_addr":"127.0.0.1:5562","task":"repair"}
test test::import_test_too_small ... ok
{"msg":"Using repair address: 127.0.0.1:5562","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.261899373Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298,"task":"main"}
{"msg":"No SSL acceptor configured","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.261957877Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298,"task":"main"}
{"msg":"listening on 127.0.0.1:5561","v":0,"name":"crucible","level":{30"{msg":""msg":"Crucible Version: Crucible Version: 0.0.1\n,"2002 job Flush for connection UpstairsConnection { upstairs_id: 4d29a5db-9e90-4c9c-922a-08d1aebbce46, session_id: b4ff52ee-dd5c-407f-acf1-59120c5095cb, gen: 0 } waiting on 1 depstimeCommit SHA: ed48f294784d46ea7d4bb99336918b74358eca46"\n:""2023-09-22T23:14:26.262001644Z",","v"hostname"::"0,"name":ip-10-150-1-74.us-west-2.compute.internal"Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main,"\npid"":4298rustc: 1.70.0 stable x86_64-unknown-illumos\ncrucible",","taskCargo: x86_64-unknown-illumos  Debug: true Opt level: 0"":level"","mainv""}:
0,"name":"crucible","level":30:40,"time":"2023-09-22T23:14:26.262048284Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298,"task":"main","}
time":"2023-09-22T23:14:26.262053724Z"{,"hostname":""msg":"ip-10-150-1-74.us-west-2.compute.internal","Upstairs <-> Downstairs Message Version: 4"pid,""v"::0,"4298name":"crucible","level":}30
,"time":"2023-09-22T23:14:26.262099812Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298,"task":"main"}
{"msg":"Using address: 127.0.0.1:5563","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.262123948Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298,"task":"main"}
{"msg":"Repair listens on 127.0.0.1:5564","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.262349862Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298,"task":"repair"}
{"msg":"listening","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.262438324Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298,"local_addr":"127.0.0.1:5564","task":"repair"}
{"msg":"accepted connection from 127.0.0.1:36611","v":0,"name":"crucible","level":30{"msg":"Using repair address: 127.0.0.1:5564,""time":","v":2023-09-22T23:14:26.262570016Z"0,",name":""crucible"hostname":,""level":ip-10-150-1-74.us-west-2.compute.internal"30,"pid":4298,"task":"main"}
,"time":"2023-09-22T23:14:26.262603753Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298,"task":"main"}
{"msg":"No SSL acceptor configured","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.262656536Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298,"task":"main"}
{"msg":"listening on 127.0.0.1:5563","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.262698858Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298,"task":"main"}
{"msg":"Connection request from 8917f57d-2ffe-4231-a9c4-639afb7355f1 with version 5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.262973285Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298,"task":"proc"}
{"msg":"downstairs and upstairs using different but compatible versions, Upstairs is 5, but supports [4, 5], downstairs is 4","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:26.263026807Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298,"task":"{proc"}
"msg":"{accepted connection from 127.0.0.1:34699""msg":","v":0,"name":"crucible","level":30upstairs UpstairsConnection { upstairs_id: 8917f57d-2ffe-4231-a9c4-639afb7355f1, session_id: d4ae3f49-545e-4dc9-92b9-296413a08ae6, gen: 1 } connected, version 4","v":0,"name":"crucible","level":30,","time":time"":"2023-09-22T23:14:26.263067864Z"2023-09-22T23:14:26.263063015Z",","hostname":hostname"":"ip-10-150-1-74.us-west-2.compute.internal"ip-10-150-1-74.us-west-2.compute.internal,"","pid"pid:":42984298,",task"":"task":proc""main"}
}
{"msg":"Connection request from ab4ca4d2-5662-4514-b278-a597af86ed19 with version 8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.263232632Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298,"task":"proc"}
{"msg":"downstairs and upstairs using different but compatible versions, Upstairs is 8, but supports [3, 4, 5], downstairs is 4","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:26.263281934Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298,"task":"proc"}
{"msg":"upstairs UpstairsConnection { upstairs_id: ab4ca4d2-5662-4514-b278-a597af86ed19, session_id: 023d6dc1-cb61-433b-973a-d4952af26fd5, gen: 1 } connected, version 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.263325283Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298,"task":"proc"}
test test::test_version_match ... ok
test test::two_job_chains ... ok
test test::unblock_job ... ok
{"msg":"UUID: c9c30354-b358-4f32-b67e-4473d8e10b4f","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.264001501Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Blocks per extent:4 Total Extents: 5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.264037546Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: ed48f294784d46ea7d4bb99336918b74358eca46\nCommit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main\nrustc: 1.70.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: true Opt level: 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.26656329Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298,"task":"main"}
{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.26661995Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298,"task":"main"}
{"msg":"Using address: 127.0.0.1:5579","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.266665994Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298,"task":"main"}
{"msg":"Repair listens on 127.0.0.1:5560","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.266838921Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298,"task":"repair"}
{"msg":"listening","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.266982912Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298,"local_addr":"127.0.0.1:5560","task":"repair"}
{"msg":"Using repair address: 127.0.0.1:5560","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.267096195Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298,"task":"main"}
{"msg":"No SSL acceptor configured","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.267143473Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298,"task":"main"}
test test::test_version_uprev_compatable ... ok
{"msg":"listening on 127.0.0.1:5579","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.267290872Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298,"task":"main"}
{"msg":"1002 job Read for connection UpstairsConnection { upstairs_id: 22fba83c-ad11-4df7-879f-4755fbab107b, session_id: 09611d86-6d2c-4de4-a752-1ff1b5a5a940, gen: 0 } waiting on 1 deps","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:26.267437884Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"1002 job Read for connection UpstairsConnection { upstairs_id: 2b7ff4d2-06f8-47d4-9f52-149c4f0909af, session_id: cf5e0281-b491-4321-b274-e3a7d839621b, gen: 0 } waiting on 1 deps","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:26.267501707Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"1002 job Read for connection UpstairsConnection { upstairs_id: b8c2874c-4b76-449d-92ad-d2d522190d37, session_id: 724a3ccc-41c5-4120-9882-ab3be9385edc, gen: 0 } waiting on 1 deps","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:26.267722781Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
{"msg":"accepted connection from 127.0.0.1:65411","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.267881588Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298,"task":"main"}
{"msg":"Connection request from 859b4fe6-13ef-4e36-b8ad-25ce7fbfbb3e with version 5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:26.267955525Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298,"task":"proc"}
{"msg":"connection (127.0.0.1:65411) Exits with error: Required version 4, Or [5] got 5","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:14:26.268037825Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
test test::test_version_uprev_list ... ok
test test::unblock_job_chain ... ok
test test::unblock_job_chain_first_is_flush ... ok
test test::unblock_job_chain_second_is_flush ... ok
test test::test_version_uprev_only ... ok
test test::you_had_one_job ... ok
{"msg":"1002 job Flush for connection UpstairsConnection { upstairs_id: 3cd5bddc-7b50-446f-b9b5-925e8378021b, session_id: 58226fae-67bc-48ad-ad12-e9aee903862e, gen: 0 } waiting on 1 deps","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:26.270953412Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4298}
test test::unblock_job_upstairs_sends_big_deps ... ok
test region::test::test_flush_after_multiple_disjoint_writes ... ok
test region::test::test_big_extent_full_write_and_flush ... ok

test result: ok. 156 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 1.50s

     Running `/work/oxidecomputer/crucible/target/debug/deps/crucible_downstairs-3ed9735920c1592d --nocapture`

running 0 tests

test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s

     Running `/work/oxidecomputer/crucible/target/debug/deps/crucible_hammer-9622fb9be260fb45 --nocapture`

running 0 tests

test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s

     Running `/work/oxidecomputer/crucible/target/debug/deps/crucible_integration_tests-8902d603847d3610 --nocapture`

running 57 tests
Sep 22 23:14:27.540 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.540 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.540 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.540 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.540 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.540 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.540 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.540 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.544 INFO Created new region file "/tmp/downstairs-4I4s6nIW/region.json"
Sep 22 23:14:27.544 INFO Created new region file "/tmp/downstairs-ibnrqPCO/region.json"
Sep 22 23:14:27.544 INFO Created new region file "/tmp/downstairs-oCVv3Vu2/region.json"
Sep 22 23:14:27.544 INFO Created new region file "/tmp/downstairs-mVeBUKlC/region.json"
Sep 22 23:14:27.544 INFO Created new region file "/tmp/downstairs-7lqwXVde/region.json"
Sep 22 23:14:27.544 INFO Created new region file "/tmp/downstairs-ouxUBI5R/region.json"
Sep 22 23:14:27.544 INFO Created new region file "/tmp/downstairs-uVlKmCnF/region.json"
Sep 22 23:14:27.544 INFO Created new region file "/tmp/downstairs-iZvqbKa5/region.json"
Sep 22 23:14:27.556 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.556 INFO Opened existing region file "/tmp/downstairs-ouxUBI5R/region.json"
Sep 22 23:14:27.556 INFO Database read version 1
Sep 22 23:14:27.556 INFO Database write version 1
Sep 22 23:14:27.558 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.558 INFO Opened existing region file "/tmp/downstairs-ibnrqPCO/region.json"
Sep 22 23:14:27.558 INFO Database read version 1
Sep 22 23:14:27.558 INFO Database write version 1
Sep 22 23:14:27.558 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.558 INFO Opened existing region file "/tmp/downstairs-uVlKmCnF/region.json"
Sep 22 23:14:27.558 INFO Database read version 1
Sep 22 23:14:27.558 INFO Database write version 1
Sep 22 23:14:27.558 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.558 INFO Opened existing region file "/tmp/downstairs-7lqwXVde/region.json"
Sep 22 23:14:27.558 INFO Database read version 1
Sep 22 23:14:27.558 INFO Database write version 1
Sep 22 23:14:27.559 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.559 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.559 INFO Opened existing region file "/tmp/downstairs-iZvqbKa5/region.json"
Sep 22 23:14:27.559 INFO Opened existing region file "/tmp/downstairs-mVeBUKlC/region.json"
Sep 22 23:14:27.559 INFO Database read version 1
Sep 22 23:14:27.559 INFO Database read version 1
Sep 22 23:14:27.559 INFO Database write version 1
Sep 22 23:14:27.559 INFO Database write version 1
Sep 22 23:14:27.560 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.560 INFO Opened existing region file "/tmp/downstairs-oCVv3Vu2/region.json"
Sep 22 23:14:27.560 INFO Database read version 1
Sep 22 23:14:27.560 INFO Database write version 1
Sep 22 23:14:27.563 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.563 INFO Opened existing region file "/tmp/downstairs-4I4s6nIW/region.json"
Sep 22 23:14:27.563 INFO Database read version 1
Sep 22 23:14:27.563 INFO Database write version 1
Sep 22 23:14:27.563 INFO UUID: d2285a39-bf6a-4487-9493-8622023b1e4b
Sep 22 23:14:27.563 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.564 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.564 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.564 INFO Using address: 127.0.0.1:32776, task: main
Sep 22 23:14:27.564 INFO UUID: b63c1426-990b-43e7-baa9-a926ab8df052
Sep 22 23:14:27.564 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.564 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.564 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.564 INFO Using address: 127.0.0.1:45897, task: main
Sep 22 23:14:27.565 INFO UUID: 6b339ff0-fd62-4d3d-96a2-00a3cd97fb5c
Sep 22 23:14:27.565 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.565 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.565 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.565 INFO Using address: 127.0.0.1:43186, task: main
Sep 22 23:14:27.566 INFO UUID: daca29a6-1615-4360-b286-c9a82fbfcd3d
Sep 22 23:14:27.567 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.567 INFO UUID: 97377133-8bb1-47f5-a2f0-3b28d74593db
Sep 22 23:14:27.567 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.567 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.567 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.567 INFO Using address: 127.0.0.1:33768, task: main
Sep 22 23:14:27.567 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.567 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.567 INFO Using address: 127.0.0.1:58086, task: main
Sep 22 23:14:27.567 INFO UUID: 9008985f-4d59-43db-b559-a48d3978dfda
Sep 22 23:14:27.567 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.567 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.567 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.567 INFO Using address: 127.0.0.1:36835, task: main
Sep 22 23:14:27.567 INFO UUID: 122e249d-ed33-49c6-bb2e-b79a59ebf9a6
Sep 22 23:14:27.567 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.567 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.567 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.567 INFO Using address: 127.0.0.1:42970, task: main
Sep 22 23:14:27.567 INFO UUID: 35ccb45b-0c22-4a14-bf83-50fbfaffa7a6
Sep 22 23:14:27.568 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.568 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.568 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.568 INFO Using address: 127.0.0.1:56524, task: main
Sep 22 23:14:27.568 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.568 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.568 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.568 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.568 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.568 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.568 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.568 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.570 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:48710, task: repair
Sep 22 23:14:27.570 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:37143, task: repair
Sep 22 23:14:27.570 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:38525, task: repair
Sep 22 23:14:27.570 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:42417, task: repair
Sep 22 23:14:27.570 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:50356, task: repair
Sep 22 23:14:27.570 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:55024, task: repair
Sep 22 23:14:27.570 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:39068, task: repair
Sep 22 23:14:27.571 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:44953, task: repair
Sep 22 23:14:27.571 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:50356, task: repair
Sep 22 23:14:27.571 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:37143, task: repair
Sep 22 23:14:27.571 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:48710, task: repair
Sep 22 23:14:27.571 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:38525, task: repair
Sep 22 23:14:27.571 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:42417, task: repair
Sep 22 23:14:27.571 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:55024, task: repair
Sep 22 23:14:27.571 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:39068, task: repair
Sep 22 23:14:27.571 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:44953, task: repair
Sep 22 23:14:27.571 INFO listening, local_addr: 127.0.0.1:44953, task: repair
Sep 22 23:14:27.571 INFO listening, local_addr: 127.0.0.1:37143, task: repair
Sep 22 23:14:27.572 INFO listening, local_addr: 127.0.0.1:42417, task: repair
Sep 22 23:14:27.572 INFO listening, local_addr: 127.0.0.1:50356, task: repair
Sep 22 23:14:27.572 INFO listening, local_addr: 127.0.0.1:48710, task: repair
Sep 22 23:14:27.572 INFO listening, local_addr: 127.0.0.1:39068, task: repair
Sep 22 23:14:27.572 INFO listening, local_addr: 127.0.0.1:55024, task: repair
Sep 22 23:14:27.572 INFO listening, local_addr: 127.0.0.1:38525, task: repair
Sep 22 23:14:27.577 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:39068, task: repair
Sep 22 23:14:27.577 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:38525, task: repair
Sep 22 23:14:27.577 INFO Using repair address: 127.0.0.1:39068, task: main
Sep 22 23:14:27.577 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.577 INFO Using repair address: 127.0.0.1:38525, task: main
Sep 22 23:14:27.577 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.577 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:48710, task: repair
Sep 22 23:14:27.577 INFO Using repair address: 127.0.0.1:48710, task: main
Sep 22 23:14:27.577 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.577 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:55024, task: repair
Sep 22 23:14:27.577 INFO Using repair address: 127.0.0.1:55024, task: main
Sep 22 23:14:27.577 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.577 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:42417, task: repair
Sep 22 23:14:27.577 INFO Using repair address: 127.0.0.1:42417, task: main
Sep 22 23:14:27.577 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.577 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:50356, task: repair
Sep 22 23:14:27.577 INFO Using repair address: 127.0.0.1:50356, task: main
Sep 22 23:14:27.577 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.577 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.577 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.577 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.577 INFO Created new region file "/tmp/downstairs-i6X8vigb/region.json"
Sep 22 23:14:27.577 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.577 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:37143, task: repair
Sep 22 23:14:27.577 INFO Created new region file "/tmp/downstairs-Zh66PBhN/region.json"
Sep 22 23:14:27.577 INFO Created new region file "/tmp/downstairs-WtiCXF2Q/region.json"
Sep 22 23:14:27.577 INFO Using repair address: 127.0.0.1:37143, task: main
Sep 22 23:14:27.577 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.577 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:44953, task: repair
Sep 22 23:14:27.577 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.577 INFO Created new region file "/tmp/downstairs-G9ZJdv0G/region.json"
Sep 22 23:14:27.577 INFO Using repair address: 127.0.0.1:44953, task: main
Sep 22 23:14:27.577 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.577 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.577 INFO Created new region file "/tmp/downstairs-oHRV06JF/region.json"
Sep 22 23:14:27.578 INFO Created new region file "/tmp/downstairs-iuS0VtCf/region.json"
Sep 22 23:14:27.578 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.578 INFO Created new region file "/tmp/downstairs-FuV2Esuv/region.json"
Sep 22 23:14:27.578 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.578 INFO Created new region file "/tmp/downstairs-PnHwI8Ne/region.json"
Sep 22 23:14:27.581 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.581 INFO Opened existing region file "/tmp/downstairs-i6X8vigb/region.json"
Sep 22 23:14:27.581 INFO Database read version 1
Sep 22 23:14:27.581 INFO Database write version 1
Sep 22 23:14:27.582 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.582 INFO Opened existing region file "/tmp/downstairs-WtiCXF2Q/region.json"
Sep 22 23:14:27.582 INFO Database read version 1
Sep 22 23:14:27.582 INFO Database write version 1
Sep 22 23:14:27.582 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.582 INFO Opened existing region file "/tmp/downstairs-Zh66PBhN/region.json"
Sep 22 23:14:27.582 INFO Database read version 1
Sep 22 23:14:27.582 INFO Database write version 1
Sep 22 23:14:27.583 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.583 INFO Opened existing region file "/tmp/downstairs-G9ZJdv0G/region.json"
Sep 22 23:14:27.583 INFO Database read version 1
Sep 22 23:14:27.583 INFO Database write version 1
Sep 22 23:14:27.583 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.583 INFO Opened existing region file "/tmp/downstairs-oHRV06JF/region.json"
Sep 22 23:14:27.583 INFO Database read version 1
Sep 22 23:14:27.583 INFO Database write version 1
Sep 22 23:14:27.584 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.584 INFO Opened existing region file "/tmp/downstairs-iuS0VtCf/region.json"
Sep 22 23:14:27.584 INFO Database read version 1
Sep 22 23:14:27.584 INFO Database write version 1
Sep 22 23:14:27.585 INFO UUID: fe394f4d-b22e-45c5-ad25-c835a8779c91
Sep 22 23:14:27.585 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.585 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.585 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.585 INFO Using address: 127.0.0.1:33784, task: main
Sep 22 23:14:27.586 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.586 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:58243, task: repair
Sep 22 23:14:27.586 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:58243, task: repair
Sep 22 23:14:27.586 INFO listening, local_addr: 127.0.0.1:58243, task: repair
Sep 22 23:14:27.586 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:58243, task: repair
Sep 22 23:14:27.586 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.586 INFO Opened existing region file "/tmp/downstairs-FuV2Esuv/region.json"
Sep 22 23:14:27.586 INFO Using repair address: 127.0.0.1:58243, task: main
Sep 22 23:14:27.586 INFO Database read version 1
Sep 22 23:14:27.586 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.586 INFO Database write version 1
Sep 22 23:14:27.586 INFO UUID: 54deed7d-517b-47d9-a428-2bbdc0729fba
Sep 22 23:14:27.586 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.586 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.586 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.586 INFO Using address: 127.0.0.1:52116, task: main
Sep 22 23:14:27.587 INFO UUID: 80932238-df83-4f6e-96fc-9d0aac0f8ced
Sep 22 23:14:27.587 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.587 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.587 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.587 INFO Using address: 127.0.0.1:54745, task: main
Sep 22 23:14:27.587 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.587 INFO UUID: 594f2637-eae4-459c-bc2e-92b501adec4b
Sep 22 23:14:27.587 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.587 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:47921, task: repair
Sep 22 23:14:27.587 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:47921, task: repair
Sep 22 23:14:27.587 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.587 INFO listening, local_addr: 127.0.0.1:47921, task: repair
Sep 22 23:14:27.587 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.587 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.587 INFO Using address: 127.0.0.1:50563, task: main
Sep 22 23:14:27.587 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:53000, task: repair
Sep 22 23:14:27.587 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.587 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:53000, task: repair
Sep 22 23:14:27.587 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:47921, task: repair
Sep 22 23:14:27.587 INFO UUID: 1a8f9cb4-4a64-4013-8234-292244fb0112
Sep 22 23:14:27.587 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.587 INFO Using repair address: 127.0.0.1:47921, task: main
Sep 22 23:14:27.587 INFO listening, local_addr: 127.0.0.1:53000, task: repair
Sep 22 23:14:27.587 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.587 INFO Created new region file "/tmp/downstairs-LpKqZye7/region.json"
Sep 22 23:14:27.587 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.587 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.587 INFO Using address: 127.0.0.1:43400, task: main
Sep 22 23:14:27.587 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:53000, task: repair
Sep 22 23:14:27.587 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.587 INFO Using repair address: 127.0.0.1:53000, task: main
Sep 22 23:14:27.588 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.588 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:62168, task: repair
Sep 22 23:14:27.588 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.588 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:62168, task: repair
Sep 22 23:14:27.588 INFO listening, local_addr: 127.0.0.1:62168, task: repair
Sep 22 23:14:27.588 INFO Created new region file "/tmp/downstairs-U8m0aCFa/region.json"
Sep 22 23:14:27.588 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.588 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:62168, task: repair
Sep 22 23:14:27.588 INFO Using repair address: 127.0.0.1:62168, task: main
Sep 22 23:14:27.588 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:60847, task: repair
Sep 22 23:14:27.588 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.588 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:60847, task: repair
Sep 22 23:14:27.588 INFO listening, local_addr: 127.0.0.1:60847, task: repair
Sep 22 23:14:27.588 INFO UUID: 9191f668-24b7-4176-b853-43cf34e8ccd3
Sep 22 23:14:27.588 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.588 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:60847, task: repair
Sep 22 23:14:27.588 INFO Using repair address: 127.0.0.1:60847, task: main
Sep 22 23:14:27.588 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.588 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.588 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.588 INFO Using address: 127.0.0.1:62060, task: main
Sep 22 23:14:27.588 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.588 INFO Created new region file "/tmp/downstairs-WYIwOmtG/region.json"
Sep 22 23:14:27.588 INFO UUID: 45f3b954-a0cc-44a1-92c8-91ac1da9b5f8
Sep 22 23:14:27.589 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.589 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.589 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.589 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.589 INFO Created new region file "/tmp/downstairs-JIyoAurl/region.json"
Sep 22 23:14:27.589 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.589 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.589 INFO Opened existing region file "/tmp/downstairs-PnHwI8Ne/region.json"
Sep 22 23:14:27.589 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.589 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:55098, task: repair
Sep 22 23:14:27.589 INFO Database read version 1
Sep 22 23:14:27.589 INFO Using address: 127.0.0.1:46381, task: main
Sep 22 23:14:27.589 INFO Database write version 1
Sep 22 23:14:27.589 INFO Created new region file "/tmp/downstairs-wiRB3PPR/region.json"
Sep 22 23:14:27.589 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:55098, task: repair
Sep 22 23:14:27.589 INFO listening, local_addr: 127.0.0.1:55098, task: repair
Sep 22 23:14:27.589 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:55098, task: repair
Sep 22 23:14:27.589 INFO Using repair address: 127.0.0.1:55098, task: main
Sep 22 23:14:27.589 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.589 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.589 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:65411, task: repair
Sep 22 23:14:27.589 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:65411, task: repair
Sep 22 23:14:27.589 INFO listening, local_addr: 127.0.0.1:65411, task: repair
Sep 22 23:14:27.589 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:65411, task: repair
Sep 22 23:14:27.589 INFO Using repair address: 127.0.0.1:65411, task: main
Sep 22 23:14:27.589 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.590 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.590 INFO Created new region file "/tmp/downstairs-VWuvNJ4J/region.json"
Sep 22 23:14:27.590 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.590 INFO Created new region file "/tmp/downstairs-LdYRdLcv/region.json"
Sep 22 23:14:27.592 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.592 INFO Opened existing region file "/tmp/downstairs-LpKqZye7/region.json"
Sep 22 23:14:27.592 INFO Database read version 1
Sep 22 23:14:27.592 INFO Database write version 1
Sep 22 23:14:27.592 INFO UUID: 53cd4d63-78de-43d1-b2db-bd20d98528e7
Sep 22 23:14:27.592 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.592 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.592 INFO Opened existing region file "/tmp/downstairs-U8m0aCFa/region.json"
Sep 22 23:14:27.592 INFO Database read version 1
Sep 22 23:14:27.592 INFO Database write version 1
Sep 22 23:14:27.592 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.592 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.592 INFO Using address: 127.0.0.1:50371, task: main
Sep 22 23:14:27.593 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.593 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.593 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:53985, task: repair
Sep 22 23:14:27.593 INFO Opened existing region file "/tmp/downstairs-WYIwOmtG/region.json"
Sep 22 23:14:27.593 INFO Database read version 1
Sep 22 23:14:27.593 INFO Database write version 1
Sep 22 23:14:27.593 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:53985, task: repair
Sep 22 23:14:27.593 INFO listening, local_addr: 127.0.0.1:53985, task: repair
Sep 22 23:14:27.593 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:53985, task: repair
Sep 22 23:14:27.593 INFO Using repair address: 127.0.0.1:53985, task: main
Sep 22 23:14:27.593 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.594 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.594 INFO Opened existing region file "/tmp/downstairs-wiRB3PPR/region.json"
Sep 22 23:14:27.594 INFO Database read version 1
Sep 22 23:14:27.594 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.594 INFO Database write version 1
Sep 22 23:14:27.594 INFO Opened existing region file "/tmp/downstairs-JIyoAurl/region.json"
Sep 22 23:14:27.594 INFO Database read version 1
Sep 22 23:14:27.594 INFO Database write version 1
Sep 22 23:14:27.597 INFO UUID: 4bb2c32c-5001-4960-9cd0-22694b9c6a8c
Sep 22 23:14:27.597 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.597 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.597 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.597 INFO Using address: 127.0.0.1:39493, task: main
Sep 22 23:14:27.597 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.597 INFO UUID: 67caa349-8834-4aba-8026-7fdaaae2f119
Sep 22 23:14:27.597 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.597 INFO Created new region file "/tmp/downstairs-N9ygFYLS/region.json"
Sep 22 23:14:27.597 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.597 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.597 INFO Using address: 127.0.0.1:56258, task: main
Sep 22 23:14:27.598 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.598 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:51273, task: repair
Sep 22 23:14:27.598 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:51273, task: repair
Sep 22 23:14:27.598 INFO listening, local_addr: 127.0.0.1:51273, task: repair
Sep 22 23:14:27.598 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.598 INFO Opened existing region file "/tmp/downstairs-VWuvNJ4J/region.json"
Sep 22 23:14:27.598 INFO Database read version 1
Sep 22 23:14:27.598 INFO Database write version 1
Sep 22 23:14:27.598 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:51273, task: repair
Sep 22 23:14:27.598 INFO Using repair address: 127.0.0.1:51273, task: main
Sep 22 23:14:27.598 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.598 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.598 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:44814, task: repair
Sep 22 23:14:27.598 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:44814, task: repair
Sep 22 23:14:27.598 INFO listening, local_addr: 127.0.0.1:44814, task: repair
Sep 22 23:14:27.598 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:44814, task: repair
Sep 22 23:14:27.598 INFO Using repair address: 127.0.0.1:44814, task: main
Sep 22 23:14:27.598 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.598 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.598 INFO Opened existing region file "/tmp/downstairs-LdYRdLcv/region.json"
Sep 22 23:14:27.598 INFO Database read version 1
Sep 22 23:14:27.598 INFO Database write version 1
Sep 22 23:14:27.598 INFO UUID: bb12d53e-6697-4186-ab01-3149bf11c327
Sep 22 23:14:27.599 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.599 INFO UUID: 7f911e5b-f4b1-49f1-9a5d-fe84ec8226ed
Sep 22 23:14:27.599 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.599 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.599 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.599 INFO Using address: 127.0.0.1:39722, task: main
Sep 22 23:14:27.599 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.599 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.599 INFO Using address: 127.0.0.1:60582, task: main
Sep 22 23:14:27.599 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.599 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.599 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:60166, task: repair
Sep 22 23:14:27.599 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:40192, task: repair
Sep 22 23:14:27.599 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:60166, task: repair
Sep 22 23:14:27.599 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:40192, task: repair
Sep 22 23:14:27.599 INFO listening, local_addr: 127.0.0.1:60166, task: repair
Sep 22 23:14:27.599 INFO listening, local_addr: 127.0.0.1:40192, task: repair
Sep 22 23:14:27.599 INFO UUID: 922c2bc8-c976-47a3-990e-e4ec311f6c5b
Sep 22 23:14:27.599 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.599 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:60166, task: repair
Sep 22 23:14:27.599 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:40192, task: repair
Sep 22 23:14:27.599 INFO Using repair address: 127.0.0.1:60166, task: main
Sep 22 23:14:27.599 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.599 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.599 INFO Using repair address: 127.0.0.1:40192, task: main
Sep 22 23:14:27.599 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.599 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.599 INFO Using address: 127.0.0.1:33349, task: main
Sep 22 23:14:27.600 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.600 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:46884, task: repair
Sep 22 23:14:27.600 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:46884, task: repair
Sep 22 23:14:27.600 INFO listening, local_addr: 127.0.0.1:46884, task: repair
Sep 22 23:14:27.600 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:46884, task: repair
Sep 22 23:14:27.600 INFO Using repair address: 127.0.0.1:46884, task: main
Sep 22 23:14:27.600 INFO No SSL acceptor configured, task: main
{"msg":"Upstairs starts","v":0,"name":"crucible"{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"level":30{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.601346175Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
,"time":"2023-09-22T23:14:27.601282745Z{","hostname":""ip-10-150-1-74.us-west-2.compute.internal"msg",:""pid":4301Crucible Version: BuildInfo {\n}    version: 
\"0.0.1\",\n    git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",{\n    git_commit_timestamp: \","2023-09-22T22:51:18.000000000Z\"time",:"\n    git_branch: \"2023-09-22T23:14:27.601850906Z"main\",\n,"    rustc_semver: \"hostname"1.70.0\":,\n"    rustc_channel: \"stable\",ip-10-150-1-74.us-west-2.compute.internal"\n,"pid":    rustc_host_triple: \"4301"msg":x86_64-unknown-illumos\"}"
Crucible Version: BuildInfo {,\n\n    version: \"    rustc_commit_sha: 0.0.1{\"\",\n    git_sha: \"90c541806f23a127002de5b4038be731ba1458ca\"",\nmsged48f294784d46ea7d4bb99336918b74358eca46"\":,"\n    cargo_triple: \"Crucible Version: BuildInfo {    git_commit_timestamp: \n\"x86_64-unknown-illumos    version: \"\"2023-09-22T22:51:18.000000000Z0.0.1\"\",,\n\n,\n    git_branch:     git_sha: \"\"    debug: true,main\n\"ed48f294784d46ea7d4bb99336918b74358eca46,\"\n,\n    opt_level: 0,\n    rustc_semver:     git_commit_timestamp: \"\"}"1.70.0\"2023-09-22T22:51:18.000000000Z,\"\n,\n    rustc_channel: ,"\"    git_branch: stable\"\",v":main\n\"0,    rustc_host_triple: \n\","name"    rustc_semver: x86_64-unknown-illumos\"\"1.70.0,\"\n,\n:"    rustc_commit_sha: crucible\"    rustc_channel: \""stable,90c541806f23a127002de5b4038be731ba1458ca\"\",\n,\n"    rustc_host_triple:     cargo_triple: \"\"level":x86_64-unknown-illumos30\"x86_64-unknown-illumos,\"\n,\n    rustc_commit_sha:     debug: true,\"\n    opt_level: 0,\n}"90c541806f23a127002de5b4038be731ba1458ca\",",\nv":    cargo_triple: 0\","namex86_64-unknown-illumos"\":",\ncrucible"    debug: true,\n,"level"    opt_level: 0,:\n}"30,"v":,0","name":"crucible"time",:""level":302023-09-22T23:14:27.602253889Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"}time"
,:""{time"2023-09-22T23:14:27.602267561Z:"","hostname2023-09-22T23:14:27.602277069Z"":{",""msg"hostnameip-10-150-1-74.us-west-2.compute.internal":msg""",:""pid""::4301ip-10-150-1-74.us-west-2.compute.internal"}"
,"Upstairs <-> Downstairs Message Version: 4Upstairs startspid""{:",4301""}msg
",:"v"{"v":"Upstairs <-> Downstairs Message Version: 40:msg"",:,"""v"Upstairs <-> Downstairs Message Version: 4:"name",0",v""name:"0:,""0namecrucible"":,"":levelcrucible"":,30",level"""namecrucible""::,"30"level"crucible,":time"30:"","level":2023-09-22T23:14:27.602351704Z"30,,""hostnametime""::""2023-09-22T23:14:27.602360327Z"ip-10-150-1-74.us-west-2.compute.internal",,""pidhostname""::"4301}
ip-10-150-1-74.us-west-2.compute.internal","pid":4301{}
",msg{":"""time"msg":":Crucible stats registered with UUID: a697641e-15fb-475d-87b4-6f9c9b92b978"",2023-09-22T23:14:27.602365964Z""v"Crucible stats registered with UUID: 66688a47-67b1-4e47-8e76-e0e145532b3a:"0,,""namev""::",crucible0",""name,"":hostname"level""::crucible,30""","time":level":"30ip-10-150-1-74.us-west-2.compute.internal"2023-09-22T23:14:27.602370327Z","pid":,4301,""}hostname":"
,time""ip-10-150-1-74.us-west-2.compute.internal":time,"":"{2023-09-22T23:14:27.602407209Z""pid",2023-09-22T23:14:27.602413581Z":"hostname"msg"",:""hostname":":"4301ip-10-150-1-74.us-west-2.compute.internalip-10-150-1-74.us-west-2.compute.internal"",,""pid"pid:"4301:Crucible stats registered with UUID: 1f7159d2-76c3-4968-a4c3-9cc9b7758094"}}4301
},"
{
{"vmsg"":msg""{:"":0,"name"Crucible 66688a47-67b1-4e47-8e76-e0e145532b3a has session id: 74575cda-5ddb-4e62-8d17-231d991256f7:"",Crucible a697641e-15fb-475d-87b4-6f9c9b92b978 has session id: c1dd7802-7a2b-465d-9b93-1deda33a45f5""vmsg":""",:crucible"Crucible Version: BuildInfo {,0",v""":level\n0"    version: :,name""name:"":\"300.0.1"cruciblecrucible"",,""\"level,"level:"30:\n30    git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n    git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n,"    git_branch: \"time"main,:",\"time""time:"":"2023-09-22T23:14:27.60249869Z"",2023-09-22T23:14:27.602497473Z"",,hostname""\n2023-09-22T23:14:27.602490949Z":hostname,"""hostname":"    rustc_semver: \"ip-10-150-1-74.us-west-2.compute.internal:"",1.70.0\""ip-10-150-1-74.us-west-2.compute.internalpid"":,4301,"}ip-10-150-1-74.us-west-2.compute.internal"pid
":,"4301pid"}:
4301}
{"msg":"\nCrucible 1f7159d2-76c3-4968-a4c3-9cc9b7758094 has session id: 9808e339-fe8d-4257-bb78-48915f23db00",    rustc_channel: \""stable\"v":,\n0,"name":"    rustc_host_triple: \"crucible","x86_64-unknown-illumoslevel"\":,30\n    rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n    cargo_triple: \"x86_64-unknown-illumos\",\n    debug: true,\n    opt_level: 0,\n}","v":,"0time",:""name":"2023-09-22T23:14:27.602582533Z"crucible",","hostname"level":":30ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
,"time":"2023-09-22T23:14:27.60260638Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30The guest has requested activation
,"time":"2023-09-22T23:14:27.602838226Z"The guest has requested activation
,"hostname":"The guest has requested activation
ip-10-150-1-74.us-west-2.compute.internal","The guest has requested activation
pid":4301}
Sep 22 23:14:27.602 INFO UUID: b37a15c7-b55a-4f3b-ab04-a618bf111318
{"msg":"Crucible stats registered with UUID: 9ed4d6bc-e430-4200-9ad2-68905b0ff40c","v":0,"name":"crucible","Sep 22 23:14:27.602 INFO Blocks per extent:5 Total Extents: 2
level":30,"time":"2023-09-22T23:14:27.602914349Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"Crucible 9ed4d6bc-e430-4200-9ad2-68905b0ff40c has session id: 6a66447a-3d57-4eb2-927b-5246d1b47acc","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.602999576Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"Upstairs starts","v":0,"name":"crucible","level":30Sep 22 23:14:27.603 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.603 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.603 INFO Using address: 127.0.0.1:42513, task: main
,"time":"2023-09-22T23:14:27.603091445Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"Crucible Version: BuildInfo {\n    version: \"0.0.1\",\n    git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n    git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n    git_branch: \"main\",\n    rustc_semver: \"1.70.0\",\n    rustc_channel: \"stable\",\n    rustc_host_triple: \"x86_64-unknown-illumos\",\n    rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n    cargo_triple: \"x86_64-unknown-illumos\",\n    debug: true,\n    opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.603348005Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.603395093Z","hostname":"Sep 22 23:14:27.603 INFO UUID: ed248c78-99b2-4b20-8580-19eb24b6c3f5
ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"Crucible stats registered with UUID: 64be6ec7-0ac5-4896-883a-00ec34140b84"Sep 22 23:14:27.603 INFO Blocks per extent:5 Total Extents: 2
,"v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.603446539Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"Crucible 64be6ec7-0ac5-4896-883a-00ec34140b84 has session id: b259cf28-eda8-4b6d-9178-3013a298e183","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.603487313Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
Sep 22 23:14:27.603 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.603 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.603 INFO Using address: 127.0.0.1:49774, task: main
The guest has requested activation
Sep 22 23:14:27.604 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.604 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.604 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.604 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.604 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.604 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.604 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.604 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.604 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.604 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.604 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.604 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:42268, task: repair
Sep 22 23:14:27.604 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.604 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.604 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.604 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:42268, task: repair
Sep 22 23:14:27.604 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.604 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.604 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.604 INFO listening, local_addr: 127.0.0.1:42268, task: repair
Sep 22 23:14:27.604 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:44564, task: repair
Sep 22 23:14:27.604 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:44564, task: repair
Sep 22 23:14:27.604 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.604 INFO Opened existing region file "/tmp/downstairs-N9ygFYLS/region.json"
Sep 22 23:14:27.604 INFO listening, local_addr: 127.0.0.1:44564, task: repair
Sep 22 23:14:27.604 INFO Database read version 1
Sep 22 23:14:27.604 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:42268, task: repair
Sep 22 23:14:27.604 INFO Database write version 1
Sep 22 23:14:27.604 INFO Using repair address: 127.0.0.1:42268, task: main
Sep 22 23:14:27.604 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.604 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:44564, task: repair
Sep 22 23:14:27.604 INFO Using repair address: 127.0.0.1:44564, task: main
Sep 22 23:14:27.604 INFO No SSL acceptor configured, task: main
{"msg":"[0] connecting to 127.0.0.1:32776","v":0,"name":"crucible","level"{:30"msg":"{[0] connecting to 127.0.0.1:45897","v":"0msg",":name"":"crucible"[0] connecting to 127.0.0.1:36835","level,"":v":300,"name":"{crucible","level":30,"time"":"msg":"2023-09-22T23:14:27.605218154Z","[0] connecting to 127.0.0.1:56524"hostname":","v":ip-10-150-1-74.us-west-2.compute.internal"{,0","pid"name:""4301msg:"":"crucible,",[0] connecting to 127.0.0.1:33768"",",,""v"level:time""time"looper::"""0:2023-09-22T23:14:27.605238214Z2023-09-22T23:14:27.605234223Z"""30,:,,""hostname"""name"0":hostname""}:"ip-10-150-1-74.us-west-2.compute.internal"
ip-10-150-1-74.us-west-2.compute.internal,"",pid""pid:"4301:,:""time,crucible"":4301","looper,"":"looper0"":2023-09-22T23:14:27.605269529Z""level"}"
0":,"}30hostname":"
ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"0"}
,"time":"2023-09-22T23:14:27.605310738Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"0"}
{"msg":"[1] connecting to 127.0.0.1:54745","v":0,"name":"crucible","level":30,"{time":"2023-09-22T23:14:27.605546571Z","hostname":""msgip-10-150-1-74.us-west-2.compute.internal"":,""pid":4301[1] connecting to 127.0.0.1:50563","looper",:"Sep 22 23:14:27.605 INFO Upstairs starts
"v"1":0},
"name":"crucible","level":30,"time{":"2023-09-22T23:14:27.605609847Z",""hostname":"msg":"ip-10-150-1-74.us-west-2.compute.internal","[1] connecting to 127.0.0.1:43400pid"Sep 22 23:14:27.605 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
",":v"4301:0,","looper"name":":1""crucible"},
{{"Sep 22 23:14:27.605 INFO Upstairs <-> Downstairs Message Version: 4
"msg":"[2] connecting to 127.0.0.1:39722","v":0,"name":"crucible","level":30,"Sep 22 23:14:27.605 INFO Crucible stats registered with UUID: a50fc3fb-3de2-4743-9c50-cc80cfba77db
time":"2023-09-22T23:14:27.605823038Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"2"}
msg":"[1] connecting to 127.0.0.1:52116"","v":0level,""name":"Sep 22 23:14:27.605 INFO Crucible a50fc3fb-3de2-4743-9c50-cc80cfba77db has session id: b014a775-2625-4651-87a0-a19fedb96440
crucible:30","level":30,","timetime"":":"2023-09-22T23:14:27.605884947Z2023-09-22T23:14:27.605887144Z"",",hostname"":"hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"1"}
ip-10-150-1-74.us-west-2.compute.internal","{pid":4301,"looper":"1""}
msg":"[1] connecting to 127.0.0.1:33784"{,"v":0,""msgname":":""crucible"[2] connecting to 127.0.0.1:60582,""level":,"30v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.605963816Z",,""time"hostname:":""2023-09-22T23:14:27.605969179Z"ip-10-150-1-74.us-west-2.compute.internal",","hostnamepid""::"4301,"ip-10-150-1-74.us-west-2.compute.internallooper"":,""pid"1":4301}
,"looper":"2"}
{"{msg":"[2] connecting to 127.0.0.1:39493""msg":","vup_listen starts"":,0","v"name"::"0crucible,"",name"":level"":crucible30","level":30The guest has requested activation
,"time":","time":2023-09-22T23:14:27.60604096Z"","2023-09-22T23:14:27.606043396Z"hostname":","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":ip-10-150-1-74.us-west-2.compute.internal"4301,"pid":,"4301looper":"2","}task
":"up_listen"}
{"msg":"{up_listen starts","v":"0msg":","name":"crucibleWait for all three downstairs to come online"",","vlevel""::300,"name":"crucible","level":30,"time":"Sep 22 23:14:27.606 INFO listening on 127.0.0.1:0, task: main
2023-09-22T23:14:27.606098935Z,"",time"":"hostname":"2023-09-22T23:14:27.606103539Z","ip-10-150-1-74.us-west-2.compute.internal"hostname",:""pid":4301ip-10-150-1-74.us-west-2.compute.internal",,""pid"task"::"4301up_listen"{}
"}msg"
{:""msg"[2] connecting to 127.0.0.1:56258:""{,"v":Wait for all three downstairs to come online0",,""namev""::0","crucible""name",":level""msg"crucible:":,30""level":30Flush timeout: 0.5","v":0,"name":"crucible","level":30Sep 22 23:14:27.606 INFO listening on 127.0.0.1:0, task: main
,,""timetime""::""2023-09-22T23:14:27.606167938Z2023-09-22T23:14:27.606166662Z"",,""hostnamehostname""::"","time":ip-10-150-1-74.us-west-2.compute.internalip-10-150-1-74.us-west-2.compute.internal"",""pid,"":pid4301"2023-09-22T23:14:27.606174432Z"}:
4301,"hostname"{,:"""msglooper""::""ip-10-150-1-74.us-west-2.compute.internal"Flush timeout: 0.52"",,}""
v"pid"::04301,"name":}"
crucible","level":30{Sep 22 23:14:27.606 INFO listening on 127.0.0.1:0, task: main
{""msg"msg,"":time"":up_listen starts"":","v":02023-09-22T23:14:27.60622089Z"64be6ec7-0ac5-4896-883a-00ec34140b84 active request set",,"",hostnamename""::"""v"crucibleip-10-150-1-74.us-west-2.compute.internal"",,""pid"level"::304301:0},"
name":"crucible","level":30,"time":"2023-09-22T23:14:27.606247706Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal",{"pid":4301"msg,"":"task":"up_listen"}
a697641e-15fb-475d-87b4-6f9c9b92b978 active request set",","time"{v"::"0",msg"":name""2023-09-22T23:14:27.606256787Z":",crucibleWait for all three downstairs to come online"",",""hostname"levelv""::300:","name":"crucible"ip-10-150-1-74.us-west-2.compute.internal",","level":pid"30:4301}
,"time":","2023-09-22T23:14:27.606292501Z"time":","hostname":"2023-09-22T23:14:27.606300362Z","Sep 22 23:14:27.606 INFO [0] connecting to 127.0.0.1:42970, looper: 0
ip-10-150-1-74.us-west-2.compute.internalhostname"",:""pid"ip-10-150-1-74.us-west-2.compute.internal:"4301,"pid"}:
4301}
{{""msg":"msg":"Flush timeout: 0.5"up_listen starts,""v":,0",v"":name"0:","cruciblename"":,""level"crucible":,"30level":30,"time":"2023-09-22T23:14:27.606352442Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal,""time,"":pid"":43012023-09-22T23:14:27.606353878Z"}
,"hostname":"{ip-10-150-1-74.us-west-2.compute.internal"",msg":""pidSep 22 23:14:27.606 INFO accepted connection from 127.0.0.1:47572, task: main
Sep 22 23:14:27.606 INFO accepted connection from 127.0.0.1:43136, task: main
Sep 22 23:14:27.606 INFO [1] connecting to 127.0.0.1:62060, looper: 1
":4301,"task":"up_listen"}
{"msg":"Wait for all three downstairs to come online","1f7159d2-76c3-4968-a4c3-9cc9b7758094 active request setv"":0,"name",:""crucible"v,"":level"0:,30"Sep 22 23:14:27.606 INFO accepted connection from 127.0.0.1:53904, task: main
name":"crucible","level":30,"time":"2023-09-22T23:14:27.606511701Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30Sep 22 23:14:27.606 INFO accepted connection from 127.0.0.1:50653, task: main
,"time":","time"2023-09-22T23:14:27.606521997Z":"2023-09-22T23:14:27.606612421Z,""hostname":,""hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":ip-10-150-1-74.us-west-2.compute.internal"4301,"pid":}4301
}
{"msg":"66688a47-67b1-4e47-8e76-e0e145532b3a active request set","v":0,"name":"crucible","level":30Sep 22 23:14:27.606 INFO [2] connecting to 127.0.0.1:42513, looper: 2
,"time":"2023-09-22T23:14:27.606664215Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[2] connecting to 127.0.0.1:33349",Sep 22 23:14:27.606 INFO accepted connection from 127.0.0.1:33852, task: main
{"v":0,""name":"msgcrucible"":","level":30[0] 64be6ec7-0ac5-4896-883a-00ec34140b84 looper connected"Sep 22 23:14:27.606 INFO accepted connection from 127.0.0.1:52656, task: main
,"v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.606760183Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"2"}
,"time":"{2023-09-22T23:14:27.606775017Z",""msg":"hostname":"up_listen starts","v"ip-10-150-1-74.us-west-2.compute.internal":,"0pid",:"4301name":",crucible"","looper":level"":0"30}
{"msg":","time":"[0] Proc runs for 127.0.0.1:45897 in state New"2023-09-22T23:14:27.606816076Z,""v,"":hostname":0","name":"ip-10-150-1-74.us-west-2.compute.internalcrucible"",","level"pid:"30:4301,"task":"up_listen"}
{,"time":""msg":"2023-09-22T23:14:27.606845572Z","hostname"Wait for all three downstairs to come online":","v":0ip-10-150-1-74.us-west-2.compute.internal",",name"":"pid"crucible:"4301,"level"}:
30{,"time":"{2023-09-22T23:14:27.606878016Z"",""msghostname"msg""::"":"ip-10-150-1-74.us-west-2.compute.internal[0] 1f7159d2-76c3-4968-a4c3-9cc9b7758094 looper connected[0] 66688a47-67b1-4e47-8e76-e0e145532b3a looper connected""",",,""vv""::pid"00,,""name:"4301:name"":"crucible"},crucible""level,"":level"30:
30{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,,""timetime""::""2023-09-22T23:14:27.606925106Z2023-09-22T23:14:27.606920964Z"",,""hostnamehostname""::""ip-10-150-1-74.us-west-2.compute.internalip-10-150-1-74.us-west-2.compute.internal"",,""pidpid""::4301,"4301,"timelooper,"":"looper""0:"":}0"
}{"
2023-09-22T23:14:27.606937217Z"","msg":hostname"":"Upstairs starts",ip-10-150-1-74.us-west-2.compute.internal""v",:"pid"0:4301,"name"}:"
crucible","level":30{"msg":"9ed4d6bc-e430-4200-9ad2-68905b0ff40c active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.606991722Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":,4301"time"}:"
2023-09-22T23:14:27.606982458Z","hostname":"{"ip-10-150-1-74.us-west-2.compute.internal"msg",:""pid":4301}
[0] 9ed4d6bc-e430-4200-9ad2-68905b0ff40c looper connected","v":0,"name":"{crucible","level":30"msg":"Crucible Version: BuildInfo {\n    version: \"0.0.1\",\n    git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n    git_commit_timestamp: \","2023-09-22T22:51:18.000000000Z\"time,"\n:"    git_branch: \"main2023-09-22T23:14:27.607035815Z\"",\n,"hostname    rustc_semver: "\":"1.70.0\",\nip-10-150-1-74.us-west-2.compute.internal    rustc_channel: "\","stable\"pid",:\n4301,    rustc_host_triple: "\"looper":"x86_64-unknown-illumos0\"",\n}
    rustc_commit_sha: \"{90c541806f23a127002de5b4038be731ba1458ca\",\n"msg":    cargo_triple: "\"x86_64-unknown-illumos\",\n[0] Proc runs for 127.0.0.1:36835 in state New"    debug: true,,\n"v":    opt_level: 0,0\n,}""name":"crucible",,""vlevel""::030,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.607075029Z",","hostname":time"":"ip-10-150-1-74.us-west-2.compute.internal2023-09-22T23:14:27.607078469Z"",","pid":hostname"4301:"}
ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.607113692Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"Crucible stats registered with UUID: 5b98377f-d3ca-45bf-893d-9aae2fd5a48e","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.607151179Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"Crucible 5b98377f-d3ca-45bf-893d-9aae2fd5a48e has session id: 00bca160-fbd9-4074-8c7d-5e932e8b713d","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.607184332Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[0] Proc runs for 127.0.0.1:32776 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.607276855Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
Sep 22 23:14:27.607 INFO UUID: bb88743f-4e54-484f-8e97-8fa2d4bde987
Sep 22 23:14:27.607 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.607 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.607 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.607 INFO accepted connection from 127.0.0.1:45775, task: main
Sep 22 23:14:27.607 INFO accepted connection from 127.0.0.1:65350, task: main
Sep 22 23:14:27.607 INFO accepted connection from 127.0.0.1:58018, task: main
Sep 22 23:14:27.607 INFO Using address: 127.0.0.1:54983, task: main
Sep 22 23:14:27.607 INFO up_listen starts, task: up_listen
Sep 22 23:14:27.607 INFO Wait for all three downstairs to come online
The guest has requested activation
Sep 22 23:14:27.607 INFO accepted connection from 127.0.0.1:34921, task: main
Sep 22 23:14:27.607 INFO accepted connection from 127.0.0.1:59139, task: main
Sep 22 23:14:27.607 INFO Flush timeout: 0.5
{"msg":"[0] Proc runs for 127.0.0.1:33768 in state New","v":0,"name":"crucible","level":30Sep 22 23:14:27.607 INFO listening on 127.0.0.1:0, task: main
,"time":"2023-09-22T23:14:27.607654183Z","hostname":"Sep 22 23:14:27.607 INFO accepted connection from 127.0.0.1:52230, task: main
ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
Sep 22 23:14:27.607 INFO accepted connection from 127.0.0.1:45705, task: main
Sep 22 23:14:27.607 INFO a50fc3fb-3de2-4743-9c50-cc80cfba77db active request set
Sep 22 23:14:27.607 INFO accepted connection from 127.0.0.1:56039, task: main
Sep 22 23:14:27.607 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.607 INFO listening on 127.0.0.1:0, task: main
{"msg":"[1] 1f7159d2-76c3-4968-a4c3-9cc9b7758094 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.607847431Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"1"}{
"Sep 22 23:14:27.607 INFO accepted connection from 127.0.0.1:55691, task: main
msg":"[0] connecting to 127.0.0.1:43186","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.607899715Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"0"}
Sep 22 23:14:27.607 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.607 INFO accepted connection from 127.0.0.1:34793, task: main
Sep 22 23:14:27.607 INFO accepted connection from 127.0.0.1:40268, task: main
{{""Sep 22 23:14:27.608 INFO accepted connection from 127.0.0.1:62689, task: main
msg"msg"::""[1] connecting to 127.0.0.1:46381"[1] 64be6ec7-0ac5-4896-883a-00ec34140b84 looper connected,""v":0,",name""v":":crucible"0,,""level"name"::"30crucible","level":30{"msg":"[0] a697641e-15fb-475d-87b4-6f9c9b92b978 looper connected","v",:"0time":,""name":",crucible""time2023-09-22T23:14:27.608073878Z,""level"::""302023-09-22T23:14:27.608077112Z",",hostname"":"hostname":"ip-10-150-1-74.us-west-2.compute.internal",ip-10-150-1-74.us-west-2.compute.internal""pid",":pid4301":4301,"looperSep 22 23:14:27.608 INFO [0] a50fc3fb-3de2-4743-9c50-cc80cfba77db looper connected, looper: 0
":",1,Sep 22 23:14:27.608 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:48117, task: repair
"time"":""}looper"2023-09-22T23:14:27.608105664Z":
","1"hostname":"}
{{ip-10-150-1-74.us-west-2.compute.internal","pid":4301""{,msgmsg""::""""looper"msg":{"0"msg":Sep 22 23:14:27.608 INFO Connection request from 64be6ec7-0ac5-4896-883a-00ec34140b84 with version 4, task: proc
"[1] 9ed4d6bc-e430-4200-9ad2-68905b0ff40c looper connected","v":0,"name":"crucible","level":30Sep 22 23:14:27.608 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:48117, task: repair
,"time":""2023-09-22T23:14:27.608212946Z","}hostname
":"Sep 22 23:14:27.608 INFO Connection request from 9ed4d6bc-e430-4200-9ad2-68905b0ff40c with version 4, task: proc
ip-10-150-1-74.us-west-2.compute.internal{""msg":","pid":4301[0] Proc runs for 127.0.0.1:56524 in state New,"looper":"1"}"
,"v":{{0Sep 22 23:14:27.608 INFO listening, local_addr: 127.0.0.1:48117, task: repair
"msg":","msgname"":":""crucible"[1] Proc runs for 127.0.0.1:54745 in state New","level,"":v":300,"[2] connecting to 127.0.0.1:49774name"[1] Proc runs for 127.0.0.1:52116 in state New"":",,""v"crucible"v",:"Sep 22 23:14:27.608 INFO upstairs UpstairsConnection { upstairs_id: 9ed4d6bc-e430-4200-9ad2-68905b0ff40c, session_id: ad0fa1b4-52c3-4329-aeea-287c43c0dc1e, gen: 1 } connected, version 4, task: proc
:level,"":time300":,"0","2023-09-22T23:14:27.608280986Zname"name"":",crucible"":,,hostname""""level"crucible"time:"":,""ip-10-150-1-74.us-west-2.compute.internal:level"2023-09-22T23:14:27.608301943Z"",",:pid"":hostname430130"}:
"30ip-10-150-1-74.us-west-2.compute.internal{","pid":4301"msg":"}
[1] a697641e-15fb-475d-87b4-6f9c9b92b978 looper connected","v":0{,"name":""cruciblemsg"":,""level":30,,"[2] 1f7159d2-76c3-4968-a4c3-9cc9b7758094 looper connectedtime"""time,""::",v""time:"02023-09-22T23:14:27.608338666Z",:""name"",:2023-09-22T23:14:27.608352358Z""2023-09-22T23:14:27.608333519Z",crucible",hostname""":"",hostname""ip-10-150-1-74.us-west-2.compute.internal"level,"":pid30":Sep 22 23:14:27.608 INFO upstairs UpstairsConnection { upstairs_id: 64be6ec7-0ac5-4896-883a-00ec34140b84, session_id: 58d25f3b-f138-43ff-93b7-02b5e6bdc53e, gen: 1 } connected, version 4, task: proc
hostname:4301,",":""time"":"looper"ip-10-150-1-74.us-west-2.compute.internal"ip-10-150-1-74.us-west-2.compute.internal",","pid":2023-09-22T23:14:27.608384425Z:pid"""1":},
"4301hostname"4301:,""looper"}:
ip-10-150-1-74.us-west-2.compute.internal"","2pidSep 22 23:14:27.608 INFO Connection request from 9ed4d6bc-e430-4200-9ad2-68905b0ff40c with version 4, task: proc
""}{
:"Sep 22 23:14:27.608 INFO accepted connection from 127.0.0.1:37430, task: main
{Sep 22 23:14:27.608 INFO upstairs UpstairsConnection { upstairs_id: 9ed4d6bc-e430-4200-9ad2-68905b0ff40c, session_id: ad0fa1b4-52c3-4329-aeea-287c43c0dc1e, gen: 1 } connected, version 4, task: proc
":"[1] Proc runs for 127.0.0.1:43400 in state New","v":0,"name":"crucible"msg,":""level":[2] 9ed4d6bc-e430-4200-9ad2-68905b0ff40c looper connected"30,"v":0,"name"[1] 66688a47-67b1-4e47-8e76-e0e145532b3a looper connected:""crucible",",level"":v"30:0,"name":"crucible"msg",":"level,"":timeup_listen starts""30:","v":2023-09-22T23:14:27.60848186Z"0,,4301"Sep 22 23:14:27.608 INFO Connection request from 64be6ec7-0ac5-4896-883a-00ec34140b84 with version 4, task: proc
"name",hostname"",,""timetime""::""looper:""2023-09-22T23:14:27.608489865Z2023-09-22T23:14:27.608500482Z""ip-10-150-1-74.us-west-2.compute.internal,,"":":hostnamehostname""::"Sep 22 23:14:27.608 INFO upstairs UpstairsConnection { upstairs_id: 64be6ec7-0ac5-4896-883a-00ec34140b84, session_id: 58d25f3b-f138-43ff-93b7-02b5e6bdc53e, gen: 1 } connected, version 4, task: proc
crucible",""level2"""",}"
:ip-10-150-1-74.us-west-2.compute.internalip-10-150-1-74.us-west-2.compute.internal"",,""30pid"Sep 22 23:14:27.608 INFO Connection request from 9ed4d6bc-e430-4200-9ad2-68905b0ff40c with version 4, task: proc
Sep 22 23:14:27.608 INFO accepted connection from 127.0.0.1:33050, task: main
:pidpid""::430143014301,,}""
,looperlooper":"":1"{"2""time"}}
:Sep 22 23:14:27.608 INFO upstairs UpstairsConnection { upstairs_id: 9ed4d6bc-e430-4200-9ad2-68905b0ff40c, session_id: ad0fa1b4-52c3-4329-aeea-287c43c0dc1e, gen: 1 } connected, version 4, task: proc
"{
"2023-09-22T23:14:27.608552249Z"msg,{"hostname":""":msg"":"ip-10-150-1-74.us-west-2.compute.internal","pid":[2] Proc runs for 127.0.0.1:33349 in state New[1] Proc runs for 127.0.0.1:50563 in state New""4301,,""v"v:":,"00task,,""namename""::"""cruciblecrucible"",:,""levellevel"""::3030up_listen"}
Sep 22 23:14:27.608 INFO Connection request from 64be6ec7-0ac5-4896-883a-00ec34140b84 with version 4, task: proc
{,,"""timetimemsg"""::"":"2023-09-22T23:14:27.608618803Z2023-09-22T23:14:27.608619148Z"",,""Wait for all three downstairs to come online"hostnamehostname""::"","v":0ip-10-150-1-74.us-west-2.compute.internalip-10-150-1-74.us-west-2.compute.internal"",",,Sep 22 23:14:27.608 INFO upstairs UpstairsConnection { upstairs_id: 64be6ec7-0ac5-4896-883a-00ec34140b84, session_id: 58d25f3b-f138-43ff-93b7-02b5e6bdc53e, gen: 1 } connected, version 4, task: proc
"pid""pidname"":Sep 22 23:14:27.608 INFO accepted connection from 127.0.0.1:42022, task: main
:43014301:}
}"
{crucible",""level"msg"::"30[2] 66688a47-67b1-4e47-8e76-e0e145532b3a looper connected","v":0,"name":"crucible","level":30,"time":","time":"2023-09-22T23:14:27.608677768Z","2023-09-22T23:14:27.608685361Z"hostname",":"hostname":"ip-10-150-1-74.us-west-2.compute.internal","ip-10-150-1-74.us-west-2.compute.internalpid"":,4301"pid":}4301
,"looper":"{2"}
{"msg":"{"msgFlush timeout: 0.5""":"msg",:"[1] Proc runs for 127.0.0.1:33784 in state New""v",:[2] Proc runs for 127.0.0.1:39722 in state New""v":,0","0namev""::"0,,crucible""name",:""level":crucible30"",name"":level"":30crucible","level":30,"time":","time2023-09-22T23:14:27.608745803Z"":","hostname"2023-09-22T23:14:27.608751103Z:"","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":ip-10-150-1-74.us-west-2.compute.internal4301","}pid
":,"4301{time":}"
"2023-09-22T23:14:27.608753921Zmsg":"","hostname":"[2] a697641e-15fb-475d-87b4-6f9c9b92b978 looper connected","ip-10-150-1-74.us-west-2.compute.internalv"",:"0pid",":name4301":"crucible"},
"levelSep 22 23:14:27.608 INFO Connection request from 66688a47-67b1-4e47-8e76-e0e145532b3a with version 4, task: proc
{":"30msg":"5b98377f-d3ca-45bf-893d-9aae2fd5a48e active request set","v"":msg"0:,""name":"crucible","[2] 64be6ec7-0ac5-4896-883a-00ec34140b84 looper connected,""level",time""v:""::030,2023-09-22T23:14:27.608808005Z""name",:""hostnamecrucible"":","level":30ip-10-150-1-74.us-west-2.compute.internal",Sep 22 23:14:27.608 INFO upstairs UpstairsConnection { upstairs_id: 66688a47-67b1-4e47-8e76-e0e145532b3a, session_id: ff839372-f996-4bbe-90dd-222e8d070509, gen: 1 } connected, version 4, task: proc
",pid"":time4301",:"",looper":2023-09-22T23:14:27.608830716Z""2""time",}"
hostname"::"{"ip-10-150-1-74.us-west-2.compute.internal""msg",:""pid"2023-09-22T23:14:27.608826655Z":[2] Proc runs for 127.0.0.1:39493 in state New4301",,,""v"looper:"0:,""2"name":}"
crucible"",{"hostname":"level"":msg"30:"ip-10-150-1-74.us-west-2.compute.internal","pid":[2] Proc runs for 127.0.0.1:60582 in state New"4301,"v":}0
,,""timename""::""crucible","2023-09-22T23:14:27.608880642Z"level",:"30{hostname":""ip-10-150-1-74.us-west-2.compute.internalmsg"":","pid":4301}
,"[0] 5b98377f-d3ca-45bf-893d-9aae2fd5a48e looper connected"time":,""v":02023-09-22T23:14:27.608895219Z,""name":","crucible"hostname,"":level"":30ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{{""msgmsg""::""[0] 9ed4d6bc-e430-4200-9ad2-68905b0ff40c (ad0fa1b4-52c3-4329-aeea-287c43c0dc1e) New New New ds_transition to WaitActive[0] 64be6ec7-0ac5-4896-883a-00ec34140b84 (58d25f3b-f138-43ff-93b7-02b5e6bdc53e) New New New ds_transition to WaitActive"",,""vv""::0,"0,"time"name,"":name"":crucible"":"Sep 22 23:14:27.608 INFO Connection request from 66688a47-67b1-4e47-8e76-e0e145532b3a with version 4, task: proc
crucible,""2023-09-22T23:14:27.60892007Z"level,"":level30":,30"hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"0"}
,,""timetime""::""2023-09-22T23:14:27.608957939Z2023-09-22T23:14:27.608956552Z""{,,""hostnamehostname""::"""msgSep 22 23:14:27.608 INFO upstairs UpstairsConnection { upstairs_id: 66688a47-67b1-4e47-8e76-e0e145532b3a, session_id: ff839372-f996-4bbe-90dd-222e8d070509, gen: 1 } connected, version 4, task: proc
":"ip-10-150-1-74.us-west-2.compute.internalip-10-150-1-74.us-west-2.compute.internal"",,""pidpid"":[0] Proc runs for 127.0.0.1:43186 in state New"4301:4301},}

{"v":{"0msg"":"msg",:[0] Transition from New to WaitActive""","name":v[0] Transition from New to WaitActive"":,0",v"":name0","":"cruciblenamecrucible"":,"""levelcrucible"":,30","levellevel":"30Sep 22 23:14:27.608 INFO Connection request from a697641e-15fb-475d-87b4-6f9c9b92b978 with version 4, task: proc
:,30"time":"2023-09-22T23:14:27.609018791Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal",,""pid"time:":4301"}
2023-09-22T23:14:27.609028642Z",,{""hostname"time":""msg"::ip-10-150-1-74.us-west-2.compute.internal""","pid":43012023-09-22T23:14:27.609022954Z"}
[0] client is_active_req TRUE, promote! session ad0fa1b4-52c3-4329-aeea-287c43c0dc1e",{,"""vmsg"":Sep 22 23:14:27.609 INFO upstairs UpstairsConnection { upstairs_id: a697641e-15fb-475d-87b4-6f9c9b92b978, session_id: 01d84cdc-ab54-426d-be83-dc686be9c9e6, gen: 1 } connected, version 4, task: proc
0,:""hostname"name":":[0] client is_active_req TRUE, promote! session 58d25f3b-f138-43ff-93b7-02b5e6bdc53e""crucible,""ip-10-150-1-74.us-west-2.compute.internalv,"":level0"Sep 22 23:14:27.609 INFO Connection request from 66688a47-67b1-4e47-8e76-e0e145532b3a with version 4, task: proc
,"",:"30name":"pid":crucible"4301,"level":}30
,"time":"2023-09-22T23:14:27.609084767Z",{","hostname"time:"":""2023-09-22T23:14:27.609092144Zip-10-150-1-74.us-west-2.compute.internal""msg",,""pid"hostname::""4301:"Sep 22 23:14:27.609 INFO upstairs UpstairsConnection { upstairs_id: 66688a47-67b1-4e47-8e76-e0e145532b3a, session_id: ff839372-f996-4bbe-90dd-222e8d070509, gen: 1 } connected, version 4, task: proc
ip-10-150-1-74.us-west-2.compute.internal}[1] 5b98377f-d3ca-45bf-893d-9aae2fd5a48e looper connected","
"v,{"pid":4301"}msg
":""{:0",msg":""[1] 9ed4d6bc-e430-4200-9ad2-68905b0ff40c (ad0fa1b4-52c3-4329-aeea-287c43c0dc1e) WaitActive New New ds_transition to WaitActive"name":,""v":crucible"0[1] 64be6ec7-0ac5-4896-883a-00ec34140b84 (58d25f3b-f138-43ff-93b7-02b5e6bdc53e) WaitActive New New ds_transition to WaitActive",,,""v"name:"0:,"""namecrucible"":,""levellevelcrucible"":,30"":level":3030,"time":"Sep 22 23:14:27.609 INFO Connection request from a697641e-15fb-475d-87b4-6f9c9b92b978 with version 4, task: proc
,2023-09-22T23:14:27.609150402Z""Sep 22 23:14:27.609 INFO [0] Proc runs for 127.0.0.1:42970 in state New
time",":Sep 22 23:14:27.608 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:48117, task: repair
hostname":""2023-09-22T23:14:27.609153217Z"ip-10-150-1-74.us-west-2.compute.internal",",pid"":hostname4301":"}
ip-10-150-1-74.us-west-2.compute.internal"Sep 22 23:14:27.609 INFO upstairs UpstairsConnection { upstairs_id: a697641e-15fb-475d-87b4-6f9c9b92b978, session_id: 01d84cdc-ab54-426d-be83-dc686be9c9e6, gen: 1 } connected, version 4, task: proc
,{"pid":4301",msg":""looper":"1"[1] Transition from New to WaitActive"}
,"v":0,"name":"crucible"{,"level":30"msg":"[1] Proc runs for 127.0.0.1:46381 in state New"Sep 22 23:14:27.609 INFO Using repair address: 127.0.0.1:48117, task: main
,"v":0,",name":""crucible"time",":level"":302023-09-22T23:14:27.609226036Z","hostname":"Sep 22 23:14:27.609 INFO No SSL acceptor configured, task: main
ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{,""timemsg"":":"2023-09-22T23:14:27.609246809Z","[1] client is_active_req TRUE, promote! session ad0fa1b4-52c3-4329-aeea-287c43c0dc1ehostname":""Sep 22 23:14:27.609 INFO [1] a50fc3fb-3de2-4743-9c50-cc80cfba77db looper connected, looper: 1
ip-10-150-1-74.us-west-2.compute.internal",Sep 22 23:14:27.609 INFO Connection request from 5b98377f-d3ca-45bf-893d-9aae2fd5a48e with version 4, task: proc
Sep 22 23:14:27.609 INFO UpstairsConnection { upstairs_id: 9ed4d6bc-e430-4200-9ad2-68905b0ff40c, session_id: ad0fa1b4-52c3-4329-aeea-287c43c0dc1e, gen: 1 } is now active (read-write)
,""pidv""::04301,"name":"}
crucible","level":30{"Sep 22 23:14:27.609 INFO Connection request from a697641e-15fb-475d-87b4-6f9c9b92b978 with version 4, task: proc
msg"Sep 22 23:14:27.609 INFO upstairs UpstairsConnection { upstairs_id: 5b98377f-d3ca-45bf-893d-9aae2fd5a48e, session_id: d3623aa7-ee8c-40f8-a982-d0eadbcd4a76, gen: 1 } connected, version 4, task: proc
:","time":"[2] 5b98377f-d3ca-45bf-893d-9aae2fd5a48e looper connected"2023-09-22T23:14:27.609301836Z",",v"":hostname0":","name":"ip-10-150-1-74.us-west-2.compute.internal"crucible,"",pid"":level4301"Sep 22 23:14:27.609 INFO [1] Proc runs for 127.0.0.1:62060 in state New
:30}Sep 22 23:14:27.609 INFO upstairs UpstairsConnection { upstairs_id: a697641e-15fb-475d-87b4-6f9c9b92b978, session_id: 01d84cdc-ab54-426d-be83-dc686be9c9e6, gen: 1 } connected, version 4, task: proc

{",msg":""time":"2023-09-22T23:14:27.609342925Z","hostname":"[2] 9ed4d6bc-e430-4200-9ad2-68905b0ff40c (ad0fa1b4-52c3-4329-aeea-287c43c0dc1e) WaitActive WaitActive New ds_transition to WaitActive"ip-10-150-1-74.us-west-2.compute.internal",","vpid""::04301,"name":,""looper":crucible"","2"level":}30
{"msg":"[2] Proc runs for 127.0.0.1:49774 in state New",",time":""v":02023-09-22T23:14:27.609378985Z",","name":hostname":""crucible","ip-10-150-1-74.us-west-2.compute.internal"level,""Sep 22 23:14:27.609 INFO UpstairsConnection { upstairs_id: 9ed4d6bc-e430-4200-9ad2-68905b0ff40c, session_id: ad0fa1b4-52c3-4329-aeea-287c43c0dc1e, gen: 1 } is now active (read-write)
Sep 22 23:14:27.609 INFO Connection request from 5b98377f-d3ca-45bf-893d-9aae2fd5a48e with version 4, task: proc
:pid30":4301Sep 22 23:14:27.609 INFO [2] a50fc3fb-3de2-4743-9c50-cc80cfba77db looper connected, looper: 2
}
{,"time":"{2023-09-22T23:14:27.609415347Z"""msg,msg""::""Sep 22 23:14:27.609 INFO upstairs UpstairsConnection { upstairs_id: 5b98377f-d3ca-45bf-893d-9aae2fd5a48e, session_id: d3623aa7-ee8c-40f8-a982-d0eadbcd4a76, gen: 1 } connected, version 4, task: proc
"[0] 66688a47-67b1-4e47-8e76-e0e145532b3a (ff839372-f996-4bbe-90dd-222e8d070509) New New New ds_transition to WaitActive[2] Transition from New to WaitActive""hostname":,,""v":v0",:"0name,"":"name"crucible:"",""crucible"level",:"level30":ip-10-150-1-74.us-west-2.compute.internal"30,"pid":4301}
,,""timetime""::""2023-09-22T23:14:27.609457223Z2023-09-22T23:14:27.609459069Z""Sep 22 23:14:27.609 INFO [2] Proc runs for 127.0.0.1:42513 in state New
,,""hostnamehostname""::""ip-10-150-1-74.us-west-2.compute.internalip-10-150-1-74.us-west-2.compute.internal"",,""pidpid""::43014301}
}
{"{msg":""msg":"[0] Transition from New to WaitActive"[2] client is_active_req TRUE, promote! session ad0fa1b4-52c3-4329-aeea-287c43c0dc1e",",v"":v"0:0,","name":name"":"crucible"crucible",","level"level:":3030{"msg":",,""[0] a697641e-15fb-475d-87b4-6f9c9b92b978 (01d84cdc-ab54-426d-be83-dc686be9c9e6) New New New ds_transition to WaitActivetime"Sep 22 23:14:27.609 INFO UpstairsConnection { upstairs_id: 9ed4d6bc-e430-4200-9ad2-68905b0ff40c, session_id: ad0fa1b4-52c3-4329-aeea-287c43c0dc1e, gen: 1 } is now active (read-write)
Sep 22 23:14:27.609 INFO Connection request from 5b98377f-d3ca-45bf-893d-9aae2fd5a48e with version 4, task: proc
,"time:"":""v":2023-09-22T23:14:27.60950878Z2023-09-22T23:14:27.609508835Z""0,,"","hostnamehostname""::""name":"crucible"ip-10-150-1-74.us-west-2.compute.internalip-10-150-1-74.us-west-2.compute.internal"",,""pidpid":"4301:,4301}}

Sep 22 23:14:27.609 INFO upstairs UpstairsConnection { upstairs_id: 5b98377f-d3ca-45bf-893d-9aae2fd5a48e, session_id: d3623aa7-ee8c-40f8-a982-d0eadbcd4a76, gen: 1 } connected, version 4, task: proc
"level":{30"msg":"[0] client is_active_req TRUE, promote! session ff839372-f996-4bbe-90dd-222e8d070509","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.609566405Z","hostname",:""time":"ip-10-150-1-74.us-west-2.compute.internal","2023-09-22T23:14:27.609578209Zpid"":,4301"hostname":"}
ip-10-150-1-74.us-west-2.compute.internal"Sep 22 23:14:27.609 INFO UpstairsConnection { upstairs_id: 64be6ec7-0ac5-4896-883a-00ec34140b84, session_id: 58d25f3b-f138-43ff-93b7-02b5e6bdc53e, gen: 1 } is now active (read-write)
{Sep 22 23:14:27.609 INFO UpstairsConnection { upstairs_id: 66688a47-67b1-4e47-8e76-e0e145532b3a, session_id: ff839372-f996-4bbe-90dd-222e8d070509, gen: 1 } is now active (read-write)
"msg":"[0] Transition from New to WaitActive","v":0,",name"":"time"crucible":","level":2023-09-22T23:14:27.609153139Z"30,"hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{,""timemsg":"":"[1] Transition from New to WaitActive"2023-09-22T23:14:27.609635126Z","v",":hostname0",":"name":"crucible"ip-10-150-1-74.us-west-2.compute.internal,""level",":pid"30:4301}
{"msg":","time":"[0] client is_active_req TRUE, promote! session 01d84cdc-ab54-426d-be83-dc686be9c9e6"2023-09-22T23:14:27.60967046Z",",v":"0hostname,"":"name":"crucible",ip-10-150-1-74.us-west-2.compute.internal"","levelpid""::430130}
{"msg":"[1] client is_active_req TRUE, promote! session 58d25f3b-f138-43ff-93b7-02b5e6bdc53e,"",time":""v":02023-09-22T23:14:27.609702445Z,""Sep 22 23:14:27.609 INFO UpstairsConnection { upstairs_id: 64be6ec7-0ac5-4896-883a-00ec34140b84, session_id: 58d25f3b-f138-43ff-93b7-02b5e6bdc53e, gen: 1 } is now active (read-write)
,"name":"hostname":crucible"","level":30ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"Sep 22 23:14:27.609 INFO UpstairsConnection { upstairs_id: 66688a47-67b1-4e47-8e76-e0e145532b3a, session_id: ff839372-f996-4bbe-90dd-222e8d070509, gen: 1 } is now active (read-write)
[1] a697641e-15fb-475d-87b4-6f9c9b92b978 (01d84cdc-ab54-426d-be83-dc686be9c9e6) WaitActive New New ds_transition to WaitActive,""time":,""v":02023-09-22T23:14:27.609735591Z","name,"":hostname"":"crucible","levelip-10-150-1-74.us-west-2.compute.internal"",":pid"30:4301}
{Sep 22 23:14:27.609 INFO UpstairsConnection { upstairs_id: a697641e-15fb-475d-87b4-6f9c9b92b978, session_id: 01d84cdc-ab54-426d-be83-dc686be9c9e6, gen: 1 } is now active (read-write)
,""time":"msg":"2023-09-22T23:14:27.609773659Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal"[2] 64be6ec7-0ac5-4896-883a-00ec34140b84 (58d25f3b-f138-43ff-93b7-02b5e6bdc53e) WaitActive WaitActive New ds_transition to WaitActive",",pid"":v"4301:0,"}name":"
crucible","level":30{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time"Sep 22 23:14:27.609 INFO UpstairsConnection { upstairs_id: 64be6ec7-0ac5-4896-883a-00ec34140b84, session_id: 58d25f3b-f138-43ff-93b7-02b5e6bdc53e, gen: 1 } is now active (read-write)
,"time":"2023-09-22T23:14:27.609830279Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal",,""pid":pid"4301:4301}
}
Sep 22 23:14:27.609 INFO UpstairsConnection { upstairs_id: 66688a47-67b1-4e47-8e76-e0e145532b3a, session_id: ff839372-f996-4bbe-90dd-222e8d070509, gen: 1 } is now active (read-write)
{{"msg":""msg":"[1] client is_active_req TRUE, promote! session 01d84cdc-ab54-426d-be83-dc686be9c9e6","v":0,"name":"[1] 66688a47-67b1-4e47-8e76-e0e145532b3a (ff839372-f996-4bbe-90dd-222e8d070509) WaitActive New New ds_transition to WaitActive"crucible",,""level"v"::030,"name":"crucible","level":30Sep 22 23:14:27.609 INFO UpstairsConnection { upstairs_id: a697641e-15fb-475d-87b4-6f9c9b92b978, session_id: 01d84cdc-ab54-426d-be83-dc686be9c9e6, gen: 1 } is now active (read-write)
,","time":"time":"2023-09-22T23:14:27.60989648Z"2023-09-22T23:14:27.609900443Z",","hostname"hostname":":"ip-10-150-1-74.us-west-2.compute.internal"ip-10-150-1-74.us-west-2.compute.internal,"","pid"pid"::43014301}
}
{"{msg":"[1] Transition from New to WaitActive"","msg":"v":0,"name":"crucible","level":30[2] a697641e-15fb-475d-87b4-6f9c9b92b978 (01d84cdc-ab54-426d-be83-dc686be9c9e6) WaitActive WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.60996031Z","hostname":"Sep 22 23:14:27.609 INFO UpstairsConnection { upstairs_id: 5b98377f-d3ca-45bf-893d-9aae2fd5a48e, session_id: d3623aa7-ee8c-40f8-a982-d0eadbcd4a76, gen: 1 } is now active (read-write)
ip-10-150-1-74.us-west-2.compute.internal,""time":","pid":2023-09-22T23:14:27.609967071Z"4301,"}hostname":
Sep 22 23:14:27.609 INFO Connection request from a50fc3fb-3de2-4743-9c50-cc80cfba77db with version 4, task: proc
{""msg":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301[1] client is_active_req TRUE, promote! session ff839372-f996-4bbe-90dd-222e8d070509"}
Sep 22 23:14:27.609 INFO UpstairsConnection { upstairs_id: a697641e-15fb-475d-87b4-6f9c9b92b978, session_id: 01d84cdc-ab54-426d-be83-dc686be9c9e6, gen: 1 } is now active (read-write)
{,""v"msg:":Sep 22 23:14:27.610 INFO upstairs UpstairsConnection { upstairs_id: a50fc3fb-3de2-4743-9c50-cc80cfba77db, session_id: 2b698136-bbd2-4fcb-ab7c-5f19b2267eb8, gen: 1 } connected, version 4, task: proc
0,""name":"crucible",[2] Transition from New to WaitActive""level",:"30v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.610042999Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal",,""timepid""::"43012023-09-22T23:14:27.610048332Z"}
,"hostname":"ip-10-150-1-74.us-west-2.compute.internal"{,"pid":4301"}msg"
:"{[2] 66688a47-67b1-4e47-8e76-e0e145532b3a (ff839372-f996-4bbe-90dd-222e8d070509) WaitActive WaitActive New ds_transition to WaitActive"",msg""Sep 22 23:14:27.610 INFO UpstairsConnection { upstairs_id: 5b98377f-d3ca-45bf-893d-9aae2fd5a48e, session_id: d3623aa7-ee8c-40f8-a982-d0eadbcd4a76, gen: 1 } is now active (read-write)
v"::0","name":"crucible","level":30[2] client is_active_req TRUE, promote! session 01d84cdc-ab54-426d-be83-dc686be9c9e6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.610111392Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,Sep 22 23:14:27.610 INFO Connection request from a50fc3fb-3de2-4743-9c50-cc80cfba77db with version 4, task: proc
}
"time":"2023-09-22T23:14:27.610118082Z"{,"hostname":""msg":ip-10-150-1-74.us-west-2.compute.internal"","pid":[2] Transition from New to WaitActive4301","}v"
:0Sep 22 23:14:27.610 INFO upstairs UpstairsConnection { upstairs_id: a50fc3fb-3de2-4743-9c50-cc80cfba77db, session_id: 2b698136-bbd2-4fcb-ab7c-5f19b2267eb8, gen: 1 } connected, version 4, task: proc
,"name":"crucible","level":30,"time":"{2023-09-22T23:14:27.610169361Z","hostname":""ip-10-150-1-74.us-west-2.compute.internal"msg":,""pid":4301}
[0] 5b98377f-d3ca-45bf-893d-9aae2fd5a48e (d3623aa7-ee8c-40f8-a982-d0eadbcd4a76) New New New ds_transition to WaitActive","v":0,{"name":"crucible",""level"msg:"30:"[2] client is_active_req TRUE, promote! session ff839372-f996-4bbe-90dd-222e8d070509","v":0,"name":"Sep 22 23:14:27.610 INFO UpstairsConnection { upstairs_id: 5b98377f-d3ca-45bf-893d-9aae2fd5a48e, session_id: d3623aa7-ee8c-40f8-a982-d0eadbcd4a76, gen: 1 } is now active (read-write)
,crucible""time",":"level":302023-09-22T23:14:27.610207196Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{,"time":""msg":"2023-09-22T23:14:27.610234067Z","[0] Transition from New to WaitActive"hostname":","v":0ip-10-150-1-74.us-west-2.compute.internal",",Sep 22 23:14:27.610 INFO Connection request from a50fc3fb-3de2-4743-9c50-cc80cfba77db with version 4, task: proc
"pid"name":":crucible4301","level}"
:30Sep 22 23:14:27.610 INFO upstairs UpstairsConnection { upstairs_id: a50fc3fb-3de2-4743-9c50-cc80cfba77db, session_id: 2b698136-bbd2-4fcb-ab7c-5f19b2267eb8, gen: 1 } connected, version 4, task: proc
,"time":"2023-09-22T23:14:27.610278239Z":,""hostname":"2023-09-22T23:14:27.609817026Z"ip-10-150-1-74.us-west-2.compute.internal",","hostnamepid""::"4301}ip-10-150-1-74.us-west-2.compute.internal
","pid":4301{}
"msg":"{[0] client is_active_req TRUE, promote! session d3623aa7-ee8c-40f8-a982-d0eadbcd4a76","v"":msg0":,""name":"crucible"[2] Transition from New to WaitActive,""level":,"30v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.610338693Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal",,""timepid":"":43012023-09-22T23:14:27.610343754Z"}
,"hostname":"{ip-10-150-1-74.us-west-2.compute.internal","pid":4301"msg":}"
[1] 5b98377f-d3ca-45bf-893d-9aae2fd5a48e (d3623aa7-ee8c-40f8-a982-d0eadbcd4a76) WaitActive New New ds_transition to WaitActive","v":0{,"name":"crucible",""levelmsg""::30"[2] client is_active_req TRUE, promote! session 58d25f3b-f138-43ff-93b7-02b5e6bdc53e","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.610392879Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":,"4301time":"}
2023-09-22T23:14:27.610402812Z","hostname":"Sep 22 23:14:27.610 INFO [0] a50fc3fb-3de2-4743-9c50-cc80cfba77db (2b698136-bbd2-4fcb-ab7c-5f19b2267eb8) New New New ds_transition to WaitActive
ip-10-150-1-74.us-west-2.compute.internal"{,"pid":4301"}msg"
:"[1] Transition from New to WaitActive"{,"v":0,""msg"Sep 22 23:14:27.610 INFO [0] Transition from New to WaitActive
:"name":"crucible","level":30[0] downstairs client at 127.0.0.1:45897 has UUID b63c1426-990b-43e7-baa9-a926ab8df052","v":0,"name":"crucible","level":30,Sep 22 23:14:27.610 INFO [0] client is_active_req TRUE, promote! session 2b698136-bbd2-4fcb-ab7c-5f19b2267eb8
,""time":"time":"2023-09-22T23:14:27.610461629Z"2023-09-22T23:14:27.610455543Z","hostname,"":"hostname":"ip-10-150-1-74.us-west-2.compute.internal",ip-10-150-1-74.us-west-2.compute.internal"","pidpid""::43014301}}

{"{msg":""msg":"[1] client is_active_req TRUE, promote! session d3623aa7-ee8c-40f8-a982-d0eadbcd4a76","v":0[0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: b63c1426-990b-43e7-baa9-a926ab8df052, encrypted: true, database_read_version: 1, database_write_version: 1 }","name",":"v"crucible":,"0level",:"30name":"crucible","level":30,"time":"2023-09-22T23:14:27.610525091Z",,""hostname"time":":"2023-09-22T23:14:27.61052837Z"ip-10-150-1-74.us-west-2.compute.internal",,""hostnamepid""::"4301}ip-10-150-1-74.us-west-2.compute.internal
","pid":4301{}
"msg":"{"msg[2] 5b98377f-d3ca-45bf-893d-9aae2fd5a48e (d3623aa7-ee8c-40f8-a982-d0eadbcd4a76) WaitActive WaitActive New ds_transition to WaitActive"":","v":Sep 22 23:14:27.610 INFO [1] a50fc3fb-3de2-4743-9c50-cc80cfba77db (2b698136-bbd2-4fcb-ab7c-5f19b2267eb8) WaitActive New New ds_transition to WaitActive
64be6ec7-0ac5-4896-883a-00ec34140b84 WaitActive WaitActive WaitActive"0,","vname""::"0crucible",",name"":"level":crucible"30,Sep 22 23:14:27.610 INFO [1] Transition from New to WaitActive
"level":30,",time"":"time":"Sep 22 23:14:27.610 INFO [1] client is_active_req TRUE, promote! session 2b698136-bbd2-4fcb-ab7c-5f19b2267eb8
2023-09-22T23:14:27.61060414Z"2023-09-22T23:14:27.610599829Z","hostname,"":"hostname":"ip-10-150-1-74.us-west-2.compute.internal",ip-10-150-1-74.us-west-2.compute.internal"",pid""pid:"4301:4301}
}
{"msg"{:""msg":"[1] downstairs client at 127.0.0.1:43400 has UUID 1a8f9cb4-4a64-4013-8234-292244fb0112Sep 22 23:14:27.610 INFO Current flush_numbers [0..12]: [0, 0]
[2] Transition from New to WaitActive"",",v"":v"0:,"0name",":name":""crucible"crucible",",level"":level"30:30,"time":,""time":2023-09-22T23:14:27.610683717Z"","2023-09-22T23:14:27.610684907Z"hostname":","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pidip-10-150-1-74.us-west-2.compute.internal"",:"4301pid":4301}
}
Sep 22 23:14:27.610 INFO [2] a50fc3fb-3de2-4743-9c50-cc80cfba77db (2b698136-bbd2-4fcb-ab7c-5f19b2267eb8) WaitActive WaitActive New ds_transition to WaitActive
{{"msg":""msg":"[2] client is_active_req TRUE, promote! session d3623aa7-ee8c-40f8-a982-d0eadbcd4a76","v":0,"name":"crucible"[1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 1a8f9cb4-4a64-4013-8234-292244fb0112, encrypted: true, database_read_version: 1, database_write_version: 1 },""level":Sep 22 23:14:27.610 INFO [2] Transition from New to WaitActive
,"30v":0,"name":"crucible","level"Sep 22 23:14:27.610 INFO Current flush_numbers [0..12]: [0, 0]
:30,Sep 22 23:14:27.610 INFO [2] client is_active_req TRUE, promote! session 2b698136-bbd2-4fcb-ab7c-5f19b2267eb8
"time":"2023-09-22T23:14:27.610751993Z",","time":"hostname":"2023-09-22T23:14:27.61076398Z","ip-10-150-1-74.us-west-2.compute.internalhostname"":","pid":4301Sep 22 23:14:27.610 INFO Current flush_numbers [0..12]: [0, 0]
}ip-10-150-1-74.us-west-2.compute.internal"
,"pid":4301}{
"msg":"{[0] downstairs client at 127.0.0.1:43186 has UUID 6b339ff0-fd62-4d3d-96a2-00a3cd97fb5c",""v"msg"::"0,"name":"crucible","64be6ec7-0ac5-4896-883a-00ec34140b84 WaitActive WaitActive WaitActive"level":,"30v":0,"name":"crucible","level":30,"time":"Sep 22 23:14:27.610 INFO Current flush_numbers [0..12]: [0, 0]
2023-09-22T23:14:27.610816648Z",,""time":"hostname":"2023-09-22T23:14:27.61082176Z"ip-10-150-1-74.us-west-2.compute.internal",","hostname"pid":":4301ip-10-150-1-74.us-west-2.compute.internal"},
"pid":4301}
{"{msg":""msg":"[2] downstairs client at 127.0.0.1:60582 has UUID 7f911e5b-f4b1-49f1-9a5d-fe84ec8226ed","v":0[0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 6b339ff0-fd62-4d3d-96a2-00a3cd97fb5c, encrypted: true, database_read_version: 1, database_write_version: 1 }","name,"":"v":crucible"0,","levelname""::"30crucible","level":30,"time":"2023-09-22T23:14:27.610892963Z",","time"hostname"::""2023-09-22T23:14:27.610895692Z"ip-10-150-1-74.us-west-2.compute.internalSep 22 23:14:27.610 INFO UpstairsConnection { upstairs_id: a50fc3fb-3de2-4743-9c50-cc80cfba77db, session_id: 2b698136-bbd2-4fcb-ab7c-5f19b2267eb8, gen: 1 } is now active (read-write)
",",pid"":hostname4301":"}
ip-10-150-1-74.us-west-2.compute.internal","pid":4301{}
"msg":"{"msg":"5b98377f-d3ca-45bf-893d-9aae2fd5a48e WaitActive WaitActive WaitActive"[2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 7f911e5b-f4b1-49f1-9a5d-fe84ec8226ed, encrypted: true, database_read_version: 1, database_write_version: 1 }","v,""v"::00,"Sep 22 23:14:27.610 INFO Downstairs has completed Negotiation, task: proc
name":,""crucible"name",:""crucible"level",":Sep 22 23:14:27.610 INFO Downstairs has completed Negotiation, task: proc
30level":30Sep 22 23:14:27.610 INFO Downstairs has completed Negotiation, task: proc
,"time":"2023-09-22T23:14:27.610979154Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal",,""time"pid":":43012023-09-22T23:14:27.610982862Z"}
,"hostname":"{ip-10-150-1-74.us-west-2.compute.internal","pid":4301"msg":}"
Sep 22 23:14:27.610 INFO Downstairs has completed Negotiation, task: proc
{[1] downstairs client at 127.0.0.1:46381 has UUID 45f3b954-a0cc-44a1-92c8-91ac1da9b5f8"{,""v":0msg",msg""::""name":""crucible","level":[0] downstairs client at 127.0.0.1:56524 has UUID 35ccb45b-0c22-4a14-bf83-50fbfaffa7a630"64be6ec7-0ac5-4896-883a-00ec34140b84 WaitActive WaitActive WaitActive",,""v"v"::00,,"{"name"name":":"crucible""crucible",msg,"""level":,""time"level":"::[0] downstairs client at 127.0.0.1:32776 has UUID d2285a39-bf6a-4487-9493-8622023b1e4b"30Sep 22 23:14:27.611 INFO UpstairsConnection { upstairs_id: a50fc3fb-3de2-4743-9c50-cc80cfba77db, session_id: 2b698136-bbd2-4fcb-ab7c-5f19b2267eb8, gen: 1 } is now active (read-write)
30,","time":"2023-09-22T23:14:27.611073029Z"v,""hostname":":0ip-10-150-1-74.us-west-2.compute.internal",,""pid"name:"2023-09-22T23:14:27.611043655Z4301:""}crucible
",,"",timehostname""":":level"2023-09-22T23:14:27.611084826Z"":ip-10-150-1-74.us-west-2.compute.internal"30,","hostnamepid""::"4301}
ip-10-150-1-74.us-west-2.compute.internal","pid":4301{}
"msg,"":"time":"{2023-09-22T23:14:27.611119562Z",""hostnamemsg"Sep 22 23:14:27.611 INFO Current flush_numbers [0..12]: [0, 0]
:"[1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 45f3b954-a0cc-44a1-92c8-91ac1da9b5f8, encrypted: true, database_read_version: 1, database_write_version: 1 }"":"ip-10-150-1-74.us-west-2.compute.internal,""v",:"0pid",":name"4301:"crucible"},[0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 35ccb45b-0c22-4a14-bf83-50fbfaffa7a6, encrypted: true, database_read_version: 1, database_write_version: 1 }"
"{,"level"v":":msg300"{,:""name":"crucible",""level"msg":[0] downstairs client at 127.0.0.1:36835 has UUID 9008985f-4d59-43db-b559-a48d3978dfda30:"","v":0,"name":"crucible","level":30,"time[0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: d2285a39-bf6a-4487-9493-8622023b1e4b, encrypted: true, database_read_version: 1, database_write_version: 1 }"":","v"2023-09-22T23:14:27.611177307Z":,0",,"time"Sep 22 23:14:27.611 INFO UpstairsConnection { upstairs_id: a50fc3fb-3de2-4743-9c50-cc80cfba77db, session_id: 2b698136-bbd2-4fcb-ab7c-5f19b2267eb8, gen: 1 } is now active (read-write)
",:""timename"2023-09-22T23:14:27.611184864Z":"":"hostname"crucible:",2023-09-22T23:14:27.61119239Z""",hostname"",ip-10-150-1-74.us-west-2.compute.internal""hostname:"level",:"30:""pid":4301ip-10-150-1-74.us-west-2.compute.internalip-10-150-1-74.us-west-2.compute.internal""},,""pidpid""::43014301
}}

{{,""{msg":time""msg"":msg"a697641e-15fb-475d-87b4-6f9c9b92b978 WaitActive WaitActive WaitActive"":",""2023-09-22T23:14:27.611234968Z"v"::,0[0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 9008985f-4d59-43db-b559-a48d3978dfda, encrypted: true, database_read_version: 1, database_write_version: 1 },""""name,"":v"":crucible0",,""namelevel""::"30hostname"5b98377f-d3ca-45bf-893d-9aae2fd5a48e WaitActive WaitActive WaitActive"crucible":,,""v"level"::030,,"""name"timeip-10-150-1-74.us-west-2.compute.internal":,""crucible"":pid"",:"4301,time2023-09-22T23:14:27.611282365Z"":",}"2023-09-22T23:14:27.611292168Z""
level",hostname""hostname"::""Sep 22 23:14:27.611 INFO Current flush_numbers [0..12]: [0, 0]
ip-10-150-1-74.us-west-2.compute.internal{"ip-10-150-1-74.us-west-2.compute.internal,"",:pid30"":pid"Sep 22 23:14:27.611 INFO Downstairs has completed Negotiation, task: proc
msg""4301::"4301}
}
{66688a47-67b1-4e47-8e76-e0e145532b3a WaitActive WaitActive WaitActive"{","msgmsg""::"",""v":time"0[1] downstairs client at 127.0.0.1:33784 has UUID fe394f4d-b22e-45c5-ad25-c835a8779c919ed4d6bc-e430-4200-9ad2-68905b0ff40c WaitActive WaitActive WaitActive:""",,,""v"v:"0:,0",name"":name"""crucible:"",name":"2023-09-22T23:14:27.611331929Z"cruciblelevel"crucible",:"30level""":,30","level"hostname"::"30ip-10-150-1-74.us-west-2.compute.internal",",time"",:""pid"time2023-09-22T23:14:27.611377076Z""::,4301""hostname":"2023-09-22T23:14:27.61138063Z"},
ip-10-150-1-74.us-west-2.compute.internal"",,hostname""pid"":time":"":{2023-09-22T23:14:27.611385182Z"4301ip-10-150-1-74.us-west-2.compute.internal",},
""pid""{:hostname":msg"":""4301msg"}:
"[2] downstairs client at 127.0.0.1:49774 has UUID ed248c78-99b2-4b20-8580-19eb24b6c3f5"ip-10-150-1-74.us-west-2.compute.internal{",""v":msg",":"0pid",:"4301name"[1] downstairs client at 127.0.0.1:52116 has UUID 54deed7d-517b-47d9-a428-2bbdc0729fba[1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: fe394f4d-b22e-45c5-ad25-c835a8779c91, encrypted: true, database_read_version: 1, database_write_version: 1 }"}",,""v":v"0:,0",name""
name:""crucible","level":30{:"crucible",""level"msg":":30[1] downstairs client at 127.0.0.1:50563 has UUID 594f2637-eae4-459c-bc2e-92b501adec4b",,""v":0,"name":"time,"crucible"":,time""":2023-09-22T23:14:27.611462933Z""level":,2023-09-22T23:14:27.611469371Z""30,hostname"":"hostname":"ip-10-150-1-74.us-west-2.compute.internal"ip-10-150-1-74.us-west-2.compute.internal,"","pid"pid:":43014301}}

{{"msg":""msg":Sep 22 23:14:27.611 INFO Downstairs has completed Negotiation, task: proc
","time"[1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 54deed7d-517b-47d9-a428-2bbdc0729fba, encrypted: true, database_read_version: 1, database_write_version: 1 }a697641e-15fb-475d-87b4-6f9c9b92b978 WaitActive WaitActive WaitActive"":,,""v":v0",:"0name"":,""namecrucible"":,""2023-09-22T23:14:27.61148799Z"levelcrucible"":,,"30level"":30hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
,,""timetime""::""2023-09-22T23:14:27.611527198Z{2023-09-22T23:14:27.611525227Z"",,""hostnamehostname""::"""msg":"ip-10-150-1-74.us-west-2.compute.internal",ip-10-150-1-74.us-west-2.compute.internal""pid,"":4301pid":}4301
}
{[1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 594f2637-eae4-459c-bc2e-92b501adec4b, encrypted: true, database_read_version: 1, database_write_version: 1 }"{msg""":"msg":,""v":0,"[2] downstairs client at 127.0.0.1:39493 has UUID 4bb2c32c-5001-4960-9cd0-22694b9c6a8cname"9ed4d6bc-e430-4200-9ad2-68905b0ff40c WaitActive WaitActive WaitActive,""":v,"""v:"0:,0"cruciblename,"":name"""crucible:"",,"cruciblelevel"":,30""levellevel""::3030,"time":"2023-09-22T23:14:27.611577651Z","hostname":,""time":"ip-10-150-1-74.us-west-2.compute.internal",2023-09-22T23:14:27.611581807Z""pid",:"4301hostname"}:
"Sep 22 23:14:27.611 INFO Current flush_numbers [0..12]: [0, 0]
,{"time""ip-10-150-1-74.us-west-2.compute.internalmsg"",:""pid":":4301}2023-09-22T23:14:27.611581735Z"
,"hostname":"{[2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 4bb2c32c-5001-4960-9cd0-22694b9c6a8c, encrypted: true, database_read_version: 1, database_write_version: 1 }"",msg""v:"":ip-10-150-1-74.us-west-2.compute.internal"0,",pid"[2] downstairs client at 127.0.0.1:33349 has UUID 922c2bc8-c976-47a3-990e-e4ec311f6c5b""name",:""v"crucible":,0",level"":name30"::"4301crucible","}level"
:30,"time":"2023-09-22T23:14:27.611632811Z"{,"hostname":""msg":"ip-10-150-1-74.us-west-2.compute.internal",","time"pid:"":4301}66688a47-67b1-4e47-8e76-e0e145532b3a WaitActive WaitActive WaitActive"
2023-09-22T23:14:27.611641895Z",",{"v"hostname""msg:"":":0ip-10-150-1-74.us-west-2.compute.internal,""a697641e-15fb-475d-87b4-6f9c9b92b978 WaitActive WaitActive WaitActive,"",pid""v"::43010name},Sep 22 23:14:27.611 INFO Current flush_numbers [0..12]: [0, 0]

""name{":"":"crucible"cruciblemsg"",:""level",:"30level":30[2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 922c2bc8-c976-47a3-990e-e4ec311f6c5b, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,",name"":time"":"crucible","level"2023-09-22T23:14:27.611690958Z:"30,"hostname":"Sep 22 23:14:27.611 INFO [0] downstairs client at 127.0.0.1:42970 has UUID 122e249d-ed33-49c6-bb2e-b79a59ebf9a6
Sep 22 23:14:27.611 INFO Current flush_numbers [0..12]: [0, 0]
,",ip-10-150-1-74.us-west-2.compute.internal""time",time""pid"::"4301:"}
2023-09-22T23:14:27.611703634Z"2023-09-22T23:14:27.611692722Z",","hostname"hostname"::""ip-10-150-1-74.us-west-2.compute.internal"ip-10-150-1-74.us-west-2.compute.internal",,""pid"pid"::43014301}
}Sep 22 23:14:27.611 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 122e249d-ed33-49c6-bb2e-b79a59ebf9a6, encrypted: true, database_read_version: 1, database_write_version: 1 }
{
"msg"{:""9ed4d6bc-e430-4200-9ad2-68905b0ff40c WaitActive WaitActive WaitActive"msg":,""v":0:,""name":"Sep 22 23:14:27.611 INFO Downstairs has completed Negotiation, task: proc
[2] downstairs client at 127.0.0.1:39722 has UUID bb12d53e-6697-4186-ab01-3149bf11c327"cruciblecrucible"",",,""levellevel""::3030v":0,"name":"crucible","level":30,,""timetime""::""2023-09-22T23:14:27.611789368Z2023-09-22T23:14:27.611789529Z"",,""hostnamehostname""::""ip-10-150-1-74.us-west-2.compute.internalip-10-150-1-74.us-west-2.compute.internal,"",",""pid"pid:"4301:time"4301}:"}

2023-09-22T23:14:27.61179493Z{"Sep 22 23:14:27.611 INFO a50fc3fb-3de2-4743-9c50-cc80cfba77db WaitActive WaitActive WaitActive
","msg"hostname":":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
[2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: ed248c78-99b2-4b20-8580-19eb24b6c3f5, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":{"crucible","level":"30msg":"Sep 22 23:14:27.611 INFO Downstairs has completed Negotiation, task: proc
[2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: bb12d53e-6697-4186-ab01-3149bf11c327, encrypted: true, database_read_version: 1, database_write_version: 1 }","v",":time0":,""name":"crucible2023-09-22T23:14:27.611851597Z"",","hostnameSep 22 23:14:27.611 INFO Downstairs has completed Negotiation, task: proc
":"level"ip-10-150-1-74.us-west-2.compute.internal:"30,"pid":4301}
{"msg":"5b98377f-d3ca-45bf-893d-9aae2fd5a48e WaitActive WaitActive WaitActive","v":0,"name":"crucible",",level":"30time":"2023-09-22T23:14:27.611881413Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301},
"time":"Sep 22 23:14:27.611 INFO [1] downstairs client at 127.0.0.1:62060 has UUID 9191f668-24b7-4176-b853-43cf34e8ccd3
{2023-09-22T23:14:27.611898217Z"","msghostname"":":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
66688a47-67b1-4e47-8e76-e0e145532b3a WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30Sep 22 23:14:27.611 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 9191f668-24b7-4176-b853-43cf34e8ccd3, encrypted: true, database_read_version: 1, database_write_version: 1 }
,"time":"2023-09-22T23:14:27.611960257Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"Upstairs starts"Sep 22 23:14:27.611 INFO a50fc3fb-3de2-4743-9c50-cc80cfba77db WaitActive WaitActive WaitActive
,"v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.612013875Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"Crucible Version: BuildInfo {\n    version: Sep 22 23:14:27.612 INFO Current flush_numbers [0..12]: [0, 0]
\"0.0.1\",\n    git_sha: \"Sep 22 23:14:27.612 INFO [2] downstairs client at 127.0.0.1:42513 has UUID b37a15c7-b55a-4f3b-ab04-a618bf111318
ed48f294784d46ea7d4bb99336918b74358eca46\",\n    git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n    git_branch: \"main\",\n    rustc_semver: \"1.70.0\",\n    rustc_channel: \"stable\",\n    rustc_host_triple: \"x86_64-unknown-illumos\",\n    rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n    cargo_triple: \"x86_64-unknown-illumos\",\n    debug: true,\n    opt_level: 0,\n}"Sep 22 23:14:27.612 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: b37a15c7-b55a-4f3b-ab04-a618bf111318, encrypted: true, database_read_version: 1, database_write_version: 1 }
,"v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.612127745Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
Sep 22 23:14:27.612 INFO Current flush_numbers [0..12]: [0, 0]
{Sep 22 23:14:27.612 INFO a50fc3fb-3de2-4743-9c50-cc80cfba77db WaitActive WaitActive WaitActive
"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.612184121Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"Crucible stats registered with UUID: 9fe7fa85-122e-4e82-8591-96595eb36a20","v":0,"name":"crucible","level":30,"time":"The guest has requested activation
2023-09-22T23:14:27.612229436Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
Sep 22 23:14:27.612 INFO Downstairs has completed Negotiation, task: proc
{"msg":"Crucible 9fe7fa85-122e-4e82-8591-96595eb36a20 has session id: baa610ca-0e7f-4837-b89d-19a907fb476a","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.612284457Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
Sep 22 23:14:27.612 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.612 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.612 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.612 INFO Downstairs has completed Negotiation, task: proc
{"msg":"[0] 64be6ec7-0ac5-4896-883a-00ec34140b84 (58d25f3b-f138-43ff-93b7-02b5e6bdc53e) WaitActive WaitActive WaitActive ds_transition to WaitQuorum"Sep 22 23:14:27.612 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.612 INFO Current flush_numbers [0..12]: [0, 0]
,"v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.612392805Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v"Sep 22 23:14:27.612 INFO listening on 127.0.0.1:0, task: main
:0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.612446323Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40Sep 22 23:14:27.612 INFO Current flush_numbers [0..12]: [0, 0]
,"time":"2023-09-22T23:14:27.612488758Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"Sep 22 23:14:27.612 INFO Downstairs has completed Negotiation, task: proc
2023-09-22T23:14:27.612532558Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"Sep 22 23:14:27.612 INFO Downstairs has completed Negotiation, task: proc
[1] 64be6ec7-0ac5-4896-883a-00ec34140b84 (58d25f3b-f138-43ff-93b7-02b5e6bdc53e) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.612577967Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
Sep 22 23:14:27.612 INFO Downstairs has completed Negotiation, task: proc
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.612622197Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":{40"msg":"[0] 9ed4d6bc-e430-4200-9ad2-68905b0ff40c (ad0fa1b4-52c3-4329-aeea-287c43c0dc1e) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":,0",time"":name"":"crucible"2023-09-22T23:14:27.612657922Z",{,""level"hostname:":"30ip-10-150-1-74.us-west-2.compute.internal","pid":"4301msg}"
:"{[0] connecting to 127.0.0.1:58086"",msg",:"""time[1] Starts reconcile loop"":","v"2023-09-22T23:14:27.612676054Zv:"":00,",,""hostname"name:"":"name":"ip-10-150-1-74.us-west-2.compute.internalcrucible"crucible,"",Sep 22 23:14:27.612 INFO Downstairs has completed Negotiation, task: proc
""pidlevel""::430130,"}level"
:30{,""time"msg:"":"2023-09-22T23:14:27.612718265Z","[0] Transition from WaitActive to WaitQuorum"hostname":,""v":0,"ip-10-150-1-74.us-west-2.compute.internalname"":,""pidcrucible"":,4301"level}"
:30{"msg":","time":[2] 64be6ec7-0ac5-4896-883a-00ec34140b84 (58d25f3b-f138-43ff-93b7-02b5e6bdc53e) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum"",,""vtime""::0","2023-09-22T23:14:27.61272565Zname2023-09-22T23:14:27.612739601Z"":","crucible"hostname,"":""level",:"30ip-10-150-1-74.us-west-2.compute.internalhostname"",:""pid":4301}
,"ip-10-150-1-74.us-west-2.compute.internal"time"{:",""pidmsg2023-09-22T23:14:27.612761922Z"":","":[0] new RM replaced this: Nonehostname"",:""v"4301:0ip-10-150-1-74.us-west-2.compute.internal",","name"pid"::"4301,"}crucible
"looper":"{,"0"level"":msg40}":"
[2] Transition from WaitActive to WaitQuorum","v":0,"name":","crucible"time",:""{level2023-09-22T23:14:27.612801291Z"":30,"hostname":""msg":"ip-10-150-1-74.us-west-2.compute.internal","pid[1] connecting to 127.0.0.1:50371"":4301,"}v",
":time0{":,""name":"msg"":crucible2023-09-22T23:14:27.612815166Z""",[0] Starts reconcile loop"",",level"hostname""v:""::0,ip-10-150-1-74.us-west-2.compute.internal""name",:""pid"30:crucible"4301,"}level
":30{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:27.612852681Z","hostname":","time":"ip-10-150-1-74.us-west-2.compute.internal",,""2023-09-22T23:14:27.612849199Z"pidtime""::,""4301hostname":"}2023-09-22T23:14:27.612862214Z
"ip-10-150-1-74.us-west-2.compute.internal",","hostname{"pid"":msg:"":"4301ip-10-150-1-74.us-west-2.compute.internal",","looper"pid"::"4301[1] 9ed4d6bc-e430-4200-9ad2-68905b0ff40c (ad0fa1b4-52c3-4329-aeea-287c43c0dc1e) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum}"
,"1{v""":msg0",:""}Sep 22 23:14:27.612 INFO Current flush_numbers [0..12]: [0, 0]
[2] Starts reconcile loopname
"":",crucible""v",:"0{,level"":name"30:"crucible",""level"msg"::30"[2] connecting to 127.0.0.1:54983","v":0,"name":"crucible","level":30{"msg":"[0] a697641e-15fb-475d-87b4-6f9c9b92b978 (01d84cdc-ab54-426d-be83-dc686be9c9e6) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.612920009Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.612966838Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
,"time":"2023-09-22T23:14:27.613065626Z","{hostname":"ip-10-150-1-74.us-west-2.compute.internal"",msg""pid":":4301[0] Transition from WaitActive to WaitQuorum}"Sep 22 23:14:27.613 INFO Downstairs has completed Negotiation, task: proc
,"
v":0,"name":"{crucible","level":"30msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:27.613117111Z","hostname":","ip-10-150-1-74.us-west-2.compute.internal"time",:""pid":43012023-09-22T23:14:27.61312507Z"},
"hostname":"ip-10-150-1-74.us-west-2.compute.internal"{,"pid":4301"}msg
":"[0] new RM replaced this: None"{,"v":0,"name"":msg"":"crucible","[1] Starts reconcile loop"level":,"40v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.613175489Z",,""hostname":"time":"ip-10-150-1-74.us-west-2.compute.internal"2023-09-22T23:14:27.613180476Z,""pid",":hostname":"4301}ip-10-150-1-74.us-west-2.compute.internal"
,"pid":4301}{
"Sep 22 23:14:27.613 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.613 INFO accepted connection from 127.0.0.1:36467, task: main
msg"{:"[0] Starts reconcile loop"","msg"v"::0","name":"crucible","level":30[2] 9ed4d6bc-e430-4200-9ad2-68905b0ff40c (ad0fa1b4-52c3-4329-aeea-287c43c0dc1e) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.613241907Z","hostname":",ip-10-150-1-74.us-west-2.compute.internal"","time"pid"::4301"}
2023-09-22T23:14:27.613250147Z","hostname":"{ip-10-150-1-74.us-west-2.compute.internal","pid":"4301msg":"}
[1] a697641e-15fb-475d-87b4-6f9c9b92b978 (01d84cdc-ab54-426d-be83-dc686be9c9e6) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"{crucible","level":30"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30Sep 22 23:14:27.613 INFO accepted connection from 127.0.0.1:54842, task: main
,"time":"2023-09-22T23:14:27.613294848Z","hostname":","ip-10-150-1-74.us-west-2.compute.internaltime":"","pid"2023-09-22T23:14:27.613304107Z":4301,"hostname":}"
ip-10-150-1-74.us-west-2.compute.internal","pid":4301}{
"msg":"{[1] Transition from WaitActive to WaitQuorum","v"":msg"0:,""name":"[2] new RM replaced this: Nonecrucible"",",level"":v":300,"name":"crucible","level":40,"time":","2023-09-22T23:14:27.613360261Z"time":","hostname":2023-09-22T23:14:27.613364294Z"","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pidip-10-150-1-74.us-west-2.compute.internal"":,"4301pid":4301}
}
{"{msg":""[1] new RM replaced this: None"msg":","v":[2] Starts reconcile loop"0,,""namev":"":crucible0",",name""level:"":crucible"40,"level":30,"time":","2023-09-22T23:14:27.613420032Ztime"":","hostname":"2023-09-22T23:14:27.613421816Z","hostname"ip-10-150-1-74.us-west-2.compute.internal"Sep 22 23:14:27.613 INFO Downstairs has completed Negotiation, task: proc
,:""pid":4301ip-10-150-1-74.us-west-2.compute.internal"},
"pid":4301}{
"msg":"[1] Starts reconcile loop"{,"v":0,"name"":msg"":"crucible","level":[0] 127.0.0.1:36835 task reports connection:true"30,"v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.61348132Z","hostname":","ip-10-150-1-74.us-west-2.compute.internal"time":","pid":2023-09-22T23:14:27.613487039Z"4301,"}hostname
":"ip-10-150-1-74.us-west-2.compute.internal","{pid":4301}
"msg":"{"msg":"[2] a697641e-15fb-475d-87b4-6f9c9b92b978 (01d84cdc-ab54-426d-be83-dc686be9c9e6) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":09ed4d6bc-e430-4200-9ad2-68905b0ff40c WaitQuorum WaitQuorum WaitQuorum,""name":","vcrucible"",":level"0:,"30name":"crucible","level":30,"time":","2023-09-22T23:14:27.61354247Z"time":","hostname":"2023-09-22T23:14:27.613545692Z","ip-10-150-1-74.us-west-2.compute.internalhostname":"","pid":4301ip-10-150-1-74.us-west-2.compute.internal"},"
pid":4301}
{"{msg":""[2] Transition from WaitActive to WaitQuorum"msg":","v":[0]R  flush_numbers: [0, 0]0",","name":"v"crucible:"0,,""levelname""::"30crucible","level":30,"time":","time"2023-09-22T23:14:27.613604793Z":","hostname"2023-09-22T23:14:27.613607044Z":",The guest has finished waiting for activation
"The guest has finished waiting for activation
hostname"ip-10-150-1-74.us-west-2.compute.internal":,""pid":4301ip-10-150-1-74.us-west-2.compute.internal"},"
pid":4301}
{"{msg":"[2] new RM replaced this: None""msg",:""v":[0]R  generation: [0, 0]"0,",name"":v"":crucible"0,,""name"level"::"40crucible","level":30,"time":","time":2023-09-22T23:14:27.613677912Z"","2023-09-22T23:14:27.613680411Z"hostname":","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":ip-10-150-1-74.us-west-2.compute.internal4301Sep 22 23:14:27.613 INFO Current flush_numbers [0..12]: [0, 0]
}"
,"pid":4301{}
"msg":"[2] Starts reconcile loop"{,"v":Sep 22 23:14:27.613 INFO accepted connection from 127.0.0.1:41021, task: main
0","msg"name"::""crucible"[0]R  dirty: [false, false]","level,""v"::300,"name":"crucible","level":30,"time":","2023-09-22T23:14:27.613746309Z"time":","hostname":"2023-09-22T23:14:27.613750415Z","hostname":ip-10-150-1-74.us-west-2.compute.internal"","pid":4301ip-10-150-1-74.us-west-2.compute.internal","}pid"
:4301}
{{"msg":""msg"[0] 127.0.0.1:56524 task reports connection:true":","v":[1]R  flush_numbers: [0, 0]"0,","vname":"":crucibleThe guest has finished waiting for activation
"0,","level"name"::"30crucible","level":30,"time":","time":"2023-09-22T23:14:27.613810965Z",2023-09-22T23:14:27.613813262Z""hostname":","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid"ip-10-150-1-74.us-west-2.compute.internal":,4301"pid":}4301
}
{"{msg":""msg":"a697641e-15fb-475d-87b4-6f9c9b92b978 WaitQuorum WaitQuorum WaitQuorum","[1]R  generation: [0, 0]"v":,0",v"":name"0:,""cruciblename"":","crucible"level":,"30level":30,","time":time"":"2023-09-22T23:14:27.613873108Z"2023-09-22T23:14:27.613874159Z",","hostnamehostname":"":"ip-10-150-1-74.us-west-2.compute.internal"ip-10-150-1-74.us-west-2.compute.internal",","pidpid""::43014301}}

{"msg"{:"[1]R  dirty: [false, false]",""v"Sep 22 23:14:27.613 INFO Downstairs has completed Negotiation, task: proc
msg":0:","name":"[0]R  flush_numbers: [0, 0]"crucible",,""levelv":"0:,"30name":"crucible","level":30,"time":"2023-09-22T23:14:27.613947194Z","hostname":","time":"ip-10-150-1-74.us-west-2.compute.internal",2023-09-22T23:14:27.61395059Z""pid":,"4301hostname":"}
ip-10-150-1-74.us-west-2.compute.internal","pid":4301{}
"msg":"[2]R  flush_numbers: [0, 0]"{,"v":0,"name":""msg":crucible"","level":[0]R  generation: [0, 0]"30,,""v":time0":,""name":"2023-09-22T23:14:27.61292534Zcrucible"","level",:"30hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
,"time":","time":"2023-09-22T23:14:27.614000501Z"{2023-09-22T23:14:27.614006342Z",","hostname"hostname""msg:"":":"ip-10-150-1-74.us-west-2.compute.internal","[0] 127.0.0.1:45897 task reports connection:truepid"":,4301"ip-10-150-1-74.us-west-2.compute.internal"}v
",":pid"{0:,""4301namemsg""::""}crucible"[0]R  dirty: [false, false]",
","levelv""::030,"name":"crucible{","level":30Sep 22 23:14:27.614 INFO Connection request from 9fe7fa85-122e-4e82-8591-96595eb36a20 with version 4, task: proc
",,""timetime"":":"msg":"2023-09-22T23:14:27.614049083Z"2023-09-22T23:14:27.61405302Z",",[2]R  generation: [0, 0]"hostname"":","hostnamev""ip-10-150-1-74.us-west-2.compute.internal:"",":pid"0ip-10-150-1-74.us-west-2.compute.internal:,"4301"name"},
":pid"":{crucible"4301,"}msg
""Sep 22 23:14:27.614 INFO upstairs UpstairsConnection { upstairs_id: 9fe7fa85-122e-4e82-8591-96595eb36a20, session_id: 57e450d1-3a66-4c19-9890-8b892a800667, gen: 1 } connected, version 4, task: proc
{:"level"":msg":30"64be6ec7-0ac5-4896-883a-00ec34140b84 WaitQuorum WaitQuorum WaitQuorum","[1]R  flush_numbers: [0, 0]v"":,0",v"":name0":,""namecrucible"":","crucible"level",:"30level":30,,",time""time:"":""2023-09-22T23:14:27.614106139Ztime"2023-09-22T23:14:27.614106831Z:""",,""2023-09-22T23:14:27.614099629Z"hostnamehostname""::"","hostname":"The guest has finished waiting for activation
ip-10-150-1-74.us-west-2.compute.internalip-10-150-1-74.us-west-2.compute.internal"",,""pid"pid:"4301:ip-10-150-1-74.us-west-2.compute.internal}4301
}"
{,Sep 22 23:14:27.614 INFO [0] a50fc3fb-3de2-4743-9c50-cc80cfba77db (2b698136-bbd2-4fcb-ab7c-5f19b2267eb8) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
""{msg"pid"":msg"":":[0]R  flush_numbers: [0, 0]"4301,[1]R  generation: [0, 0]""v",:"}0
,v"":name0":,""namecrucible"":","crucible"level,"":level30":{30Sep 22 23:14:27.614 INFO [0] Transition from WaitActive to WaitQuorum
"msg":",","timetime""::""[2]R  dirty: [false, false]"Sep 22 23:14:27.614 INFO Connection request from 9fe7fa85-122e-4e82-8591-96595eb36a20 with version 4, task: proc
,"2023-09-22T23:14:27.614174558Z"2023-09-22T23:14:27.614173117Z,""hostnameSep 22 23:14:27.614 WARN [0] new RM replaced this: None
v":,"":"hostname"0:"ip-10-150-1-74.us-west-2.compute.internal,""ip-10-150-1-74.us-west-2.compute.internal,"",pid"":pid4301"name":"Sep 22 23:14:27.614 INFO upstairs UpstairsConnection { upstairs_id: 9fe7fa85-122e-4e82-8591-96595eb36a20, session_id: 57e450d1-3a66-4c19-9890-8b892a800667, gen: 1 } connected, version 4, task: proc
crucible",}:
4301"}{
"levelmsg"{"::""30msg":[1]R  dirty: [false, false]"","v"[0]R  generation: [0, 0]:"0,,""vname""::0","crucible"name",:""level"crucible:"30,"level":30,"time",,""timetime""::"":"2023-09-22T23:14:27.614239738Z2023-09-22T23:14:27.61423839Z""2023-09-22T23:14:27.614231676Z",,"","hostnamehostname""::""hostname":"ip-10-150-1-74.us-west-2.compute.internalip-10-150-1-74.us-west-2.compute.internal"",,""pidpid""::43014301ip-10-150-1-74.us-west-2.compute.internal}}

{",""{pidmsg""msg:"""::[2]R  flush_numbers: [0, 0]""4301,"[0]R  dirty: [false, false]v""Sep 22 23:14:27.614 INFO [0] Starts reconcile loop
}
:,"0v",:"0name,"":"name":crucible"{"crucible",","level":level":30"30msg":"Max found gen is 1","v":0,"nameSep 22 23:14:27.614 INFO Connection request from 9fe7fa85-122e-4e82-8591-96595eb36a20 with version 4, task: proc
":,,""timetime""::""{"2023-09-22T23:14:27.614305769Z"2023-09-22T23:14:27.614304649Z",crucible"",hostname"""hostname:"":,msg",:""ip-10-150-1-74.us-west-2.compute.internaltime""","[2] Proc runs for 127.0.0.1:56258 in state New:""ip-10-150-1-74.us-west-2.compute.internalpid"":,4301""2023-09-22T23:14:27.612934972Z,level"""v"Sep 22 23:14:27.614 INFO upstairs UpstairsConnection { upstairs_id: 9fe7fa85-122e-4e82-8591-96595eb36a20, session_id: 57e450d1-3a66-4c19-9890-8b892a800667, gen: 1 } connected, version 4, task: proc
:pid:300","name":":crucible"4301,"}level"
:30{,"time":""2023-09-22T23:14:27.614359459Z"msg"{:,""hostname"[2]R  generation: [0, 0]":",""v"msg":ip-10-150-1-74.us-west-2.compute.internal"0,,:"",""nametimepid":"""[0] 66688a47-67b1-4e47-8e76-e0e145532b3a (ff839372-f996-4bbe-90dd-222e8d070509) WaitActive WaitActive WaitActive ds_transition to WaitQuorum2023-09-22T23:14:27.614373795Z"":",,""v"::hostname0",:""namecrucible"4301"ip-10-150-1-74.us-west-2.compute.internal:"},"",pid
}"crucible
{""level":,:"4301level"30}:
30{""msgmsg""::""[1]R  flush_numbers: [0, 0]"Generation requested: 1 >= found:1",,""vv""::00,",name"Sep 22 23:14:27.614 INFO [1] a50fc3fb-3de2-4743-9c50-cc80cfba77db (2b698136-bbd2-4fcb-ab7c-5f19b2267eb8) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
,,name"":":""timecruciblecrucible"""",,"time":":level""level""::302023-09-22T23:14:27.614429688Z2023-09-22T23:14:27.614435208Z"30","hostname",:""hostname":ip-10-150-1-74.us-west-2.compute.internal"Sep 22 23:14:27.614 INFO [1] Transition from WaitActive to WaitQuorum
",ip-10-150-1-74.us-west-2.compute.internal"",pid,"",time""time":"4301::""pid}2023-09-22T23:14:27.6144702Z2023-09-22T23:14:27.614472413Z""
Sep 22 23:14:27.614 WARN [1] new RM replaced this: None
",{",:hostname"""hostname:"":msg"4301"ip-10-150-1-74.us-west-2.compute.internal":,}ip-10-150-1-74.us-west-2.compute.internal"
""pid",:"4301pid":4301[0] Transition from WaitActive to WaitQuorum"}}

,"{{{v":0""msgmsg",""Sep 22 23:14:27.614 INFO [1] Starts reconcile loop
msg":""name:Next flush: 1:":"""",[1]R  generation: [0, 0]""[2]R  dirty: [false, false]"v,"":v0",:"0,",name""name:"":"cruciblev"crucible"":,,0"",levellevel""::3030"name":"crucible","level":30,,""timetime""::""2023-09-22T23:14:27.614569682Z2023-09-22T23:14:27.61456946Z""crucible,,"""hostnamehostname""::"","level":ip-10-150-1-74.us-west-2.compute.internalip-10-150-1-74.us-west-2.compute.internal30"",,"",pidpid""::43014301"}}

{time{"msg":""msg"All extents match:"","v":[1]R  dirty: [false, false]0",","namev""::"0,crucible,"""name,"":"level"crucible:"30,"time"level:"":302023-09-22T23:14:27.614591683Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":,4301"time":",}Sep 22 23:14:27.614 INFO [2] a50fc3fb-3de2-4743-9c50-cc80cfba77db (2b698136-bbd2-4fcb-ab7c-5f19b2267eb8) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
2023-09-22T23:14:27.614614613Z"",time"":hostname"":
"2023-09-22T23:14:27.61461774Z"",ip-10-150-1-74.us-west-2.compute.internal"":{"hostname,"":"pid":2023-09-22T23:14:27.614574118Z"4301ip-10-150-1-74.us-west-2.compute.internal"}"Sep 22 23:14:27.614 INFO [2] Transition from WaitActive to WaitQuorum
,"
pid":,msg"{4301:"""}msg
"[0] new RM replaced this: None:{"""hostname"No downstairs repair requiredmsg",":v":"Sep 22 23:14:27.614 WARN [2] new RM replaced this: None
,""v[2]R  flush_numbers: [0, 0]"":,0",v":"0:name,"0:,"""cruciblename"",:"""levelcrucible"":,30"ip-10-150-1-74.us-west-2.compute.internallevel"name":"30:,"pid":4301}
,,""Sep 22 23:14:27.614 INFO [2] Starts reconcile loop
"timetime""::""crucible"{2023-09-22T23:14:27.61471051Z2023-09-22T23:14:27.614706203Z"",,,"""hostnamehostname"""::msg""":"level":ip-10-150-1-74.us-west-2.compute.internalip-10-150-1-74.us-west-2.compute.internal40"",,""pid"pid"::43014301Max found gen is 1}}

{"{"msg":""msg":"No initial repair work was required"[2]R  generation: [0, 0],""v,"":v0":,0","name"name:"":"crucible"crucible",",,"time"level""level:"30:30:"2023-09-22T23:14:27.614745893Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301},
,""timetime""::""2023-09-22T23:14:27.614769614Z2023-09-22T23:14:27.614770283Z"",",hostname""{:hostname"":Sep 22 23:14:27.614 INFO [0] 127.0.0.1:42970 task reports connection:true
ip-10-150-1-74.us-west-2.compute.internal"ip-10-150-1-74.us-west-2.compute.internal""",,""pid"pid:"4301:msg"}4301
}:"
{[0] Starts reconcile loop{""msg":"","v"Set Downstairs and Upstairs activemsg":",":v"":00[2]R  dirty: [false, false],"",name"",name""v"::"0:,crucible""name,"":"levelcrucible"",:""Sep 22 23:14:27.614 INFO a50fc3fb-3de2-4743-9c50-cc80cfba77db WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:14:27.614 INFO UpstairsConnection { upstairs_id: 9fe7fa85-122e-4e82-8591-96595eb36a20, session_id: 57e450d1-3a66-4c19-9890-8b892a800667, gen: 1 } is now active (read-write)
crucible"level30",:"30level":30,"time,"":"time":"2023-09-22T23:14:27.614853895Z"2023-09-22T23:14:27.614855966Z,"",hostname"":"hostname":"ip-10-150-1-74.us-west-2.compute.internal","ip-10-150-1-74.us-west-2.compute.internal"pid,"":pid4301",}:"4301
}time"
{:"Sep 22 23:14:27.614 INFO [0]R  flush_numbers: [0, 0]
"{msg2023-09-22T23:14:27.614857535Z""":msg"":,""hostname"Max found gen is 1":9ed4d6bc-e430-4200-9ad2-68905b0ff40c is now active with session: ad0fa1b4-52c3-4329-aeea-287c43c0dc1e,""",v"":v"0:0,","nameip-10-150-1-74.us-west-2.compute.internal"name",":"":"crucible"pid",crucible""Sep 22 23:14:27.614 INFO [0]R  generation: [0, 0]
level,""::30level"4301:30}
,"{time,"":"time":""2023-09-22T23:14:27.61491817Z2023-09-22T23:14:27.614920044Z""Sep 22 23:14:27.614 INFO [0]R  dirty: [false, false]
,msg"":hostname":""ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
[1] 66688a47-67b1-4e47-8e76-e0e145532b3a (ff839372-f996-4bbe-90dd-222e8d070509) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum","v":{0,"name":""crucible"msg,"":"level"Sep 22 23:14:27.614 INFO [1]R  flush_numbers: [0, 0]
Sep 22 23:14:27.614 INFO UpstairsConnection { upstairs_id: 9fe7fa85-122e-4e82-8591-96595eb36a20, session_id: 57e450d1-3a66-4c19-9890-8b892a800667, gen: 1 } is now active (read-write)
,9ed4d6bc-e430-4200-9ad2-68905b0ff40c Set Active after no repair"":,hostname""v:""30:0,ip-10-150-1-74.us-west-2.compute.internal""name,"":"pid":crucible"4301,"}
level"Sep 22 23:14:27.614 INFO [1]R  generation: [0, 0]
:{30",msg"":"time":"Generation requested: 1 >= found:1",2023-09-22T23:14:27.614972581Z""v",:"0,",hostname"time""name"::"":"2023-09-22T23:14:27.61498456Zcrucible""Sep 22 23:14:27.614 INFO [1]R  dirty: [false, false]
ip-10-150-1-74.us-west-2.compute.internal",,"level"":hostname30",":pid"":4301}ip-10-150-1-74.us-west-2.compute.internal"
,,""pidtime""::,{"4301"}2023-09-22T23:14:27.615012302Z"
,""{hostnamemsg"v"msg"Sep 22 23:14:27.615 INFO [2]R  flush_numbers: [0, 0]
":""::""[1] Transition from WaitActive to WaitQuorum":,"0ip-10-150-1-74.us-west-2.compute.internalNotify all downstairs, region set compare is done.",",,""v""namevpid""::04301,""::name}Sep 22 23:14:27.615 INFO [2]R  generation: [0, 0]
0"
,"crucible{name"""msg:"":":"crucible"crucible",,Next flush: 1""level""",:"30v,level":""level"::30300Sep 22 23:14:27.615 INFO [2]R  dirty: [false, false]
,,""timename""::""2023-09-22T23:14:27.61508988Zcrucible"",,""level"hostname:"30:","ip-10-150-1-74.us-west-2.compute.internal"time,"":"pid":43012023-09-22T23:14:27.61509698Z"}
,,"{time""":msg""Sep 22 23:14:27.615 INFO Max found gen is 1
hostname"2023-09-22T23:14:27.615108611Z:"",,""Set check for repairhostname""time":,"":v"ip-10-150-1-74.us-west-2.compute.internal::0",,""namepid""":":4301"2023-09-22T23:14:27.615096838Z"ip-10-150-1-74.us-west-2.compute.internal"crucible,"},
",level{""Sep 22 23:14:27.615 INFO Generation requested: 1 >= found:1
hostname"":":msg30""pid"::"4301ip-10-150-1-74.us-west-2.compute.internal"All extents match,"}pid"":
,,""v{Sep 22 23:14:27.615 INFO Next flush: 1
""time:"4301:0",}"
msg"name2023-09-22T23:14:27.615167287Z"":{:,"""hostnamecrucible"":"",[1] new RM replaced this: Noneip-10-150-1-74.us-west-2.compute.internal"""msg",:"",level""Generation requested: 1 >= found:1":pid30"v"::,0"4301v"Sep 22 23:14:27.615 INFO All extents match
,,}":"0
time"name":"{:",crucible"2023-09-22T23:14:27.615217927Z",""Sep 22 23:14:27.615 INFO No downstairs repair required
name"",msg""hostname:":"":"level":crucibleip-10-150-1-74.us-west-2.compute.internal40"[1] 127.0.0.1:52116 task reports connection:true,"""pid,"":v4301",}:
0,",{"Sep 22 23:14:27.615 INFO No initial repair work was required
level""name"msg":::""30"No downstairs repair requiredcrucible""time",,""v"level:"0:,30:""name"Sep 22 23:14:27.615 INFO Set Downstairs and Upstairs active
2023-09-22T23:14:27.615259798Z":","crucible"hostname",:"","level":time30":ip-10-150-1-74.us-west-2.compute.internal"",","2023-09-22T23:14:27.615294104Zpid"time":"4301:,"},"2023-09-22T23:14:27.615287762Z"
hostname""time:"":"{{,ip-10-150-1-74.us-west-2.compute.internal"2023-09-22T23:14:27.615305098Z,"",msg"""""pidhostnamehostnamemsg"":::"":":""Sep 22 23:14:27.615 INFO a50fc3fb-3de2-4743-9c50-cc80cfba77db is now active with session: 2b698136-bbd2-4fcb-ab7c-5f19b2267eb8
ip-10-150-1-74.us-west-2.compute.internal"ip-10-150-1-74.us-west-2.compute.internal4301[0] 5b98377f-d3ca-45bf-893d-9aae2fd5a48e (d3623aa7-ee8c-40f8-a982-d0eadbcd4a76) WaitActive WaitActive WaitActive ds_transition to WaitQuorum,""",pid"[1] Starts reconcile loop"v""::}"
,",04301,"{pid"}"
namev"":msg4301"{"}:
":Sep 22 23:14:27.615 INFO a50fc3fb-3de2-4743-9c50-cc80cfba77db Set Active after no repair
{9ed4d6bc-e430-4200-9ad2-68905b0ff40c Active Active Active":""cruciblemsg"",:""0,""msgv""::"Next flush: 1level"No initial repair work was required0",,""namev"",,"":"30Sep 22 23:14:27.615 INFO Notify all downstairs, region set compare is done.
v":name":"0crucible,"name"::0","crucible"name,":""level"crucible:"30,,:""",time"""crucible:level"":,2023-09-22T23:14:27.615401901Z"",",level""level"30hostname:time"":""30:":2023-09-22T23:14:27.615420581Z",ip-10-150-1-74.us-west-2.compute.internal"",,hostname""time:"":30""Sep 22 23:14:27.615 INFO Set check for repair
,pid""time:2023-09-22T23:14:27.615434845Zip-10-150-1-74.us-west-2.compute.internal""4301":,,""pid"hostname:"4301:}"
"},
ip-10-150-1-74.us-west-2.compute.internal"2023-09-22T23:14:27.615440169Z{"{,"",""msg"hostname:":"""pidmsg"":ip-10-150-1-74.us-west-2.compute.internal4301:time"[0] Transition from WaitActive to WaitQuorum,"""pid,"":v:"4301:}"
Set check for repair"0{",}"2023-09-22T23:14:27.615448433Z"name
"",msg{:Sep 22 23:14:27.615 INFO [1] 127.0.0.1:62060 task reports connection:true
"",:""vSet Downstairs and Upstairs active"":,"0vhostname""msg"crucible:""","":name0",:"All extents match,name"":"crucible:"","crucible"level,"":level30"":ip-10-150-1-74.us-west-2.compute.internal"level""Sep 22 23:14:27.615 INFO a50fc3fb-3de2-4743-9c50-cc80cfba77db Active Active Active
,":v30",30":,time"0"pid:,""":name4301,2023-09-22T23:14:27.615540858Z""time",:""Sep 22 23:14:27.615 INFO Set check for repair
2023-09-22T23:14:27.615554032Zhostname""},:"""hostname":,"":
"ip-10-150-1-74.us-west-2.compute.internal"crucibletime"",:""{ip-10-150-1-74.us-west-2.compute.internal,"",pid"":pid4301""}:
4301msg"}{
"2023-09-22T23:14:27.615552786Z:{msg""",level"":""msg"Sep 22 23:14:27.615 INFO [2] 127.0.0.1:42513 task reports connection:true
[2] 66688a47-67b1-4e47-8e76-e0e145532b3a (ff839372-f996-4bbe-90dd-222e8d070509) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum",:[2] 127.0.0.1:33349 task reports connection:true""",v:hostname30":"64be6ec7-0ac5-4896-883a-00ec34140b84 is now active with session: 58d25f3b-f138-43ff-93b7-02b5e6bdc53e""v",:"0v:",",:00,,"""namename""::""ip-10-150-1-74.us-west-2.compute.internal""time,":""pid"crucible"crucible,"",level""level:"30:2023-09-22T23:14:27.615620212Z:"30,4301Sep 22 23:14:27.615 INFO a50fc3fb-3de2-4743-9c50-cc80cfba77db Active Active Active
name}"
hostname""{::""msg"":ip-10-150-1-74.us-west-2.compute.internal"",,[0] new RM replaced this: None,""crucible"""timetime""::"",pid,"":v4301"2023-09-22T23:14:27.615650172Z2023-09-22T23:14:27.615646927Z"""}Sep 22 23:14:27.615 INFO Set check for repair

:,,""hostname"level{0":hostname"":"",msg""name:"":":ip-10-150-1-74.us-west-2.compute.internalip-10-150-1-74.us-west-2.compute.internal"",,"crucibleNo downstairs repair required"",pid""pid:"430130,""levelv:}4301
}""::400{
",msg{"":""msg"name":,64be6ec7-0ac5-4896-883a-00ec34140b84 Set Active after no repair"",,:""time"crucible:""9ed4d6bc-e430-4200-9ad2-68905b0ff40c Active Active Active,""",v""v:"0:,""time"name0",:"":level"cruciblename"",:"""2023-09-22T23:14:27.615711047Z"cruciblelevel","",":hostname30level"":2023-09-22T23:14:27.615718899Z30":":,"Sep 22 23:14:27.615 INFO [0] received reconcile message
hostname,,""30time"ip-10-150-1-74.us-west-2.compute.internal:""time":":"ip-10-150-1-74.us-west-2.compute.internal",","2023-09-22T23:14:27.615760812Z,"",pid""hostname:"4301:"}pid"2023-09-22T23:14:27.615757489Z""time",:"ip-10-150-1-74.us-west-2.compute.internal
":2023-09-22T23:14:27.615776371Z"4301"hostname",:""{,}""msgpid"":ip-10-150-1-74.us-west-2.compute.internalhostname"4301:,}""pid
"Sep 22 23:14:27.615 INFO [0] All repairs completed, exit
"
:[0] Starts reconcile loop{"4301,""}msg
"v:"":{{No initial repair work was required0","name"":""crucible,msg""v":,0",level""name:"msg"":30:"":Sep 22 23:14:27.615 INFO [0] Starts cmd_loop
"crucibleNotify all downstairs, region set compare is done.",",",time"""[2] Transition from WaitActive to WaitQuorum"v"level:"::"300,,""v"name":2023-09-22T23:14:27.615855099Z":","0crucible","hostname,"":time"":",name"":"level"crucibleip-10-150-1-74.us-west-2.compute.internal2023-09-22T23:14:27.615874955Z"",:"30",pid"","hostname:"4301:"}level"
ip-10-150-1-74.us-west-2.compute.internal":,"{30pid":4301"msg"}:"
,{"time""[1] 5b98377f-d3ca-45bf-893d-9aae2fd5a48e (d3623aa7-ee8c-40f8-a982-d0eadbcd4a76) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorummsg"":",:""Set Downstairs and Upstairs active"v":,"02023-09-22T23:14:27.615899124Z",v"":name0",:"",namecrucible"":,Sep 22 23:14:27.615 INFO [1] received reconcile message
,"""cruciblelevel"",:"30time"level"::"302023-09-22T23:14:27.615910124Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal",",pid":"4301time,"":"time"}:2023-09-22T23:14:27.615942648Z""
Sep 22 23:14:27.615 INFO [1] All repairs completed, exit
2023-09-22T23:14:27.615946163Z,""{hostname,"":hostname""":msg"ip-10-150-1-74.us-west-2.compute.internal":","pid":4301[2] new RM replaced this: None"}
,"v":0{,"name":""cruciblemsg"":","level":40[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible"","level":30ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"Sep 22 23:14:27.615 INFO [1] Starts cmd_loop
,"hostname"time"::","msg":time"""":"2023-09-22T23:14:27.615992744Z"2023-09-22T23:14:27.615998569Za697641e-15fb-475d-87b4-6f9c9b92b978 is now active with session: 01d84cdc-ab54-426d-be83-dc686be9c9e6"",,"",v""ip-10-150-1-74.us-west-2.compute.internal"hostname"hostname:"0:,"",name"ip-10-150-1-74.us-west-2.compute.internal:"":"crucible,"",pid""pid"level:"4301:"}30
:ip-10-150-1-74.us-west-2.compute.internal"4301{,"pid":}4301
}",msg""time:"":"{
"2023-09-22T23:14:27.616044451Z[1] new RM replaced this: None""msg":,{",v"":hostname0"""msg",:"":Set check for repair""ip-10-150-1-74.us-west-2.compute.internalname"",:,Sep 22 23:14:27.616 INFO [2] received reconcile message
""pid[2] Starts reconcile loop"crucible"":,4301","v"level}"
:":v"0{40,:""0msg","name":name":"":",crucible"crucible,"a697641e-15fb-475d-87b4-6f9c9b92b978 Set Active after no repairtime""","Sep 22 23:14:27.616 INFO [2] All repairs completed, exit
,":v"""levellevel"":2023-09-22T23:14:27.616104225Z0"::,,""hostnamename""::""30crucible30ip-10-150-1-74.us-west-2.compute.internal"",,""levelpid""::304301}
{"msg":"[1] Starts reconcile loop","v":0,","name":time"":"crucible","level"2023-09-22T23:14:27.616143665Z:"30,,""hostname":time"":","2023-09-22T23:14:27.616140434Z"ip-10-150-1-74.us-west-2.compute.internal",,","time""pid:"":hostname"43012023-09-22T23:14:27.616158063Z"}:time",
"":{hostname""":msg"":"2023-09-22T23:14:27.616141914Z"ip-10-150-1-74.us-west-2.compute.internal"ip-10-150-1-74.us-west-2.compute.internal,"Notify all downstairs, region set compare is done.,hostname""",",pid"pidv""Sep 22 23:14:27.616 INFO [2] Starts cmd_loop
:"0:",:""ip-10-150-1-74.us-west-2.compute.internal":,43014301name"}:
""}{crucible"pid"
:",4301msg"":"level":}{
"[2] 5b98377f-d3ca-45bf-893d-9aae2fd5a48e (d3623aa7-ee8c-40f8-a982-d0eadbcd4a76) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum30"msg",":v{"":0,"name""[0] 127.0.0.1:32776 task reports connection:truemsg"",:""time"crucible:"",",level2023-09-22T23:14:27.616232403Z"":"30,:v""":hostname"0[1] 127.0.0.1:43400 task reports connection:true",":name":,"","crucible"timeip-10-150-1-74.us-west-2.compute.internal"":,"",pid2023-09-22T23:14:27.616249219Z":"4301"},
""The guest has finished waiting for activation
level"{hostname"::"30"msgip-10-150-1-74.us-west-2.compute.internal","pid""::"4301}Set check for repair
","v"{:0,"name"":msg"":"crucible","level":30[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.616287602Z","hostname":","time":"ip-10-150-1-74.us-west-2.compute.internal","2023-09-22T23:14:27.616300299Z,""pidtime,"":"hostname""::"2023-09-22T23:14:27.616305499Z"4301,"ip-10-150-1-74.us-west-2.compute.internal"hostname",:""pid":}4301ip-10-150-1-74.us-west-2.compute.internal"
},"
pid":{4301{}
""msg"{:"msg"":msg[1] 127.0.0.1:33784 task reports connection:true"":",""v":0,"[2] new RM replaced this: None"name":,""66688a47-67b1-4e47-8e76-e0e145532b3a WaitQuorum WaitQuorum WaitQuorum"vcrucible"":,0",level""name:"30:","crucible"v",:"0level",":name"40:",crucible"","time"level:"":302023-09-22T23:14:27.616355482Z",,""timehostname""::""2023-09-22T23:14:27.616363422Z"ip-10-150-1-74.us-west-2.compute.internal,"","hostname"pid:"":4301}
ip-10-150-1-74.us-west-2.compute.internal","pid":{4301}
"msg":"{,"a697641e-15fb-475d-87b4-6f9c9b92b978 Active Active Activemsg"",:""v"":[2] Starts reconcile loop"0time",,""namev""::"0:,crucible""name,""level:"":"crucible30","level"2023-09-22T23:14:27.616370658Z":30,"hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"}time,"":"
time2023-09-22T23:14:27.616403365Z"":","hostname":2023-09-22T23:14:27.616406322Z""{,"ip-10-150-1-74.us-west-2.compute.internal"hostname",:""pid"":ip-10-150-1-74.us-west-2.compute.internal"4301,"}pid
":msg"{4301:"}"
msg"[0]R  flush_numbers: [0, 0]":{","v"Set check for repair""msg",:""v"::0[0] 127.0.0.1:43186 task reports connection:true0",",name""v:"":,crucible0",","namename""level"::""crucible30":,""level"crucible":,"30level":30,"time":"2023-09-22T23:14:27.616457137Z","hostname",:""time":"ip-10-150-1-74.us-west-2.compute.internal"2023-09-22T23:14:27.616461942Z,""pid,"":4301hostname":"}
ip-10-150-1-74.us-west-2.compute.internal","pid"{:,4301""msg}"
:time"{":""msg"2023-09-22T23:14:27.616463328Z":[2] 127.0.0.1:39493 task reports connection:true"",","hostnamev5b98377f-d3ca-45bf-893d-9aae2fd5a48e WaitQuorum WaitQuorum WaitQuorum"":":"0,","v"name:"0:,""ip-10-150-1-74.us-west-2.compute.internalcruciblename"",:"""levelcrucible,"",:"30"levelpid""::304301}
,"time":",2023-09-22T23:14:27.616504582Z""time",":"hostname":"{2023-09-22T23:14:27.616508584Z","ip-10-150-1-74.us-west-2.compute.internal",hostname"":pid""":4301ip-10-150-1-74.us-west-2.compute.internal"},
"msgpid"":{4301:"}"
msg"[0]R  generation: [0, 0]"{:","v"":a697641e-15fb-475d-87b4-6f9c9b92b978 Active Active Activemsg""0:,"","v[0]R  flush_numbers: [0, 0]"",:"0v,"":name0",:""name"crucible:"","cruciblelevel"":,30"name"level":":30crucible","level":30,"time":"2023-09-22T23:14:27.616555931Z",,""time"hostname:"":"2023-09-22T23:14:27.616559772Z",ip-10-150-1-74.us-west-2.compute.internal"","hostname"pid:"":4301}
ip-10-150-1-74.us-west-2.compute.internal","pid":{4301},"
msg""{:"time":""msgSet check for repair"":","2023-09-22T23:14:27.616562756Z"[0]R  generation: [0, 0]v"":,0",v""name:"0:,"",namecrucible"":,""level"crucible:"30,""level":hostname"30:"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
,,""timetime""::""{2023-09-22T23:14:27.616601748Z2023-09-22T23:14:27.616598819Z"",,""hostnamehostname""::"""msg":"ip-10-150-1-74.us-west-2.compute.internalip-10-150-1-74.us-west-2.compute.internal"",,""pidpid[0]R  dirty: [false, false]"""::,43014301"}}

{v":{"0"msgmsg""::,"""name":"[0]R  dirty: [false, false][0] received reconcile message"",crucible",",v""v:"0:"0,level",""namename:""::""30cruciblecrucible"",,""levellevel""::3030:"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,,""timetime""}::""
,2023-09-22T23:14:27.616656051Z2023-09-22T23:14:27.616656127Z""",,""{time"hostnamehostname""::"":msg""ip-10-150-1-74.us-west-2.compute.internal""2023-09-22T23:14:27.616654042Z:,ip-10-150-1-74.us-west-2.compute.internal""pid,"":""4301pid"Set check for repair",",}:
4301hostname""v}{
""::"0{msg":""msg"ip-10-150-1-74.us-west-2.compute.internal,"","[0] All repairs completed, exit:""namepid""::"4301[1]R  flush_numbers: [0, 0],""}crucible
",v"",{":level"":0v,"":name0",":name""msg30"crucible:"":,crucible""level,"":level30":"30[1]R  flush_numbers: [0, 0]","v":0,"name":"crucible","level":30,",time"":time"",:2023-09-22T23:14:27.616745782Z""","2023-09-22T23:14:27.616747427Z"time"hostname,":"":,""ip-10-150-1-74.us-west-2.compute.internalhostname"":,""2023-09-22T23:14:27.616740211Ztime""pidip-10-150-1-74.us-west-2.compute.internal"":,4301",:""pid}"
:2023-09-22T23:14:27.616754126Z4301{hostname"""}msg"
:":,""[0] Starts cmd_loop{"hostname",""ip-10-150-1-74.us-west-2.compute.internal:"",vmsg""::"0ip-10-150-1-74.us-west-2.compute.internal"[1]R  generation: [0, 0],"",name"":"pid,"":pid"4301:"v"crucible:0","}4301
}name,"":level"":crucible30"{
",msg{":"""levelmsg[0] received reconcile message"":",",:[1]R  generation: [0, 0]""v",:"0v":,0",name""name:"":30"crucible""time",crucible""level,"":level30:"",":time30"2023-09-22T23:14:27.61681141Z:"","hostname"2023-09-22T23:14:27.616830251Z:"","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":ip-10-150-1-74.us-west-2.compute.internal4301",,",time""time:"":""}
pid"2023-09-22T23:14:27.61683642Z2023-09-22T23:14:27.616840076Z""{:,,"""4301msg"}:
"hostnamehostname""::"{[1] received reconcile message""",msg""v:"ip-10-150-1-74.us-west-2.compute.internalip-10-150-1-74.us-west-2.compute.internal"",:"0",pid"[1]R  dirty: [false, false],""pid""::43014301name,"":v""}}

{:crucible0",,""levelname""::30"{"crucible"msg""msg:"":","[1]R  dirty: [false, false]"[0] All repairs completed, exit",,""vv""::00,,"",level""time:"30:namename""::"""cruciblecrucible"",,""2023-09-22T23:14:27.616891461Zlevellevel""::3030,""time,"":"hostname":"2023-09-22T23:14:27.616906102Z","ip-10-150-1-74.us-west-2.compute.internal"hostname",:"pid":4301}
",,""timetime{ip-10-150-1-74.us-west-2.compute.internal""",msg""pid:""::"":[1] All repairs completed, exit4301""}2023-09-22T23:14:27.616914356Z2023-09-22T23:14:27.616913886Z""
,,,""{"hostnamehostname""::"""v"msg:":0",ip-10-150-1-74.us-west-2.compute.internalip-10-150-1-74.us-west-2.compute.internal"[2]R  flush_numbers: [0, 0]name"""",,""pid"pid:"4301:,:""v"crucible:"0,,""4301}}

{namelevel""::"30"{msg"crucible:""msg"":[2]R  flush_numbers: [0, 0]"","[0] Starts cmd_loop,"",v""v:"0:,0",level""timename,"":name"":"::30"crucible"",crucible""2023-09-22T23:14:27.616966502Z",level""level:"30:,30"hostname":","time":"ip-10-150-1-74.us-west-2.compute.internal","2023-09-22T23:14:27.616985812Zpid"":,4301"hostname}"
:,,""{timetime""::""""msg":"2023-09-22T23:14:27.616991149Z2023-09-22T23:14:27.61699255Z"",[1] Starts cmd_loopip-10-150-1-74.us-west-2.compute.internal""",,,""pidv""::43010hostname,}"
name"""hostname:""::{""ip-10-150-1-74.us-west-2.compute.internal""msg,ip-10-150-1-74.us-west-2.compute.internal"","crucible:"","pid"level[2]R  generation: [0, 0]"":pid""::43014301,30"}}

{v":0"{msg":",,""msg"":nametime""::"[1] received reconcile message""crucible""[2]R  generation: [0, 0],"",v"":v0",":name02023-09-22T23:14:27.617037647Z,"",level"":",:""30hostname"cruciblename""::,"""levelcrucible"":,30"level"ip-10-150-1-74.us-west-2.compute.internal",:",time""pid:"":3043012023-09-22T23:14:27.617067608Z"},
","hostname{":time"""msg":,""ip-10-150-1-74.us-west-2.compute.internal:"","time2023-09-22T23:14:27.61707419Z"":",[2] received reconcile messagepid"",":v4301":}"0
,"2023-09-22T23:14:27.61708062Zhostname""{name",:""":msg"hostnameip-10-150-1-74.us-west-2.compute.internal"":,"":""pidip-10-150-1-74.us-west-2.compute.internal"":,4301"[2]R  dirty: [false, false]crucible""}pid
",,""levelv":"30:{:04301",}msg"
:""name":{,"""[1] All repairs completed, exitmsg"timecrucible"":,"",""2023-09-22T23:14:27.617127648Zlevel"",:"30v:"":hostname"[2]R  dirty: [false, false]0",,""namev""::0","crucible"name,"":level"",:""crucible:"30,"ip-10-150-1-74.us-west-2.compute.internaltime""level":,"":pid"2023-09-22T23:14:27.617145429Z":,4301"30}hostname
",,"{:time""time:"":"""msg2023-09-22T23:14:27.61716812Z2023-09-22T23:14:27.61715974Z""ip-10-150-1-74.us-west-2.compute.internal",,"":"",hostnamehostname""::""[2] All repairs completed, exit""pid",:"4301vip-10-150-1-74.us-west-2.compute.internalip-10-150-1-74.us-west-2.compute.internal"",,""}"
:pidpid""::43014301{}}

{0",{""msg"namemsg""::""msg"Max found gen is 1crucible"""::"",,""Max found gen is 1[1] Starts cmd_loop"",,""vv""::00,,""name"name"::""vlevel""::030,cruciblecrucible"",,""levellevel""::3030"name":"crucible","level":30,"time":"2023-09-22T23:14:27.617231827Z","hostname":",,""timetime""::"",ip-10-150-1-74.us-west-2.compute.internal""time,"2023-09-22T23:14:27.617237509Z:""pid"2023-09-22T23:14:27.617237349Z""2023-09-22T23:14:27.617241223Z:"4301,,""},
hostname""hostname:""hostname{,""":msghostname"":"":ip-10-150-1-74.us-west-2.compute.internal:"","[2] Starts cmd_loop"ip-10-150-1-74.us-west-2.compute.internal,ip-10-150-1-74.us-west-2.compute.internal""v,"":pid0""",pid"ip-10-150-1-74.us-west-2.compute.internal":,4301"pid""::43014301name}"
}
{,}"
:"{msg{"""msg""cruciblemsg"",:""::""Generation requested: 1 >= found:1level""Generation requested: 1 >= found:1[2] received reconcile message"":,30"pid",,""vv""::00,,"":namename""::",v""time:"0:,""crucible"4301crucible"",2023-09-22T23:14:27.617319144Z,"name",:"",hostname"crucible:""""level"level:",":3030"ip-10-150-1-74.us-west-2.compute.internallevel"",:"30pid"looper"::4301"2"}
,,""time,}time""::""
2023-09-22T23:14:27.617350055Z2023-09-22T23:14:27.617350508Z""",,time"""hostname"hostname"::"":{"ip-10-150-1-74.us-west-2.compute.internalip-10-150-1-74.us-west-2.compute.internal"",,"2023-09-22T23:14:27.617353803Z""pid,"pidmsg""::43014301"}}

{":""{msg"up_listen starts"hostname"v,"::"0":msg""ip-10-150-1-74.us-west-2.compute.internal,"","name""pid:"v:"4301crucible"}"
,[2] All repairs completed, exit:"",{"Next flush: 1""":msglevel0""::"30,,v""v:"0:,"0name,"":name""Next flush: 1""name"crucible:,,""timev""::"0",2023-09-22T23:14:27.617427059Z"crucible":"","crucible"level","",":,hostnamename""::""level""30level"ip-10-150-1-74.us-west-2.compute.internalcrucible":",,""pidlevel""::43013030},:"30time"
:"{2023-09-22T23:14:27.617460122Z",""time",,""msg:"":"hostnametime""::""2023-09-22T23:14:27.617467667Z"64be6ec7-0ac5-4896-883a-00ec34140b84 Active Active Active",",ip-10-150-1-74.us-west-2.compute.internal2023-09-22T23:14:27.617470694Z"","hostnamev""::"0,",,"ip-10-150-1-74.us-west-2.compute.internalnamepid"""hostname:"4301time":}"
ip-10-150-1-74.us-west-2.compute.internal":{,""",:""pid""crucible:"4301msg},
"2023-09-22T23:14:27.617469085Zlevel{""":,"30msg":""":"pid":All extents match"4301[2] Starts cmd_loop,}"
,""{vhostname""msg:v,"":time0""0:",,"2023-09-22T23:14:27.617524574Z""name":",crucible""hostname,"::""level"":ip-10-150-1-74.us-west-2.compute.internal30":""name,"ip-10-150-1-74.us-west-2.compute.internal"pid",All extents match"":"",crucible""v,""level:"0:,pid"30":,4301"name"::"time}"
:",crucible43012023-09-22T23:14:27.617557626Z{""",time""level:""",msg""2023-09-22T23:14:27.617572724Z:"30,,hostname:"":"""Set check for repair"ip-10-150-1-74.us-west-2.compute.internal"task":",hostname""up_listen",,}""vpid""::04301,:time"":
ip-10-150-1-74.us-west-2.compute.internal""}"
name":"{2023-09-22T23:14:27.617592613Z,""{"cruciblemsg"",:""level""pid,"":msg"No downstairs repair required:":30"4301hostname"}:
","Wait for all three downstairs to come online"v"ip-10-150-1-74.us-west-2.compute.internal,:"0time,"":"name"",,"2023-09-22T23:14:27.617637861Z:""v,crucible"""hostname"",:":level"pid""ip-10-150-1-74.us-west-2.compute.internal:"30,0":pid"4301:,}"4301name"
:,}"
time":""{{crucible"2023-09-22T23:14:27.617665866Z"",",msg"":hostname"""msg:[2] 127.0.0.1:60582 task reports connection:true""level""ip-10-150-1-74.us-west-2.compute.internal,"",v""::pid:"30:04301"},
"No downstairs repair requiredname{"":"","msgcrucible"":,""v"levelNo initial repair work was required"":,30":v0,",""name":":time"crucible:"0,,""name"time:"2023-09-22T23:14:27.61769915Z":"""crucible"2023-09-22T23:14:27.61771882Z,"",level,"":,""level"30hostname":hostname30":":"ip-10-150-1-74.us-west-2.compute.internal"ip-10-150-1-74.us-west-2.compute.internal",",",pid"":time4301"pid"}:
":43012023-09-22T23:14:27.61774417Z{","hostname}"":
,msg""":"time":ip-10-150-1-74.us-west-2.compute.internal""64be6ec7-0ac5-4896-883a-00ec34140b84 Active Active Active,"",pid""v:":430102023-09-22T23:14:27.617747001Z"{},,"
name"":{hostname""msg":"""msg"crucible:":",""ip-10-150-1-74.us-west-2.compute.internalFlush timeout: 0.5"Set Downstairs and Upstairs activelevel""":,"30v,,""pid":"v:"0:4301,,""name0}time""::"",crucible"2023-09-22T23:14:27.617801903Z
""name,,"""level{:"hostname:"30:"""crucible"msg"ip-10-150-1-74.us-west-2.compute.internal":,""pid","No initial repair work was required",:"4301,"time}"
:v"level"{"::"2023-09-22T23:14:27.617829466Zmsg""30,:""0hostnameSet check for repair"":",",vip-10-150-1-74.us-west-2.compute.internal"",:"0pid,"":name4301":}"
crucible"",{"name"level""msg:"30::""crucible",","time":level5b98377f-d3ca-45bf-893d-9aae2fd5a48e is now active with session: d3623aa7-ee8c-40f8-a982-d0eadbcd4a76""":,,""30timev""::"02023-09-22T23:14:27.617852654Z",2023-09-22T23:14:27.617871046Z""name,,"":hostname"":crucible""","ip-10-150-1-74.us-west-2.compute.internal"level,""pid:"30:hostname"4301:"}
ip-10-150-1-74.us-west-2.compute.internal",{","time"",msg""time:"":"pid"[0] received reconcile message:2023-09-22T23:14:27.617882211Z4301":,"}""v",2023-09-22T23:14:27.617894311Z:
0",",""name"hostname:"{hostname":"":crucible"ip-10-150-1-74.us-west-2.compute.internal"","ip-10-150-1-74.us-west-2.compute.internal,"""pidlevel,msg"""::430130:""}pid"
9fe7fa85-122e-4e82-8591-96595eb36a20 active request set":{4301,",msg}"":""time":v"5b98377f-d3ca-45bf-893d-9aae2fd5a48e Set Active after no repair""
:2023-09-22T23:14:27.617942182Z,""0v,"":{hostname0",:""name","ip-10-150-1-74.us-west-2.compute.internal:"",msg"crucible""pid,"":level4301""}:
30name":{:"""crucible"msg":,"Set Downstairs and Upstairs active""[0] All repairs completed, exitlevel",:",,""time"v:"":30"2023-09-22T23:14:27.617984009Z0",",name""v"hostname:"":"crucible":,ip-10-150-1-74.us-west-2.compute.internal""0,level""pid:"30:,4301"}name"
:"crucible",,",{"timetime"":msg""":2023-09-22T23:14:27.618012776Z"":,""Notify all downstairs, region set compare is done."level,"""hostnamev""::"02023-09-22T23:14:27.618000987Z,:"ip-10-150-1-74.us-west-2.compute.internalname""",:""pid"crucible":,,""4301level"hostname30:}30
":"{ip-10-150-1-74.us-west-2.compute.internal"",msg"":"pid",[0] Starts cmd_loop""time",:":v""4301:02023-09-22T23:14:27.618051813Z",",name"":hostname,}""":crucible"
ip-10-150-1-74.us-west-2.compute.internal"",,""pid"level:"4301{}:
30"time"{:msg"":msg""":"2023-09-22T23:14:27.61805108Z[0] 9fe7fa85-122e-4e82-8591-96595eb36a20 looper connected"Set check for repair,"""time,,",:""v""hostnamev"":2023-09-22T23:14:27.618086866Z":"0:0,,",""name"hostname:name"":""":crucible""crucible,"ip-10-150-1-74.us-west-2.compute.internal"",level,"":"ip-10-150-1-74.us-west-2.compute.internallevel""30pid",:"30:pid"4301:}4301
,"}time{":"
"2023-09-22T23:14:27.618127393Z"msg":,""hostname":"[1] received reconcile message","v"ip-10-150-1-74.us-west-2.compute.internal,:"0,,""namepid""::4301"{crucible}"
,""{"time""levelmsg""::30"msg":":"[1] 127.0.0.1:46381 task reports connection:true","v":2023-09-22T23:14:27.618130906Z"066688a47-67b1-4e47-8e76-e0e145532b3a is now active with session: ff839372-f996-4bbe-90dd-222e8d070509",,,",""nametime""::"""crucible"2023-09-22T23:14:27.618165479Z,""hostname,level"""hostname:"30:":v"":ip-10-150-1-74.us-west-2.compute.internal"0,"ip-10-150-1-74.us-west-2.compute.internalpid",""name",":,4301"pid":}time
"::"4301"{2023-09-22T23:14:27.618185912Z",,""msg"hostname:"":""crucible"looper"ip-10-150-1-74.us-west-2.compute.internal[1] All repairs completed, exit"",,""pidv":"4301:,0},
":name{"""0":msg"":crucible""","level"5b98377f-d3ca-45bf-893d-9aae2fd5a48e Active Active Activelevel"",:}30"
v"::030,"name"{:,""timecrucible"":","level":"302023-09-22T23:14:27.618242508Z"msg",":"hostname":"[0] Proc runs for 127.0.0.1:58086 in state New"ip-10-150-1-74.us-west-2.compute.internal",,""v"pid,""::4301time0"}:
",",2023-09-22T23:14:27.618256242Z{"name,":"""hostname"msg:"":"crucible"time",":ip-10-150-1-74.us-west-2.compute.internal""",[1] Starts cmd_loop""2023-09-22T23:14:27.618251334Zlevel"",pid""v:,:"304301":}0
,"hostname"{name":"":"msg":crucible"",ip-10-150-1-74.us-west-2.compute.internal"Set check for repair",""level,""v:"30:pid"0:,"4301name":",}"
time"crucible,"",time"":level""::"{302023-09-22T23:14:27.618309896Z"2023-09-22T23:14:27.618295408Z,"""hostname"msg,":hostname"",:""ip-10-150-1-74.us-west-2.compute.internaltime"",:""pid"ip-10-150-1-74.us-west-2.compute.internal:2023-09-22T23:14:27.618328232Z4301"}""
,","hostname{"pid":""msg:"ip-10-150-1-74.us-west-2.compute.internal:"",":[2] received reconcile messagepid"",:"4301v"}:
04301,"{"}name""msg:"":"crucible66688a47-67b1-4e47-8e76-e0e145532b3a Set Active after no repair""
,[2] 127.0.0.1:49774 task reports connection:true"",",level""v:{"30:v"0:,"0msg"":name":""crucible,"",,"time"level:"":[1] 9fe7fa85-122e-4e82-8591-96595eb36a20 looper connected302023-09-22T23:14:27.618383175Z"""name",:","crucible"v",:"0level,","""timehostname""::"":name"2023-09-22T23:14:27.618398921Z"ip-10-150-1-74.us-west-2.compute.internal:","",crucible"hostname""pid:30,"":"4301level"}ip-10-150-1-74.us-west-2.compute.internal
":,"30pid{":4301}"
msg":"{[2] All repairs completed, exit"","msg"v:"":0,"name":",crucible5b98377f-d3ca-45bf-893d-9aae2fd5a48e Active Active Active"",",""time"vlevel""::030,:""name"2023-09-22T23:14:27.61842509Z:","crucible,""time,"":,level""":2023-09-22T23:14:27.618450533Z30"time,""":"hostname":"hostname"2023-09-22T23:14:27.618434288Z":ip-10-150-1-74.us-west-2.compute.internal"",,",ip-10-150-1-74.us-west-2.compute.internal""time",:""pid""pid"hostname":2023-09-22T23:14:27.61846684Z:"4301,":}"
hostname":"ip-10-150-1-74.us-west-2.compute.internal"{ip-10-150-1-74.us-west-2.compute.internal"4301,,""msgpid""}":pid
""::4301[2] Starts cmd_loop"}4301,
",v{"{"":looper"0msg,""::""name"msg"1":Set check for repair"":},
""v":crucible0",,""{levelname""::30"""crucible"msg",":"level"Notify all downstairs, region set compare is done.":[1] Proc runs for 127.0.0.1:50371 in state New"30,,",""vtimev"":":"02023-09-22T23:14:27.61853807Z":,,"",0","name"nametimehostname""::"":""ip-10-150-1-74.us-west-2.compute.internal2023-09-22T23:14:27.618547314Zcrucible":,"",,"""hostnamepid""::"4301crucible""ip-10-150-1-74.us-west-2.compute.internal}level""
,:,""pid"level30"::430130}
,"time":",2023-09-22T23:14:27.618589483Z""time,"":"hostname":"2023-09-22T23:14:27.618591814Z"ip-10-150-1-74.us-west-2.compute.internal",","hostnamepid":"4301:"}
ip-10-150-1-74.us-west-2.compute.internal","pid":4301{}
"msg":"{[2] 9fe7fa85-122e-4e82-8591-96595eb36a20 looper connected","v":0","msg":name"":"crucible"Set check for repair,""level":,"30v":0,"name":"crucible","level"Sep 22 23:14:27.615 INFO UpstairsConnection { upstairs_id: 9fe7fa85-122e-4e82-8591-96595eb36a20, session_id: 57e450d1-3a66-4c19-9890-8b892a800667, gen: 1 } is now active (read-write)
:30,"time":"2023-09-22T23:14:27.618652533Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"2","}time"
:"2023-09-22T23:14:27.618663302Z","{hostname":"ip-10-150-1-74.us-west-2.compute.internal"","msg":"pid":4301[2] Proc runs for 127.0.0.1:54983 in state New}"
,"v":0,"name":"{crucible","level":30"msg":"[1] 127.0.0.1:50563 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.618709274Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"}time
":"2023-09-22T23:14:27.61871864Z"{,"hostname":"ip-10-150-1-74.us-west-2.compute.internal"","msg":"pid":4301}
[0] 9fe7fa85-122e-4e82-8591-96595eb36a20 (57e450d1-3a66-4c19-9890-8b892a800667) New New New ds_transition to WaitActive","v":0,"{name":"crucible",""level"msg:":30"66688a47-67b1-4e47-8e76-e0e145532b3a Active Active Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.618763792Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"}time
":"2023-09-22T23:14:27.618772441Z","{hostname":"ip-10-150-1-74.us-west-2.compute.internal"","msg"pid"::"4301}[0] Transition from New to WaitActive
","v":0,"{name":"crucible","level"":msg"30:"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.618825772Z","hostname":","ip-10-150-1-74.us-west-2.compute.internal"time":","pid":43012023-09-22T23:14:27.618832895Z",}"
hostname":"ip-10-150-1-74.us-west-2.compute.internal"{,"pid":4301}
"msg":"{[0] client is_active_req TRUE, promote! session 57e450d1-3a66-4c19-9890-8b892a800667""msg,"":v"":0,"name":"[2] 127.0.0.1:39722 task reports connection:true"crucible",",v"":level0",":name":30"crucible","level":30,,""time":time"":"2023-09-22T23:14:27.618889904Z"2023-09-22T23:14:27.618887113Z",","hostname"hostname:"":"ip-10-150-1-74.us-west-2.compute.internal","ip-10-150-1-74.us-west-2.compute.internalpid"":,4301"pid":}4301
}
{"msg":{"66688a47-67b1-4e47-8e76-e0e145532b3a Active Active Active"","msg"v":":0,"name":"crucible","level":30[1] 9fe7fa85-122e-4e82-8591-96595eb36a20 (57e450d1-3a66-4c19-9890-8b892a800667) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.618944231Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid,"":time4301":"}
2023-09-22T23:14:27.618950903Z","hostname":"{ip-10-150-1-74.us-west-2.compute.internal",""pid"msg"::4301"Set check for repair"}
,"v":0,"name":"{crucible","level":30"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.618996492Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal",,""timepid""::4301"}
2023-09-22T23:14:27.619005322Z","hostname":"{ip-10-150-1-74.us-west-2.compute.internal","pid"":msg"4301:"}
[0] received reconcile message","v":{0,"name":""crucible"msg,"":"level":30[1] client is_active_req TRUE, promote! session 57e450d1-3a66-4c19-9890-8b892a800667","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.61905219Z",","time"hostname"::""2023-09-22T23:14:27.619060622Z"ip-10-150-1-74.us-west-2.compute.internal",","hostname"pid"::"4301}
ip-10-150-1-74.us-west-2.compute.internal","pid":4301}{
"msg":"{[0] All repairs completed, exit""msg",:""v":0,"name":"crucible","level":30[2] 9fe7fa85-122e-4e82-8591-96595eb36a20 (57e450d1-3a66-4c19-9890-8b892a800667) WaitActive WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time",:""time":"2023-09-22T23:14:27.619105323Z"2023-09-22T23:14:27.619112035Z",","hostname":"hostname":"ip-10-150-1-74.us-west-2.compute.internal","ip-10-150-1-74.us-west-2.compute.internal"pid,"":pid"4301:4301}}

{{""msg"msg"::""[0] Starts cmd_loop","[2] Transition from New to WaitActive"v",":v"0:,"0name",:""name":crucible"","crucible"level",:"30level":30,"time":"2023-09-22T23:14:27.619168608Z","hostname":","ip-10-150-1-74.us-west-2.compute.internal"time,"":pid"":43012023-09-22T23:14:27.619167845Z"}
,"hostname":"{"ip-10-150-1-74.us-west-2.compute.internal"msg":,""pid":4301}
[2] client is_active_req TRUE, promote! session 57e450d1-3a66-4c19-9890-8b892a800667","v":0,"name"{:"crucible","level":"30msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.619214451Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{,"time"":msg":""2023-09-22T23:14:27.619222659Z","hostname":"[0] downstairs client at 127.0.0.1:58086 has UUID 97377133-8bb1-47f5-a2f0-3b28d74593db","v"ip-10-150-1-74.us-west-2.compute.internal":,0",pid"":name"4301:"crucible"},
"level":30{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.619262179Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{,"time":""msg":"2023-09-22T23:14:27.619274625Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
[0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 97377133-8bb1-47f5-a2f0-3b28d74593db, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,{"Sep 22 23:14:27.619 DEBG IO Write 1000 has deps []
"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.619335966Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.619379831Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.619420424Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[2] Starts cmd_loop","v":Sep 22 23:14:27.619 DEBG up_ds_listen was notified
0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.619464587Z","hostname":"Sep 22 23:14:27.619 DEBG up_ds_listen process 1000
ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
Sep 22 23:14:27.619 DEBG [A] ack job 1000:1, : downstairs
{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30Sep 22 23:14:27.619 DEBG up_ds_listen checked 1 jobs, back to waiting
,"time":"2023-09-22T23:14:27.619558376Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30The guest has finished waiting for activation
,"time":"2023-09-22T23:14:27.619604231Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30Sep 22 23:14:27.619 INFO Current flush_numbers [0..12]: [0, 0]
,"time":"2023-09-22T23:14:27.619642727Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.619681847Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.619712834Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.619747488Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.619778949Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.619810117Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.61984594Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301Sep 22 23:14:27.619 INFO Downstairs has completed Negotiation, task: proc
}
name":"crucible","level":30,"time":"2023-09-22T23:14:27.619920383Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"9fe7fa85-122e-4e82-8591-96595eb36a20 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.619959926Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[1] downstairs client at 127.0.0.1:50371 has UUID 53cd4d63-78de-43d1-b2db-bd20d98528e7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.619994514Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 53cd4d63-78de-43d1-b2db-bd20d98528e7, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.620035044Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"9fe7fa85-122e-4e82-8591-96595eb36a20 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.620067867Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"Sep 22 23:14:27.620 INFO Current flush_numbers [0..12]: [0, 0]
[2] downstairs client at 127.0.0.1:54983 has UUID bb88743f-4e54-484f-8e97-8fa2d4bde987","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.62011642Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: bb88743f-4e54-484f-8e97-8fa2d4bde987, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.62016754Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"9fe7fa85-122e-4e82-8591-96595eb36a20 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.62021058Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
Sep 22 23:14:27.620 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.620 INFO Connection request from 1f7159d2-76c3-4968-a4c3-9cc9b7758094 with version 4, task: proc
Sep 22 23:14:27.620 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.620 INFO upstairs UpstairsConnection { upstairs_id: 1f7159d2-76c3-4968-a4c3-9cc9b7758094, session_id: 1b3283bb-b7f5-4e0f-94bf-8ec6efc0c054, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.620 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.620 INFO Connection request from 1f7159d2-76c3-4968-a4c3-9cc9b7758094 with version 4, task: proc
Sep 22 23:14:27.620 INFO upstairs UpstairsConnection { upstairs_id: 1f7159d2-76c3-4968-a4c3-9cc9b7758094, session_id: 1b3283bb-b7f5-4e0f-94bf-8ec6efc0c054, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.620 INFO Created new region file "/tmp/downstairs-zdyp8uYh/region.json"
Sep 22 23:14:27.620 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.620 INFO Connection request from 1f7159d2-76c3-4968-a4c3-9cc9b7758094 with version 4, task: proc
Sep 22 23:14:27.620 INFO upstairs UpstairsConnection { upstairs_id: 1f7159d2-76c3-4968-a4c3-9cc9b7758094, session_id: 1b3283bb-b7f5-4e0f-94bf-8ec6efc0c054, gen: 1 } connected, version 4, task: proc
{{""msgmsg""::""[0] 1f7159d2-76c3-4968-a4c3-9cc9b7758094 (1b3283bb-b7f5-4e0f-94bf-8ec6efc0c054) New New New ds_transition to WaitActive"[0] 9fe7fa85-122e-4e82-8591-96595eb36a20 (57e450d1-3a66-4c19-9890-8b892a800667) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v",:"v":00,,""namename""::""cruciblecrucible"",,""levellevel""::3030,,""timetime""::""2023-09-22T23:14:27.620963142Z2023-09-22T23:14:27.620963186Z"",,""hostnamehostname""::""ip-10-150-1-74.us-west-2.compute.internalip-10-150-1-74.us-west-2.compute.internal"",,""pidpid""::43014301}}

{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.621015611Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:27.621050063Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.62108079Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[1] 9fe7fa85-122e-4e82-8591-96595eb36a20 (57e450d1-3a66-4c19-9890-8b892a800667) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.621115637Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.621146448Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal"Sep 22 23:14:27.621 DEBG Read      :1000 deps:[] res:true
,"pid":4301}
{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:27.621183616Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.621219355Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[2] 9fe7fa85-122e-4e82-8591-96595eb36a20 (57e450d1-3a66-4c19-9890-8b892a800667) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.621254493Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.621285197Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:27.621314714Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.621344587Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[0] 127.0.0.1:58086 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.621375092Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"9fe7fa85-122e-4e82-8591-96595eb36a20 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.621406778Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[0]R  flush_numbers: [0, 0]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.621436689Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[0]R  generation: [0, 0]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.621466111Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[0]R  dirty: [false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.621495811Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[1]R  flush_numbers: [0, 0]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.621525333Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[1]R  generation: [0, 0]","vSep 22 23:14:27.621 DEBG Read      :1000 deps:[] res:true
":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.621561778Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[1]R  dirty: [false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.621591798Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":{4301}
"msg":"{"[0] Transition from New to WaitActive"msg":","v":0[2]R  flush_numbers: [0, 0]",","name"v:"":0crucible,"",name"":"level":crucible"30,"level":30,,""timetime""::""2023-09-22T23:14:27.621622349Z"2023-09-22T23:14:27.621621209Z",","hostname"hostname:"":"ip-10-150-1-74.us-west-2.compute.internal"ip-10-150-1-74.us-west-2.compute.internal",","pid"pid:":43014301}}

{"{msg":""msg"[2]R  generation: [0, 0]:"","v":0,"name":"crucible"[0] client is_active_req TRUE, promote! session 1b3283bb-b7f5-4e0f-94bf-8ec6efc0c054,"",level"":v"30:0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.621656998Z",","time":hostname"":"2023-09-22T23:14:27.621660769Z","ip-10-150-1-74.us-west-2.compute.internal",hostname"":pid"":4301}
ip-10-150-1-74.us-west-2.compute.internal","pid":4301{}
"msg":"[2]R  dirty: [false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.621691078Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"Max found gen is 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.62172218Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.621758289Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"Next flush: 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.621794121Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"All extents match","v":0,"name":"crucible","level":The guest has finished waiting for activation
30,"time":"2023-09-22T23:14:27.621843699Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.621884899Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.621923489Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.621962334Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"9fe7fa85-122e-4e82-8591-96595eb36a20 is now active with session: 57e450d1-3a66-4c19-9890-8b892a800667","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.622004209Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"9fe7fa85-122e-4e82-8591-96595eb36a20 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.622044751Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.622085002Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
Sep 22 23:14:27.622 DEBG Read      :1000 deps:[] res:true
{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.622121676Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[1] 127.0.0.1:50371 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.622159338Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"9fe7fa85-122e-4e82-8591-96595eb36a20 Active Active Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.622191338Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.622211621Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[2] 127.0.0.1:54983 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.622230868Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"9fe7fa85-122e-4e82-8591-96595eb36a20 Active Active Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.622265626Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{{"msg":""Set check for repairmsg"":","v":0,"name":"crucible","level":[1] 1f7159d2-76c3-4968-a4c3-9cc9b7758094 (1b3283bb-b7f5-4e0f-94bf-8ec6efc0c054) WaitActive New New ds_transition to WaitActive30","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.622299554Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"}time
":"2023-09-22T23:14:27.622305761Z"{,"hostname"":msg"":"[0] received reconcile messageip-10-150-1-74.us-west-2.compute.internal"",,""pidv""::43010,"name}":
"crucible","level":{30"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.622337892Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301},
"time":"{2023-09-22T23:14:27.622347015Z"","msg":hostname"":"[0] All repairs completed, exit"ip-10-150-1-74.us-west-2.compute.internal,"",v"":pid"0:,4301"name"}:"
crucible","level"{:30"msg":"[1] client is_active_req TRUE, promote! session 1b3283bb-b7f5-4e0f-94bf-8ec6efc0c054","v":0,"name":"crucible",,""leveltime""::"302023-09-22T23:14:27.622372098Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
,"time":"{2023-09-22T23:14:27.622383084Z""msg,"":"hostname":"[0] Starts cmd_loop","v":ip-10-150-1-74.us-west-2.compute.internal"0,,""pid"name:":4301"}crucible
","level":30,"time":"2023-09-22T23:14:27.62240594Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.622443146Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.622480455Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.62251696Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.622555392Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.622595617Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.622629306Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[2] 1f7159d2-76c3-4968-a4c3-9cc9b7758094 (1b3283bb-b7f5-4e0f-94bf-8ec6efc0c054) WaitActive WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.622762114Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.622809468Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[2] client is_active_req TRUE, promote! session 1b3283bb-b7f5-4e0f-94bf-8ec6efc0c054","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.622852153Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
Sep 22 23:14:27.622 INFO UpstairsConnection { upstairs_id: 1f7159d2-76c3-4968-a4c3-9cc9b7758094, session_id: 1b3283bb-b7f5-4e0f-94bf-8ec6efc0c054, gen: 1 } is now active (read-write)
Sep 22 23:14:27.623 INFO UpstairsConnection { upstairs_id: 1f7159d2-76c3-4968-a4c3-9cc9b7758094, session_id: 1b3283bb-b7f5-4e0f-94bf-8ec6efc0c054, gen: 1 } is now active (read-write)
Sep 22 23:14:27.623 INFO UpstairsConnection { upstairs_id: 1f7159d2-76c3-4968-a4c3-9cc9b7758094, session_id: 1b3283bb-b7f5-4e0f-94bf-8ec6efc0c054, gen: 1 } is now active (read-write)
Sep 22 23:14:27.623 DEBG Read      :1002 deps:[JobId(1001), JobId(1000)] res:true
Sep 22 23:14:27.623 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.623 DEBG Read      :1002 deps:[JobId(1001), JobId(1000)] res:true
Sep 22 23:14:27.623 INFO Opened existing region file "/tmp/downstairs-zdyp8uYh/region.json"
Sep 22 23:14:27.623 INFO Database read version 1
Sep 22 23:14:27.623 INFO Database write version 1
{"msg":"[0] downstairs client at 127.0.0.1:33768 has UUID daca29a6-1615-4360-b286-c9a82fbfcd3d","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.623648747Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301Sep 22 23:14:27.623 DEBG Read      :1002 deps:[JobId(1001), JobId(1000)] res:true
}
{"msg":"[0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: daca29a6-1615-4360-b286-c9a82fbfcd3d, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.623713827Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"1f7159d2-76c3-4968-a4c3-9cc9b7758094 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.623758766Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{Sep 22 23:14:27.623 DEBG Read      :1002 deps:[JobId(1001), JobId(1000)] res:true
"msg":"[1] downstairs client at 127.0.0.1:54745 has UUID 80932238-df83-4f6e-96fc-9d0aac0f8ced","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.623809002Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 80932238-df83-4f6e-96fc-9d0aac0f8ced, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.623859438Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"1f7159d2-76c3-4968-a4c3-9cc9b7758094 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.623908198Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[2] downstairs client at 127.0.0.1:56258 has UUID 67caa349-8834-4aba-8026-7fdaaae2f119","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.623952128Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 67caa349-8834-4aba-8026-7fdaaae2f119, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.624001484Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"1f7159d2-76c3-4968-a4c3-9cc9b7758094 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.624043887Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal",Sep 22 23:14:27.624 DEBG Read      :1002 deps:[JobId(1001), JobId(1000)] res:true
"pid":4301}
Sep 22 23:14:27.624 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.624 DEBG Read      :1002 deps:[JobId(1001), JobId(1000)] res:true
Sep 22 23:14:27.624 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.624 DEBG Read      :1002 deps:[JobId(1001), JobId(1000)] res:true
Sep 22 23:14:27.624 DEBG Read      :1002 deps:[JobId(1001), JobId(1000)] res:true
Sep 22 23:14:27.624 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.624 DEBG Read      :1002 deps:[JobId(1001), JobId(1000)] res:true
Sep 22 23:14:27.624 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.624 DEBG Read      :1002 deps:[JobId(1001), JobId(1000)] res:true
Sep 22 23:14:27.624 DEBG Read      :1002 deps:[JobId(1001), JobId(1000)] res:true
Sep 22 23:14:27.624 INFO UUID: ae376576-6754-4185-9b1d-d23a94866419
Sep 22 23:14:27.624 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.624 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.624 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.625 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.625 INFO Using address: 127.0.0.1:62845, task: main
Sep 22 23:14:27.625 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.625 DEBG Read      :1002 deps:[JobId(1001), JobId(1000)] res:true
{"msg":"[0] 1f7159d2-76c3-4968-a4c3-9cc9b7758094 (1b3283bb-b7f5-4e0f-94bf-8ec6efc0c054) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.625351544Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.625400489Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:27.625441909Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[0] Starts reconcile loop","v"Sep 22 23:14:27.625 INFO Repair listens on 127.0.0.1:0, task: repair
:0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.625488313Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[1] 1f7159d2-76c3-4968-a4c3-9cc9b7758094 (1b3283bb-b7f5-4e0f-94bf-8ec6efc0c054) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.625534912Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.625583553Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[1] new RM replaced this: None","v":0,"name":"Sep 22 23:14:27.625 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:51597, task: repair
crucible","level":40,"time":"2023-09-22T23:14:27.625632586Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","levelSep 22 23:14:27.625 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:51597, task: repair
":30,"time":"2023-09-22T23:14:27.625679187Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[2] 1f7159d2-76c3-4968-a4c3-9cc9b7758094 (1b3283bb-b7f5-4e0f-94bf-8ec6efc0c054) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorumSep 22 23:14:27.625 INFO listening, local_addr: 127.0.0.1:51597, task: repair
","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.625729113Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.625770593Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:27.625810381Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.625849072Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[0] 127.0.0.1:33768 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.625889238Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"1f7159d2-76c3-4968-a4c3-9cc9b7758094 WaitQuorum WaitQuorum WaitQuorum","Sep 22 23:14:27.625 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:51597, task: repair
v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.625937464Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[0]R  flush_numbers: [0, 0]","v":0,"name":"crucible","level":30,"time":"Sep 22 23:14:27.625 INFO Using repair address: 127.0.0.1:51597, task: main
2023-09-22T23:14:27.62597782Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{Sep 22 23:14:27.626 INFO No SSL acceptor configured, task: main
"msg":"[0]R  generation: [0, 0]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.626027222Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[0]R  dirty: [false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.626068524Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[1]R  flush_numbers: [0, 0]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.626108801Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[1]R  generation: [0, 0]"The guest has finished waiting for activation
,"v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.626151501Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[1]R  dirty: [false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.626191565Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[2]R  flush_numbers: [0, 0]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.626230413Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[2]R  generation: [0, 0]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.62626914Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[2]R  dirty: [false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.626309384Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"Max found gen is 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.626355115Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.626395714Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"Next flush: 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.626435838Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.626475665Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.626510831Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.626543126Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.626574186Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"1f7159d2-76c3-4968-a4c3-9cc9b7758094 is now active with session: 1b3283bb-b7f5-4e0f-94bf-8ec6efc0c054","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.626608212Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"1f7159d2-76c3-4968-a4c3-9cc9b7758094 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.626640924Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.626672674Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.626702994Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[1] 127.0.0.1:54745 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.62673416Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"1f7159d2-76c3-4968-a4c3-9cc9b7758094 Active Active Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.626771461Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.626808434Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[2] 127.0.0.1:56258 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.626847814Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"1f7159d2-76c3-4968-a4c3-9cc9b7758094 Active Active Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.626888393Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.626925718Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.626961604Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.627000873Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.627040029Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.627078133Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.627118473Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.627161823Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.627201754Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.627240987Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.627275795Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
Sep 22 23:14:27.627 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.627 WARN a50fc3fb-3de2-4743-9c50-cc80cfba77db request to replace downstairs 127.0.0.1:42970 with 127.0.0.1:62845
Sep 22 23:14:27.627 INFO a50fc3fb-3de2-4743-9c50-cc80cfba77db found old target: 127.0.0.1:42970 at 0
Sep 22 23:14:27.627 INFO a50fc3fb-3de2-4743-9c50-cc80cfba77db replacing old: 127.0.0.1:42970 at 0
Sep 22 23:14:27.628 INFO [0] client skip 1 in process jobs because fault, : downstairs
Sep 22 23:14:27.628 INFO [0] changed 1 jobs to fault skipped, : downstairs
Sep 22 23:14:27.628 INFO [0] a50fc3fb-3de2-4743-9c50-cc80cfba77db (2b698136-bbd2-4fcb-ab7c-5f19b2267eb8) Active Active Active ds_transition to Replacing
Sep 22 23:14:27.628 INFO [0] Transition from Active to Replacing
Sep 22 23:14:27.629 DEBG Write     :1000 deps:[] res:true
test test::integration_test_guest_downstairs_unwritten_sparse_mid ... ok
test test::integration_test_guest_downstairs_unwritten_span_2 ... ok
test test::integration_test_guest_downstairs_unwritten_sparse_end ... ok
Sep 22 23:14:27.634 DEBG Write     :1000 deps:[] res:true
Sep 22 23:14:27.634 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.634 DEBG Write     :1001 deps:[JobId(1000)] res:true
Sep 22 23:14:27.634 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.634 INFO Created new region file "/tmp/downstairs-ZvKZ4Jxx/region.json"
Sep 22 23:14:27.634 INFO Created new region file "/tmp/downstairs-uHYeHXUA/region.json"
Sep 22 23:14:27.634 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.634 INFO Created new region file "/tmp/downstairs-RigWqboV/region.json"
test test::integration_test_guest_downstairs_unwritten_span ... Sep 22 23:14:27.634 DEBG Write     :1000 deps:[] res:true
ok
Sep 22 23:14:27.635 DEBG Write     :1001 deps:[JobId(1000)] res:true
Sep 22 23:14:27.635 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.635 DEBG Read      :1002 deps:[JobId(1001), JobId(1000)] res:true
Sep 22 23:14:27.635 WARN [0] a50fc3fb-3de2-4743-9c50-cc80cfba77db WARNING finish job 1000 when downstairs state:Replacing
Sep 22 23:14:27.635 WARN [0] Dropping already skipped job 1000, : downstairs
Sep 22 23:14:27.635 INFO Created new region file "/tmp/downstairs-k0LAAe9c/region.json"
Sep 22 23:14:27.635 WARN [0] will exit pm_task, this downstairs Replacing
Sep 22 23:14:27.635 DEBG up_ds_listen was notified
Sep 22 23:14:27.635 DEBG Read      :1002 deps:[JobId(1001), JobId(1000)] res:true
Sep 22 23:14:27.635 DEBG up_ds_listen checked 0 jobs, back to waiting
Sep 22 23:14:27.635 ERRO 127.0.0.1:42970: proc: [0] client work task ended, Ok(Err([0] This downstairs now in Replacing)), so we end too, looper: 0
Sep 22 23:14:27.635 DEBG Read      :1002 deps:[JobId(1001), JobId(1000)] res:true
Sep 22 23:14:27.635 INFO [0] a50fc3fb-3de2-4743-9c50-cc80cfba77db Gone missing, transition from Replacing to Replaced
Sep 22 23:14:27.635 INFO [0] a50fc3fb-3de2-4743-9c50-cc80cfba77db connection to 127.0.0.1:42970 closed, looper: 0
Sep 22 23:14:27.635 INFO [0] 127.0.0.1:42970 task reports connection:false
Sep 22 23:14:27.635 INFO a50fc3fb-3de2-4743-9c50-cc80cfba77db Replaced Active Active
Sep 22 23:14:27.635 INFO [0] 127.0.0.1:42970 task reports offline
Sep 22 23:14:27.635 WARN upstairs UpstairsConnection { upstairs_id: a50fc3fb-3de2-4743-9c50-cc80cfba77db, session_id: 2b698136-bbd2-4fcb-ab7c-5f19b2267eb8, gen: 1 } disconnected, 0 jobs left, task: main
Sep 22 23:14:27.635 DEBG Read      :1001 deps:[JobId(1000)] res:true
Sep 22 23:14:27.635 DEBG Write     :1001 deps:[JobId(1000)] res:true
Sep 22 23:14:27.635 WARN upstairs UpstairsConnection { upstairs_id: a50fc3fb-3de2-4743-9c50-cc80cfba77db, session_id: 2b698136-bbd2-4fcb-ab7c-5f19b2267eb8, gen: 1 } was previously active, clearing, task: main
Sep 22 23:14:27.636 INFO connection (127.0.0.1:40268): all done
Sep 22 23:14:27.636 DEBG Read      :1001 deps:[JobId(1000)] res:true
Sep 22 23:14:27.637 DEBG Read      :1002 deps:[JobId(1001)] res:true
Sep 22 23:14:27.637 DEBG Read      :1002 deps:[JobId(1001)] res:true
Sep 22 23:14:27.637 DEBG Read      :1002 deps:[JobId(1001)] res:true
Sep 22 23:14:27.637 DEBG Read      :1001 deps:[JobId(1000)] res:true
Sep 22 23:14:27.637 DEBG Read      :1003 deps:[JobId(1001)] res:true
Sep 22 23:14:27.638 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.638 INFO Opened existing region file "/tmp/downstairs-RigWqboV/region.json"
Sep 22 23:14:27.638 INFO Database read version 1
Sep 22 23:14:27.638 INFO Database write version 1
Sep 22 23:14:27.638 DEBG Read      :1003 deps:[JobId(1001)] res:true
Sep 22 23:14:27.638 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.638 INFO Opened existing region file "/tmp/downstairs-uHYeHXUA/region.json"
Sep 22 23:14:27.638 DEBG Read      :1003 deps:[JobId(1001)] res:true
Sep 22 23:14:27.638 INFO Database read version 1
Sep 22 23:14:27.638 INFO Database write version 1
Sep 22 23:14:27.638 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.638 INFO Opened existing region file "/tmp/downstairs-ZvKZ4Jxx/region.json"
Sep 22 23:14:27.638 INFO Database read version 1
Sep 22 23:14:27.638 INFO Database write version 1
Sep 22 23:14:27.638 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.638 INFO Opened existing region file "/tmp/downstairs-k0LAAe9c/region.json"
Sep 22 23:14:27.638 INFO Database read version 1
Sep 22 23:14:27.638 INFO Database write version 1
Sep 22 23:14:27.639 INFO UUID: ae16b993-7324-48a0-87ab-26e33b2b0247
Sep 22 23:14:27.639 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.639 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.639 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.639 INFO Using address: 127.0.0.1:47517, task: main
Sep 22 23:14:27.639 INFO UUID: 7300b28a-83f2-47df-a265-634df7adc55c
Sep 22 23:14:27.639 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.639 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.639 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.639 INFO UUID: 6f874cfa-c3be-49c7-9256-3b7afc29b96b
Sep 22 23:14:27.639 INFO Using address: 127.0.0.1:53663, task: main
Sep 22 23:14:27.639 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.639 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.639 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.639 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:51325, task: repair
Sep 22 23:14:27.639 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.639 INFO Using address: 127.0.0.1:36461, task: main
Sep 22 23:14:27.639 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:51325, task: repair
Sep 22 23:14:27.639 INFO listening, local_addr: 127.0.0.1:51325, task: repair
Sep 22 23:14:27.639 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.639 INFO UUID: 23321635-9e40-4165-b7e2-14f978a59fa9
Sep 22 23:14:27.639 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.639 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:51325, task: repair
Sep 22 23:14:27.639 INFO Using repair address: 127.0.0.1:51325, task: main
Sep 22 23:14:27.639 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.639 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.639 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.639 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:33584, task: repair
Sep 22 23:14:27.639 INFO Using address: 127.0.0.1:49699, task: main
Sep 22 23:14:27.639 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:33584, task: repair
Sep 22 23:14:27.639 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.640 INFO listening, local_addr: 127.0.0.1:33584, task: repair
Sep 22 23:14:27.640 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:45034, task: repair
Sep 22 23:14:27.640 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.640 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:45034, task: repair
Sep 22 23:14:27.640 INFO listening, local_addr: 127.0.0.1:45034, task: repair
Sep 22 23:14:27.640 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:33584, task: repair
Sep 22 23:14:27.640 INFO Created new region file "/tmp/downstairs-dsMmDgYS/region.json"
Sep 22 23:14:27.640 INFO Using repair address: 127.0.0.1:33584, task: main
Sep 22 23:14:27.640 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.640 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.640 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:45034, task: repair
Sep 22 23:14:27.640 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:38823, task: repair
Sep 22 23:14:27.640 INFO Using repair address: 127.0.0.1:45034, task: main
Sep 22 23:14:27.640 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.640 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:38823, task: repair
Sep 22 23:14:27.640 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.640 INFO listening, local_addr: 127.0.0.1:38823, task: repair
Sep 22 23:14:27.640 INFO Created new region file "/tmp/downstairs-GUWqp6Vc/region.json"
Sep 22 23:14:27.640 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.640 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:38823, task: repair
Sep 22 23:14:27.640 INFO Using repair address: 127.0.0.1:38823, task: main
Sep 22 23:14:27.640 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.640 INFO Created new region file "/tmp/downstairs-virqsJW3/region.json"
Sep 22 23:14:27.640 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.641 INFO Created new region file "/tmp/downstairs-WtHBKHj4/region.json"
Sep 22 23:14:27.643 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.643 INFO Opened existing region file "/tmp/downstairs-dsMmDgYS/region.json"
Sep 22 23:14:27.643 INFO Database read version 1
Sep 22 23:14:27.643 INFO Database write version 1
test test::integration_test_guest_downstairs_unwritten_sparse_1 ... ok
Sep 22 23:14:27.644 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.644 INFO Created new region file "/tmp/downstairs-Xe2fQWgH/region.json"
Sep 22 23:14:27.645 INFO UUID: 08d2b418-826a-41f9-a799-580b98907013
Sep 22 23:14:27.645 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.645 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.645 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.645 INFO Using address: 127.0.0.1:64593, task: main
Sep 22 23:14:27.645 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.645 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.645 INFO Opened existing region file "/tmp/downstairs-virqsJW3/region.json"
Sep 22 23:14:27.645 INFO Opened existing region file "/tmp/downstairs-GUWqp6Vc/region.json"
Sep 22 23:14:27.645 INFO Database read version 1
Sep 22 23:14:27.645 INFO Database read version 1
Sep 22 23:14:27.645 INFO Database write version 1
Sep 22 23:14:27.645 INFO Database write version 1
Sep 22 23:14:27.645 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.645 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:62090, task: repair
Sep 22 23:14:27.645 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:62090, task: repair
Sep 22 23:14:27.645 INFO listening, local_addr: 127.0.0.1:62090, task: repair
Sep 22 23:14:27.645 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.645 INFO Opened existing region file "/tmp/downstairs-WtHBKHj4/region.json"
Sep 22 23:14:27.645 INFO Database read version 1
Sep 22 23:14:27.645 INFO Database write version 1
Sep 22 23:14:27.645 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:62090, task: repair
Sep 22 23:14:27.645 INFO Using repair address: 127.0.0.1:62090, task: main
Sep 22 23:14:27.645 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.646 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.646 INFO Created new region file "/tmp/downstairs-F0NkJkt9/region.json"
Sep 22 23:14:27.647 INFO UUID: 4c5d3d8c-93c2-4478-a576-d240d19aa24c
Sep 22 23:14:27.647 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.647 INFO UUID: 16a7ce45-bb2e-4212-a780-53352bc92fab
Sep 22 23:14:27.647 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.647 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.647 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.647 INFO Using address: 127.0.0.1:48269, task: main
Sep 22 23:14:27.647 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.647 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.647 INFO Using address: 127.0.0.1:49337, task: main
Sep 22 23:14:27.647 INFO UUID: 7423012d-517a-406f-808c-d47b1ac32a6d
Sep 22 23:14:27.647 INFO Blocks per extent:5 Total Extents: 2
test test::integration_test_guest_downstairs ... ok
Sep 22 23:14:27.648 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.648 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.648 INFO Using address: 127.0.0.1:39986, task: main
Sep 22 23:14:27.648 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.648 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.648 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.648 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:60654, task: repair
Sep 22 23:14:27.648 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:53160, task: repair
Sep 22 23:14:27.648 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:53160, task: repair
Sep 22 23:14:27.648 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:60654, task: repair
Sep 22 23:14:27.648 INFO Created new region file "/tmp/downstairs-kK2UANdj/region.json"
Sep 22 23:14:27.648 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.648 INFO listening, local_addr: 127.0.0.1:60654, task: repair
Sep 22 23:14:27.648 INFO listening, local_addr: 127.0.0.1:53160, task: repair
Sep 22 23:14:27.648 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.648 INFO Opened existing region file "/tmp/downstairs-Xe2fQWgH/region.json"
Sep 22 23:14:27.648 INFO Database read version 1
Sep 22 23:14:27.648 INFO Database write version 1
Sep 22 23:14:27.648 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:43794, task: repair
Sep 22 23:14:27.648 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:60654, task: repair
Sep 22 23:14:27.648 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:53160, task: repair
Sep 22 23:14:27.648 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:43794, task: repair
Sep 22 23:14:27.648 INFO Using repair address: 127.0.0.1:60654, task: main
Sep 22 23:14:27.648 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.648 INFO Using repair address: 127.0.0.1:53160, task: main
Sep 22 23:14:27.648 INFO listening, local_addr: 127.0.0.1:43794, task: repair
Sep 22 23:14:27.648 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.648 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:43794, task: repair
Sep 22 23:14:27.648 INFO Using repair address: 127.0.0.1:43794, task: main
Sep 22 23:14:27.648 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.649 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.649 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.649 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.649 INFO Created new region file "/tmp/downstairs-w6kg4mJs/region.json"
Sep 22 23:14:27.649 INFO Created new region file "/tmp/downstairs-L6GVuLbY/region.json"
Sep 22 23:14:27.649 INFO Created new region file "/tmp/downstairs-VKXIreUh/region.json"
Sep 22 23:14:27.649 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.649 INFO Opened existing region file "/tmp/downstairs-F0NkJkt9/region.json"
Sep 22 23:14:27.649 INFO Database read version 1
Sep 22 23:14:27.649 INFO Database write version 1
Sep 22 23:14:27.650 INFO UUID: 5ebd7851-0a07-43d6-ba5e-89e1c5b26e32
Sep 22 23:14:27.650 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.650 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.650 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.650 INFO Using address: 127.0.0.1:51538, task: main
Sep 22 23:14:27.651 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.651 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:62564, task: repair
Sep 22 23:14:27.651 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:62564, task: repair
Sep 22 23:14:27.651 INFO listening, local_addr: 127.0.0.1:62564, task: repair
Sep 22 23:14:27.651 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:62564, task: repair
Sep 22 23:14:27.651 INFO Using repair address: 127.0.0.1:62564, task: main
Sep 22 23:14:27.651 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.651 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.651 INFO Opened existing region file "/tmp/downstairs-kK2UANdj/region.json"
Sep 22 23:14:27.651 INFO Database read version 1
Sep 22 23:14:27.651 INFO Database write version 1
Sep 22 23:14:27.651 INFO UUID: af195d7e-0034-4c88-a8c6-0f914578a66f
Sep 22 23:14:27.651 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.652 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.652 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.652 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.652 INFO Using address: 127.0.0.1:46072, task: main
Sep 22 23:14:27.652 INFO Created new region file "/tmp/downstairs-pKPcS6al/region.json"
Sep 22 23:14:27.652 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.652 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.652 INFO Opened existing region file "/tmp/downstairs-w6kg4mJs/region.json"
Sep 22 23:14:27.652 INFO Database read version 1
Sep 22 23:14:27.652 INFO Database write version 1
Sep 22 23:14:27.652 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:56817, task: repair
Sep 22 23:14:27.652 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:56817, task: repair
Sep 22 23:14:27.652 INFO UUID: 892b80dc-b0fe-4fd6-9a73-88aa72d6d43b
Sep 22 23:14:27.652 INFO listening, local_addr: 127.0.0.1:56817, task: repair
Sep 22 23:14:27.652 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.652 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.652 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.652 INFO Using address: 127.0.0.1:42286, task: main
Sep 22 23:14:27.652 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:56817, task: repair
Sep 22 23:14:27.652 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.652 INFO Opened existing region file "/tmp/downstairs-L6GVuLbY/region.json"
Sep 22 23:14:27.652 INFO Using repair address: 127.0.0.1:56817, task: main
Sep 22 23:14:27.652 INFO Database read version 1
Sep 22 23:14:27.652 INFO Database write version 1
Sep 22 23:14:27.652 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.652 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.653 INFO Opened existing region file "/tmp/downstairs-VKXIreUh/region.json"
Sep 22 23:14:27.653 INFO Database read version 1
Sep 22 23:14:27.653 INFO Database write version 1
Sep 22 23:14:27.653 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.653 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:49747, task: repair
Sep 22 23:14:27.653 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:49747, task: repair
Sep 22 23:14:27.653 INFO listening, local_addr: 127.0.0.1:49747, task: repair
Sep 22 23:14:27.653 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:49747, task: repair
Sep 22 23:14:27.653 INFO Using repair address: 127.0.0.1:49747, task: main
Sep 22 23:14:27.653 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.653 INFO UUID: 05359e3f-cf4a-4b15-afb8-9fe615a364f7
Sep 22 23:14:27.653 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.653 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.653 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.653 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.653 INFO Using address: 127.0.0.1:38443, task: main
{Sep 22 23:14:27.654 INFO Created new region file "/tmp/downstairs-o4mV3CdV/region.json"
"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.654040391Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"Crucible Version: BuildInfo {\n    version: \"0.0.1\",\n    git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n    git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n    git_branch: \"main\",\n    rustc_semver: \"1.70.0\",\n    rustc_channel: \"stable\",\n    rustc_host_triple: \"x86_64-unknown-illumos\",\n    rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n    cargo_triple: \"x86_64-unknown-illumos\",\n    debug: true,\n    opt_level: 0,\n}","v":0,"name":"crucible","level":30Sep 22 23:14:27.654 DEBG Read      :1003 deps:[JobId(1002), JobId(1000)] res:true
,"time":"2023-09-22T23:14:27.65412444Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{The guest has requested activation
"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.654173753Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"Crucible stats registered with UUID: 44f3bfa1-82bf-4e34-9a0d-b9e0db308303","v":0,"name":"crucible","level":30,"Sep 22 23:14:27.654 INFO listening on 127.0.0.1:0, task: main
time":"2023-09-22T23:14:27.654216893Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"Crucible 44f3bfa1-82bf-4e34-9a0d-b9e0db308303 has session id: f210ac55-dab5-4449-83a0-f7a80ed3d221","v":0,"name":"crucible","level":30Sep 22 23:14:27.654 INFO Repair listens on 127.0.0.1:0, task: repair
,"Sep 22 23:14:27.654 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.654 INFO UUID: 24249434-76be-4dd4-92c1-0907ffa4ae22
time":"2023-09-22T23:14:27.654268337Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
Sep 22 23:14:27.654 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.654 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.654 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:43048, task: repair
Sep 22 23:14:27.654 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.654 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.654 INFO UUID: 3a5e7f4a-366b-4efc-b559-fdf518e087fb
Sep 22 23:14:27.654 INFO Using address: 127.0.0.1:38320, task: main
Sep 22 23:14:27.654 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:43048, task: repair
Sep 22 23:14:27.654 INFO Blocks per extent:5 Total Extents: 2
{"msg":"[0] connecting to 127.0.0.1:47517","v":0,"name":"crucible","level":30Sep 22 23:14:27.654 INFO listening, local_addr: 127.0.0.1:43048, task: repair
,"time":"2023-09-22T23:14:27.654464485Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"0"}
{"msg":"[1] connecting to 127.0.0.1:64593","v"Sep 22 23:14:27.654 DEBG Read      :1003 deps:[JobId(1002), JobId(1000)] res:true
:0,"name":"crucible","level":30Sep 22 23:14:27.654 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
,"time":"2023-09-22T23:14:27.654596777Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"1"}
Sep 22 23:14:27.654 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.654 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:43048, task: repair
Sep 22 23:14:27.654 INFO Using address: 127.0.0.1:61853, task: main
{"msg":"[2] connecting to 127.0.0.1:46072","v":0,"name":"crucible","level":30Sep 22 23:14:27.654 INFO Using repair address: 127.0.0.1:43048, task: main
,"time":"2023-09-22T23:14:27.654686434Z","hostname":"Sep 22 23:14:27.654 INFO No SSL acceptor configured, task: main
ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"2"}
Sep 22 23:14:27.654 INFO Repair listens on 127.0.0.1:0, task: repair
{"msg":"up_listen starts","v":0,"name":"crucible","level":30Sep 22 23:14:27.654 INFO current number of open files limit 65536 is already the maximum
,"time":"2023-09-22T23:14:27.654782358Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"task":"up_listen"}
Sep 22 23:14:27.654 INFO Opened existing region file "/tmp/downstairs-pKPcS6al/region.json"
{"msg":"Wait for all three downstairs to come online","v":0,"name":"Sep 22 23:14:27.654 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:58528, task: repair
crucible"Sep 22 23:14:27.654 INFO Database read version 1
,"level":30Sep 22 23:14:27.654 INFO Database write version 1
,"time":"2023-09-22T23:14:27.654843701Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"Sep 22 23:14:27.654 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:58528, task: repair
msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.654890574Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"Sep 22 23:14:27.654 INFO listening, local_addr: 127.0.0.1:58528, task: repair
44f3bfa1-82bf-4e34-9a0d-b9e0db308303 active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.654933878Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
Sep 22 23:14:27.654 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.655 INFO accepted connection from 127.0.0.1:51756, task: main
Sep 22 23:14:27.655 DEBG Read      :1003 deps:[JobId(1002), JobId(1000)] res:true
Sep 22 23:14:27.655 INFO Upstairs starts
Sep 22 23:14:27.655 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:58528, task: repair
Sep 22 23:14:27.655 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:46245, task: repair
Sep 22 23:14:27.655 INFO accepted connection from 127.0.0.1:53016, task: main
Sep 22 23:14:27.655 INFO Using repair address: 127.0.0.1:58528, task: main
Sep 22 23:14:27.655 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:14:27.655 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.655 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:14:27.655 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:46245, task: repair
Sep 22 23:14:27.655 INFO Crucible stats registered with UUID: a2292a1d-0eb4-4a70-b0af-896ace339068
Sep 22 23:14:27.655 INFO listening, local_addr: 127.0.0.1:46245, task: repair
Sep 22 23:14:27.655 INFO Crucible a2292a1d-0eb4-4a70-b0af-896ace339068 has session id: e1751d6e-8c4b-406c-915e-1c5bc2910fa3
Sep 22 23:14:27.655 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:46245, task: repair
Sep 22 23:14:27.655 INFO accepted connection from 127.0.0.1:64660, task: main
Sep 22 23:14:27.655 INFO Using repair address: 127.0.0.1:46245, task: main
Sep 22 23:14:27.655 INFO No SSL acceptor configured, task: main
The guest has requested activation
{"msg":"[0] 44f3bfa1-82bf-4e34-9a0d-b9e0db308303 looper connected"Sep 22 23:14:27.655 INFO listening on 127.0.0.1:0, task: main
,"v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.655523244Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"0"}
{"msg":Sep 22 23:14:27.655 INFO listening on 127.0.0.1:0, task: main
"[0] Proc runs for 127.0.0.1:47517 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.655580383Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"Sep 22 23:14:27.655 INFO listening on 127.0.0.1:0, task: main
[1] 44f3bfa1-82bf-4e34-9a0d-b9e0db308303 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.655624217Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"1"}
{"msg":"[1] Proc runs for 127.0.0.1:64593 in state New","v":0,"name":"crucible","level":30Sep 22 23:14:27.655 INFO [0] connecting to 127.0.0.1:36461, looper: 0
,"time":"2023-09-22T23:14:27.655664627Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[2] 44f3bfa1-82bf-4e34-9a0d-b9e0db308303 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.655705425Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"2"}
{"msg":"[2] Proc runs for 127.0.0.1:46072 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.655746399Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
Sep 22 23:14:27.655 INFO [1] connecting to 127.0.0.1:48269, looper: 1
Sep 22 23:14:27.655 INFO Connection request from 44f3bfa1-82bf-4e34-9a0d-b9e0db308303 with version 4, task: proc
Sep 22 23:14:27.655 INFO UUID: 189bf2c8-c3ff-478c-9ad5-fadafbe3e365
Sep 22 23:14:27.655 INFO upstairs UpstairsConnection { upstairs_id: 44f3bfa1-82bf-4e34-9a0d-b9e0db308303, session_id: bc6a14b0-7427-4a2e-9f5b-e7289d18983f, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.655 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.655 INFO [2] connecting to 127.0.0.1:38443, looper: 2
Sep 22 23:14:27.655 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.655 INFO Upstairs <-> Downstairs Message Version: 4, task: main
{Sep 22 23:14:27.656 INFO Using address: 127.0.0.1:38499, task: main
"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.656035216Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
Sep 22 23:14:27.656 INFO Connection request from 44f3bfa1-82bf-4e34-9a0d-b9e0db308303 with version 4, task: proc
{"msg":"Crucible Version: BuildInfo {\n    version: \"0.0.1\",\n    git_sha: \"Sep 22 23:14:27.656 INFO up_listen starts, task: up_listen
ed48f294784d46ea7d4bb99336918b74358eca46\",\n    git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n    git_branch: \"main\",\nSep 22 23:14:27.656 INFO upstairs UpstairsConnection { upstairs_id: 44f3bfa1-82bf-4e34-9a0d-b9e0db308303, session_id: bc6a14b0-7427-4a2e-9f5b-e7289d18983f, gen: 1 } connected, version 4, task: proc
    rustc_semver: \"1.70.0\",\n    rustc_channel: \"stable\",\nSep 22 23:14:27.656 INFO Wait for all three downstairs to come online
    rustc_host_triple: \"x86_64-unknown-illumos\",\n    rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n    cargo_triple: \"x86_64-unknown-illumos\",\n    debug: true,\n    opt_level: 0,\n}","vThe guest has requested activation
Sep 22 23:14:27.656 INFO Flush timeout: 0.5
{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.656169149Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
Sep 22 23:14:27.656 INFO listening on 127.0.0.1:0, task: main
{Sep 22 23:14:27.656 INFO Connection request from 44f3bfa1-82bf-4e34-9a0d-b9e0db308303 with version 4, task: proc
"msg":"Crucible Version: BuildInfo {\n    version: \"Sep 22 23:14:27.656 INFO a2292a1d-0eb4-4a70-b0af-896ace339068 active request set
The guest has requested activation
0.0.1\",\n    git_sha: \"Sep 22 23:14:27.656 INFO upstairs UpstairsConnection { upstairs_id: 44f3bfa1-82bf-4e34-9a0d-b9e0db308303, session_id: bc6a14b0-7427-4a2e-9f5b-e7289d18983f, gen: 1 } connected, version 4, task: proc
ed48f294784d46ea7d4bb99336918b74358eca46\",\n    git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n    git_branch: \"main\",\n    rustc_semver: \"1.70.0\",\n    rustc_channel: \"stable\",\n    rustc_host_triple: \"x86_64-unknown-illumos\",\n    rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n    cargo_triple: \"Sep 22 23:14:27.656 INFO listening on 127.0.0.1:0, task: main
x86_64-unknown-illumos\",\n    debug: true,\n    opt_level: 0,\n}","v":0,"name":"crucible","level":30,"Sep 22 23:14:27.656 INFO listening on 127.0.0.1:0, task: main
time":"2023-09-22T23:14:27.656277906Z"Sep 22 23:14:27.656 INFO current number of open files limit 65536 is already the maximum
,"hostname":Sep 22 23:14:27.656 INFO listening on 127.0.0.1:0, task: main
"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
Sep 22 23:14:27.656 INFO Opened existing region file "/tmp/downstairs-o4mV3CdV/region.json"
{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30Sep 22 23:14:27.656 INFO Database read version 1
Sep 22 23:14:27.656 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.656 INFO Repair listens on 127.0.0.1:0, task: repair
,"time":"2023-09-22T23:14:27.656350432Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301Sep 22 23:14:27.656 INFO Database write version 1
}
{"msg"Sep 22 23:14:27.656 INFO accepted connection from 127.0.0.1:52309, task: main
:"Crucible stats registered with UUID: f3c3fd03-7420-43e1-a477-c989612ec069","v":0,"name":"crucible","level":30Sep 22 23:14:27.656 INFO listening on 127.0.0.1:0, task: main
{,"time":"2023-09-22T23:14:27.6564084Z","hostname"":"msg":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
[0] 44f3bfa1-82bf-4e34-9a0d-b9e0db308303 (bc6a14b0-7427-4a2e-9f5b-e7289d18983f) New New New ds_transition to WaitActive","v":{0,"name":"crucible"","msglevel""::"30Crucible f3c3fd03-7420-43e1-a477-c989612ec069 has session id: 54bf6715-0ba8-499a-b96f-9e9970f9520b"Sep 22 23:14:27.656 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:43831, task: repair
,"v":0,"name":"crucible","level":30Sep 22 23:14:27.656 INFO accepted connection from 127.0.0.1:34139, task: main
,"time":"2023-09-22T23:14:27.65644981Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal,"","time"pid":":43012023-09-22T23:14:27.65646628Z"},
"hostname":"ip-10-150-1-74.us-west-2.compute.internal",{"pid":4301Sep 22 23:14:27.656 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:43831, task: repair
}"
msg":"[0] Transition from New to WaitActive"{,"v":0,""name"msg"::""crucible"[0] connecting to 127.0.0.1:53663",",level""v"::030,"name":"crucible","level":30Sep 22 23:14:27.656 INFO accepted connection from 127.0.0.1:45771, task: main
,Sep 22 23:14:27.656 INFO listening, local_addr: 127.0.0.1:43831, task: repair
"time":"2023-09-22T23:14:27.656530868Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[0] client is_active_req TRUE, promote! session bc6a14b0-7427-4a2e-9f5b-e7289d18983f","v":0,"name":"crucible","level":30Sep 22 23:14:27.656 INFO [0] a2292a1d-0eb4-4a70-b0af-896ace339068 looper connected, looper: 0
Sep 22 23:14:27.656 INFO [0] Proc runs for 127.0.0.1:36461 in state New
,"time":"2023-09-22T23:14:27.65653442Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,","time"looper:"":"0"2023-09-22T23:14:27.65658531Z"}
,Sep 22 23:14:27.656 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:43831, task: repair
{"hostname":""msg":"ip-10-150-1-74.us-west-2.compute.internal","[1] connecting to 127.0.0.1:49337"pid":,4301"v":0},
"name":"crucible","level":30{"msg":"[1] 44f3bfa1-82bf-4e34-9a0d-b9e0db308303 (bc6a14b0-7427-4a2e-9f5b-e7289d18983f) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible"Sep 22 23:14:27.656 INFO [1] a2292a1d-0eb4-4a70-b0af-896ace339068 looper connected, looper: 1
,,"Sep 22 23:14:27.656 INFO Using repair address: 127.0.0.1:43831, task: main
level"":time30":"2023-09-22T23:14:27.656753341Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"1"}
Sep 22 23:14:27.656 INFO UpstairsConnection { upstairs_id: 44f3bfa1-82bf-4e34-9a0d-b9e0db308303, session_id: bc6a14b0-7427-4a2e-9f5b-e7289d18983f, gen: 1 } is now active (read-write)
,{Sep 22 23:14:27.656 INFO No SSL acceptor configured, task: main
"Sep 22 23:14:27.656 INFO [1] Proc runs for 127.0.0.1:48269 in state New
"msg"time":":"[2] connecting to 127.0.0.1:61853"2023-09-22T23:14:27.65678526Z","v",:"0hostname,""name":":"crucible",ip-10-150-1-74.us-west-2.compute.internal"","levelpid""::430130}
{"msg":"[1] Transition from New to WaitActive","v":0,","timename""::""crucible","2023-09-22T23:14:27.656838971Z"level":,30"hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"2"}
{,"time":""2023-09-22T23:14:27.65686539Zmsg":"",up_listen starts"",hostname"":v"":0,"nameip-10-150-1-74.us-west-2.compute.internal"":,""pid":crucible4301","}level
":30{"msg":"[1] client is_active_req TRUE, promote! session bc6a14b0-7427-4a2e-9f5b-e7289d18983f","v":0,"name":"Sep 22 23:14:27.656 INFO UpstairsConnection { upstairs_id: 44f3bfa1-82bf-4e34-9a0d-b9e0db308303, session_id: bc6a14b0-7427-4a2e-9f5b-e7289d18983f, gen: 1 } is now active (read-write)
,"Sep 22 23:14:27.656 INFO [2] a2292a1d-0eb4-4a70-b0af-896ace339068 looper connected, looper: 2
crucible"timeSep 22 23:14:27.656 INFO accepted connection from 127.0.0.1:42001, task: main
":0,"name":"crucible","level":30Sep 22 23:14:27.656 INFO [2] Proc runs for 127.0.0.1:38443 in state New
,"time":"2023-09-22T23:14:27.656981316Z",,""hostname":level"":30ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg",":"time":"Upstairs <-> Downstairs Message Version: 4"2023-09-22T23:14:27.657016236Z","v",":hostname"Sep 22 23:14:27.657 INFO UpstairsConnection { upstairs_id: 44f3bfa1-82bf-4e34-9a0d-b9e0db308303, session_id: bc6a14b0-7427-4a2e-9f5b-e7289d18983f, gen: 1 } is now active (read-write)
Sep 22 23:14:27.657 INFO current number of open files limit 65536 is already the maximum
:"0,"name":"ip-10-150-1-74.us-west-2.compute.internal"crucible",",pid"":level"4301:30}
{"msg":","time"Sep 22 23:14:27.657 INFO accepted connection from 127.0.0.1:52789, task: main
:[2] 44f3bfa1-82bf-4e34-9a0d-b9e0db308303 (bc6a14b0-7427-4a2e-9f5b-e7289d18983f) WaitActive WaitActive New ds_transition to WaitActive"Sep 22 23:14:27.657 INFO accepted connection from 127.0.0.1:41119, task: main
2023-09-22T23:14:27.657061489Z"",","hostname"v":":0,"ip-10-150-1-74.us-west-2.compute.internal"name":","crucible"pid",":level"4301:30}
{"msg":"Crucible stats registered with UUID: 6478de22-510f-412e-afa6-6a9c47f1d3b8",","time"v"::"0,"name2023-09-22T23:14:27.657107143Z"":",crucible"","hostname"level:"":30ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[2] Transition from New to WaitActive",,""time"v"::"0,"name"2023-09-22T23:14:27.657135435Z":",crucible"","hostnameSep 22 23:14:27.657 INFO accepted connection from 127.0.0.1:48885, task: main
":level"Sep 22 23:14:27.657 INFO Created new region file "/tmp/downstairs-9O3BNKFH/region.json"
Sep 22 23:14:27.657 INFO accepted connection from 127.0.0.1:43369, task: main
":"2023-09-22T23:14:27.656907966Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"task":"up_listen"}
{Sep 22 23:14:27.657 INFO Connection request from a2292a1d-0eb4-4a70-b0af-896ace339068 with version 4, task: proc
"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30Sep 22 23:14:27.657 INFO accepted connection from 127.0.0.1:54706, task: main
Sep 22 23:14:27.657 INFO upstairs UpstairsConnection { upstairs_id: a2292a1d-0eb4-4a70-b0af-896ace339068, session_id: 6eb51b2f-ab23-44be-a134-dc7d267c1b0b, gen: 1 } connected, version 4, task: proc
,"time":"2023-09-22T23:14:27.657237122Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30:30,"time":"2023-09-22T23:14:27.657283835Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301},
"time":"{2023-09-22T23:14:27.65728835Z",""msg"hostname:"":"f3c3fd03-7420-43e1-a477-c989612ec069 active request set"ip-10-150-1-74.us-west-2.compute.internal",",v""pid"::04301,"name":"}crucible
","level":30{"msg":"[2] client is_active_req TRUE, promote! session bc6a14b0-7427-4a2e-9f5b-e7289d18983f","v":0,"name":"crucible"Sep 22 23:14:27.657 INFO UUID: 41cf1bdf-fb18-466b-8427-1ff781e7de45
,,"Sep 22 23:14:27.657 INFO Connection request from a2292a1d-0eb4-4a70-b0af-896ace339068 with version 4, task: proc
level":"30time":"2023-09-22T23:14:27.657337669Z"Sep 22 23:14:27.657 INFO Blocks per extent:5 Total Extents: 2
,"hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
,"time":"2023-09-22T23:14:27.657366672Z"{,"hostname":"Sep 22 23:14:27.657 INFO upstairs UpstairsConnection { upstairs_id: a2292a1d-0eb4-4a70-b0af-896ace339068, session_id: 6eb51b2f-ab23-44be-a134-dc7d267c1b0b, gen: 1 } connected, version 4, task: proc
ip-10-150-1-74.us-west-2.compute.internal"",msg":""pid":4301[0] f3c3fd03-7420-43e1-a477-c989612ec069 looper connected"}
,"v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.657411145Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"0"}
{"Sep 22 23:14:27.657 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
msg":"[0] Proc runs for 127.0.0.1:53663 in state New","v":0,"name":"crucible","level":30Sep 22 23:14:27.657 INFO Upstairs <-> Downstairs Message Version: 4, task: main
,"time":"2023-09-22T23:14:27.65746288Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
Sep 22 23:14:27.657 INFO Connection request from a2292a1d-0eb4-4a70-b0af-896ace339068 with version 4, task: proc
{Sep 22 23:14:27.657 INFO Using address: 127.0.0.1:52962, task: main
"msg":"[1] f3c3fd03-7420-43e1-a477-c989612ec069 looper connected","v":0,"name":"crucible","level":30Sep 22 23:14:27.657 INFO upstairs UpstairsConnection { upstairs_id: a2292a1d-0eb4-4a70-b0af-896ace339068, session_id: 6eb51b2f-ab23-44be-a134-dc7d267c1b0b, gen: 1 } connected, version 4, task: proc
,"time":"2023-09-22T23:14:27.657511448Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"1"}
{"msg":"[1] Proc runs for 127.0.0.1:49337 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.657561328Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[2] f3c3fd03-7420-43e1-a477-c989612ec069 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.65760441Z","hostname{":"ip-10-150-1-74.us-west-2.compute.internal","pid"":4301msg":","looper":"2"}
Sep 22 23:14:27.657 INFO Connection request from 6478de22-510f-412e-afa6-6a9c47f1d3b8 with version 4, task: proc
{[0] downstairs client at 127.0.0.1:47517 has UUID ae16b993-7324-48a0-87ab-26e33b2b0247"Sep 22 23:14:27.657 INFO [0] a2292a1d-0eb4-4a70-b0af-896ace339068 (6eb51b2f-ab23-44be-a134-dc7d267c1b0b) New New New ds_transition to WaitActive
,""v"msg"::"0,"name":"[2] Proc runs for 127.0.0.1:61853 in state New"crucible",",v":"0level",":name":30Sep 22 23:14:27.657 INFO [0] Transition from New to WaitActive
Sep 22 23:14:27.657 INFO upstairs UpstairsConnection { upstairs_id: 6478de22-510f-412e-afa6-6a9c47f1d3b8, session_id: 2614894d-39a8-4be9-92fd-93de9a377555, gen: 1 } connected, version 4, task: proc
"crucible","level":30,"time":"2023-09-22T23:14:27.657664603Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301Sep 22 23:14:27.657 INFO [0] client is_active_req TRUE, promote! session 6eb51b2f-ab23-44be-a134-dc7d267c1b0b
},"
time":"2023-09-22T23:14:27.657677888Z","{hostname":"ip-10-150-1-74.us-west-2.compute.internal"","msgpid""::"Sep 22 23:14:27.657 INFO Connection request from f3c3fd03-7420-43e1-a477-c989612ec069 with version 4, task: proc
4301}
[0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: ae16b993-7324-48a0-87ab-26e33b2b0247, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30Sep 22 23:14:27.657 INFO upstairs UpstairsConnection { upstairs_id: f3c3fd03-7420-43e1-a477-c989612ec069, session_id: fe77f777-565a-4e57-8434-1353313e888f, gen: 1 } connected, version 4, task: proc
,"time":"2023-09-22T23:14:27.657731955Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid"":4301}
ip-10-150-1-74.us-west-2.compute.internal","pid":4301{Sep 22 23:14:27.657 INFO Connection request from 6478de22-510f-412e-afa6-6a9c47f1d3b8 with version 4, task: proc
"}msg":"
44f3bfa1-82bf-4e34-9a0d-b9e0db308303 WaitActive WaitActive WaitActive","v":0,"{name":"crucible","level":30"msg":"Sep 22 23:14:27.657 INFO [1] a2292a1d-0eb4-4a70-b0af-896ace339068 (6eb51b2f-ab23-44be-a134-dc7d267c1b0b) WaitActive New New ds_transition to WaitActive
Crucible 6478de22-510f-412e-afa6-6a9c47f1d3b8 has session id: d7d9f4d8-e572-4413-a570-f9bc320c45b5"Sep 22 23:14:27.657 INFO upstairs UpstairsConnection { upstairs_id: 6478de22-510f-412e-afa6-6a9c47f1d3b8, session_id: 2614894d-39a8-4be9-92fd-93de9a377555, gen: 1 } connected, version 4, task: proc
,"v":0,"name":","crucible"time",":"level":302023-09-22T23:14:27.657789133Z","hostname":"Sep 22 23:14:27.657 INFO [1] Transition from New to WaitActive
ip-10-150-1-74.us-west-2.compute.internal","pid":4301Sep 22 23:14:27.657 INFO Repair listens on 127.0.0.1:0, task: repair
}
,"time":"{2023-09-22T23:14:27.657809126Z","hostname"":msg""Sep 22 23:14:27.657 INFO [1] client is_active_req TRUE, promote! session 6eb51b2f-ab23-44be-a134-dc7d267c1b0b
ip-10-150-1-74.us-west-2.compute.internal:"","pid":[1] downstairs client at 127.0.0.1:64593 has UUID 08d2b418-826a-41f9-a799-580b989070134301Sep 22 23:14:27.657 INFO Connection request from f3c3fd03-7420-43e1-a477-c989612ec069 with version 4, task: proc
}
","v":0,"name":"{crucible","level":30"msg":"[0] connecting to 127.0.0.1:49699","v":0,"name":"crucible","level":30Sep 22 23:14:27.657 INFO upstairs UpstairsConnection { upstairs_id: f3c3fd03-7420-43e1-a477-c989612ec069, session_id: fe77f777-565a-4e57-8434-1353313e888f, gen: 1 } connected, version 4, task: proc
,"time":"2023-09-22T23:14:27.657860834Z","hostname":",ip-10-150-1-74.us-west-2.compute.internal"","time"pid"::4301"}
2023-09-22T23:14:27.657869828Z"Sep 22 23:14:27.657 INFO Connection request from 6478de22-510f-412e-afa6-6a9c47f1d3b8 with version 4, task: proc
{,""hostname"msg"::""ip-10-150-1-74.us-west-2.compute.internal","pid":4301,Sep 22 23:14:27.657 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:52248, task: repair
"[1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 08d2b418-826a-41f9-a799-580b98907013, encrypted: true, database_read_version: 1, database_write_version: 1 }"looper":","0"v":}0
,"name":"Sep 22 23:14:27.657 INFO upstairs UpstairsConnection { upstairs_id: 6478de22-510f-412e-afa6-6a9c47f1d3b8, session_id: 2614894d-39a8-4be9-92fd-93de9a377555, gen: 1 } connected, version 4, task: proc
{crucible"","msg"level":":30[1] connecting to 127.0.0.1:39986","v":0,"name":"crucible","level":30Sep 22 23:14:27.657 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:52248, task: repair
,",time"Sep 22 23:14:27.657 INFO [2] a2292a1d-0eb4-4a70-b0af-896ace339068 (6eb51b2f-ab23-44be-a134-dc7d267c1b0b) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:14:27.657 INFO Connection request from f3c3fd03-7420-43e1-a477-c989612ec069 with version 4, task: proc
time"":":"2023-09-22T23:14:27.657944974Z"2023-09-22T23:14:27.657938312Z",","hostname"hostname:":""ip-10-150-1-74.us-west-2.compute.internal"ip-10-150-1-74.us-west-2.compute.internal",",pid"":pid4301"Sep 22 23:14:27.657 INFO upstairs UpstairsConnection { upstairs_id: f3c3fd03-7420-43e1-a477-c989612ec069, session_id: fe77f777-565a-4e57-8434-1353313e888f, gen: 1 } connected, version 4, task: proc
,"Sep 22 23:14:27.657 INFO listening, local_addr: 127.0.0.1:52248, task: repair
:Sep 22 23:14:27.657 INFO [2] Transition from New to WaitActive
4301looper":"}1"
}
{"{msg":""msg":"44f3bfa1-82bf-4e34-9a0d-b9e0db308303 WaitActive WaitActive WaitActive","[2] connecting to 127.0.0.1:38320"v":,0","vname"":Sep 22 23:14:27.658 INFO [2] client is_active_req TRUE, promote! session 6eb51b2f-ab23-44be-a134-dc7d267c1b0b
:"0,"crucible"name":","crucible"level",":level"30:30,"time":"2023-09-22T23:14:27.658048863Z",,""time":hostname"":"2023-09-22T23:14:27.658049804Z"ip-10-150-1-74.us-west-2.compute.internal",,""hostname"pid":":4301}
ip-10-150-1-74.us-west-2.compute.internal","pid":{4301","msg"looperSep 22 23:14:27.658 INFO Current flush_numbers [0..12]: [0, 0]
:"":"2"[2] downstairs client at 127.0.0.1:46072 has UUID af195d7e-0034-4c88-a8c6-0f914578a66f"},"
v":0,"name":"crucible",{"level":30"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.658110004Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
,"time":"2023-09-22T23:14:27.658119355Z"{,"hostname":""msg"ip-10-150-1-74.us-west-2.compute.internal":","pid":4301,"task":"up_listen"}
[2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: af195d7e-0034-4c88-a8c6-0f914578a66f, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"{name":"crucible",""levelmsg":"":30Wait for all three downstairs to come online","v":0,"name":"crucible","level":30Sep 22 23:14:27.658 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:52248, task: repair
,"time":"2023-09-22T23:14:27.658163557Z"Sep 22 23:14:27.658 INFO UpstairsConnection { upstairs_id: a2292a1d-0eb4-4a70-b0af-896ace339068, session_id: 6eb51b2f-ab23-44be-a134-dc7d267c1b0b, gen: 1 } is now active (read-write)
,,""timehostname""::""2023-09-22T23:14:27.658170387Z"ip-10-150-1-74.us-west-2.compute.internal",,""hostname":"pid":4301ip-10-150-1-74.us-west-2.compute.internal}"
,"pid":Sep 22 23:14:27.658 INFO Using repair address: 127.0.0.1:52248, task: main
{4301"msg"}:"
44f3bfa1-82bf-4e34-9a0d-b9e0db308303 WaitActive WaitActive WaitActive","v":0,"{name":"crucible"Sep 22 23:14:27.658 INFO No SSL acceptor configured, task: main
,""level"msg:"30:"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.658233155Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
,"time":"2023-09-22T23:14:27.658240728Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"6478de22-510f-412e-afa6-6a9c47f1d3b8 active request set","v":0,"name":"crucible","level":30Sep 22 23:14:27.658 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.658 INFO UpstairsConnection { upstairs_id: a2292a1d-0eb4-4a70-b0af-896ace339068, session_id: 6eb51b2f-ab23-44be-a134-dc7d267c1b0b, gen: 1 } is now active (read-write)
,"time":"2023-09-22T23:14:27.65828727Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[0] 6478de22-510f-412e-afa6-6a9c47f1d3b8 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.658339363Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"0"}
Sep 22 23:14:27.658 INFO UpstairsConnection { upstairs_id: 6478de22-510f-412e-afa6-6a9c47f1d3b8, session_id: 2614894d-39a8-4be9-92fd-93de9a377555, gen: 1 } is now active (read-write)
{Sep 22 23:14:27.658 INFO current number of open files limit 65536 is already the maximum
"msg":"[0] Proc runs for 127.0.0.1:49699 in state New","v":0,"name":"crucible","level":30Sep 22 23:14:27.658 INFO UpstairsConnection { upstairs_id: a2292a1d-0eb4-4a70-b0af-896ace339068, session_id: 6eb51b2f-ab23-44be-a134-dc7d267c1b0b, gen: 1 } is now active (read-write)
,"time":"2023-09-22T23:14:27.658399038Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"Sep 22 23:14:27.658 INFO UpstairsConnection { upstairs_id: f3c3fd03-7420-43e1-a477-c989612ec069, session_id: fe77f777-565a-4e57-8434-1353313e888f, gen: 1 } is now active (read-write)
[1] 6478de22-510f-412e-afa6-6a9c47f1d3b8 looper connected","v":0,"name":"crucible","level":30{"msg":"[0] f3c3fd03-7420-43e1-a477-c989612ec069 (fe77f777-565a-4e57-8434-1353313e888f) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.658460181Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"1Sep 22 23:14:27.658 INFO UpstairsConnection { upstairs_id: 6478de22-510f-412e-afa6-6a9c47f1d3b8, session_id: 2614894d-39a8-4be9-92fd-93de9a377555, gen: 1 } is now active (read-write)
"}
,"time":"2023-09-22T23:14:27.658486758Z"{,"hostname"Sep 22 23:14:27.658 INFO Created new region file "/tmp/downstairs-8awGcOg4/region.json"
":msg"":"ip-10-150-1-74.us-west-2.compute.internal"[1] Proc runs for 127.0.0.1:39986 in state New",","pid"v"::04301,"name":"}crucible
","level":30{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.658544518Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
Sep 22 23:14:27.658 INFO UpstairsConnection { upstairs_id: f3c3fd03-7420-43e1-a477-c989612ec069, session_id: fe77f777-565a-4e57-8434-1353313e888f, gen: 1 } is now active (read-write)
{,""timemsg":"":"2023-09-22T23:14:27.658556622Z"[2] 6478de22-510f-412e-afa6-6a9c47f1d3b8 looper connected",","hostname"v"::"0,"name":"ip-10-150-1-74.us-west-2.compute.internal"crucible",","pidlevel":"30:4301}
{"msg":","time":"[0] client is_active_req TRUE, promote! session fe77f777-565a-4e57-8434-1353313e888f"2023-09-22T23:14:27.65860208Z",","v"hostname"::"Sep 22 23:14:27.658 INFO UpstairsConnection { upstairs_id: 6478de22-510f-412e-afa6-6a9c47f1d3b8, session_id: 2614894d-39a8-4be9-92fd-93de9a377555, gen: 1 } is now active (read-write)
ip-10-150-1-74.us-west-2.compute.internal"0,","pidname""::"4301crucible",,""level"looper":":2"30}
{"msg":"[2] Proc runs for 127.0.0.1:38320 in state New","v":0,","name":time"":"crucible","level"2023-09-22T23:14:27.658645195Z":30,"hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{,""time"msg"::""2023-09-22T23:14:27.658669363Z","hostname":"[1] f3c3fd03-7420-43e1-a477-c989612ec069 (fe77f777-565a-4e57-8434-1353313e888f) WaitActive New New ds_transition to WaitActive"ip-10-150-1-74.us-west-2.compute.internal"Sep 22 23:14:27.658 INFO UpstairsConnection { upstairs_id: f3c3fd03-7420-43e1-a477-c989612ec069, session_id: fe77f777-565a-4e57-8434-1353313e888f, gen: 1 } is now active (read-write)
,","pidv""::04301,"name":"}crucible
","level":30{"msg":"[0] 6478de22-510f-412e-afa6-6a9c47f1d3b8 (2614894d-39a8-4be9-92fd-93de9a377555) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.65871412Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
,"time{":""2023-09-22T23:14:27.658729363Zmsg":"","[1] Transition from New to WaitActive"hostname",":v"":0,"name":"ip-10-150-1-74.us-west-2.compute.internal"crucible",","pid"level"::430130}
{"msg":"[0] Transition from New to WaitActive",",time"":"v":02023-09-22T23:14:27.658764149Z,""name":","crucible"hostname":,""level":30ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[1] client is_active_req TRUE, promote! session fe77f777-565a-4e57-8434-1353313e888f",,""time":"v":02023-09-22T23:14:27.658791046Z","name,"":"hostname":crucible"","level":30ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
,"time":"{2023-09-22T23:14:27.658818964Z","hostname":""msg":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}[0] client is_active_req TRUE, promote! session 2614894d-39a8-4be9-92fd-93de9a377555"
,"v":0,"name":{"crucible","level"":msg30":"[2] f3c3fd03-7420-43e1-a477-c989612ec069 (fe77f777-565a-4e57-8434-1353313e888f) WaitActive WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.658855835Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,}"
time":"2023-09-22T23:14:27.658867958Z"Sep 22 23:14:27.658 INFO [0] downstairs client at 127.0.0.1:36461 has UUID 6f874cfa-c3be-49c7-9256-3b7afc29b96b
,{""hostname"msg:"":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
[1] 6478de22-510f-412e-afa6-6a9c47f1d3b8 (2614894d-39a8-4be9-92fd-93de9a377555) WaitActive New New ds_transition to WaitActive","v":0{,"name":"crucible"","msglevel""::"30[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30Sep 22 23:14:27.658 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 6f874cfa-c3be-49c7-9256-3b7afc29b96b, encrypted: true, database_read_version: 1, database_write_version: 1 }
,"time":"2023-09-22T23:14:27.658919767Z",,""time"hostname"::""2023-09-22T23:14:27.658926828Z"ip-10-150-1-74.us-west-2.compute.internal",","hostnamepid":"":4301ip-10-150-1-74.us-west-2.compute.internal}"
,"pid":4301}
{{"msg":""msg":"[1] Transition from New to WaitActive","v":0,"[2] client is_active_req TRUE, promote! session fe77f777-565a-4e57-8434-1353313e888f"name":","crucible"v",":level"0:,"30name":"crucible","level":30,"time":","2023-09-22T23:14:27.658979842Z"time":","hostname":"2023-09-22T23:14:27.658983028Z","hostname":ip-10-150-1-74.us-west-2.compute.internal"","Sep 22 23:14:27.658 INFO a2292a1d-0eb4-4a70-b0af-896ace339068 WaitActive WaitActive WaitActive
pid":ip-10-150-1-74.us-west-2.compute.internal4301","pid":}4301
}
{"msg":"[1] client is_active_req TRUE, promote! session 2614894d-39a8-4be9-92fd-93de9a377555","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.659041693Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[2] 6478de22-510f-412e-afa6-6a9c47f1d3b8 (2614894d-39a8-4be9-92fd-93de9a377555) WaitActive WaitActive New ds_transition to WaitActive","v":0,"name":"Sep 22 23:14:27.659 INFO [1] downstairs client at 127.0.0.1:48269 has UUID 4c5d3d8c-93c2-4478-a576-d240d19aa24c
crucible","level":30,"time":"2023-09-22T23:14:27.659095248Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":Sep 22 23:14:27.659 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 4c5d3d8c-93c2-4478-a576-d240d19aa24c, encrypted: true, database_read_version: 1, database_write_version: 1 }
4301}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.659143212Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":Sep 22 23:14:27.659 INFO a2292a1d-0eb4-4a70-b0af-896ace339068 WaitActive WaitActive WaitActive
"[2] client is_active_req TRUE, promote! session 2614894d-39a8-4be9-92fd-93de9a377555","v":0,"name":"crucible","level":30Sep 22 23:14:27.659 INFO Current flush_numbers [0..12]: [0, 0]
,"time":"2023-09-22T23:14:27.659192279Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[0] downstairs client at 127.0.0.1:49699 has UUID 23321635-9e40-4165-b7e2-14f978a59fa9","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.659235444Z","Sep 22 23:14:27.659 INFO [2] downstairs client at 127.0.0.1:38443 has UUID 05359e3f-cf4a-4b15-afb8-9fe615a364f7
hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 23321635-9e40-4165-b7e2-14f978a59fa9, encrypted: true, database_read_version: 1, database_write_version: 1 }"Sep 22 23:14:27.659 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 05359e3f-cf4a-4b15-afb8-9fe615a364f7, encrypted: true, database_read_version: 1, database_write_version: 1 }
,"v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.659302336Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"6478de22-510f-412e-afa6-6a9c47f1d3b8 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30Sep 22 23:14:27.659 INFO a2292a1d-0eb4-4a70-b0af-896ace339068 WaitActive WaitActive WaitActive
,"{time":"2023-09-22T23:14:27.659347229Z","hostname":""msg"ip-10-150-1-74.us-west-2.compute.internal":,""pid":4301}
[0] downstairs client at 127.0.0.1:53663 has UUID 7300b28a-83f2-47df-a265-634df7adc55c","v":{0,"name":""cruciblemsg"":","level":30[1] downstairs client at 127.0.0.1:39986 has UUID 7423012d-517a-406f-808c-d47b1ac32a6d","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.659389568Z",","time":hostname":""2023-09-22T23:14:27.659396914Z"ip-10-150-1-74.us-west-2.compute.internal",","hostname":"pid":4301ip-10-150-1-74.us-west-2.compute.internal","}pid"
:4301}
{"{msg":""msg":"[0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 7300b28a-83f2-47df-a265-634df7adc55c, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"[1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 7423012d-517a-406f-808c-d47b1ac32a6d, encrypted: true, database_read_version: 1, database_write_version: 1 }"name":","crucible"v",:"0level",:"30name":"crucible","level":30,"time":",2023-09-22T23:14:27.65946005Z"",Sep 22 23:14:27.659 INFO Current flush_numbers [0..12]: [0, 0]
"time":hostname"":"2023-09-22T23:14:27.659463286Z"ip-10-150-1-74.us-west-2.compute.internal",","hostname":"pid":4301ip-10-150-1-74.us-west-2.compute.internal"},
"pid":4301}{
"msg":"{f3c3fd03-7420-43e1-a477-c989612ec069 WaitActive WaitActive WaitActive","v":"0msg",":"name":"crucible","level6478de22-510f-412e-afa6-6a9c47f1d3b8 WaitActive WaitActive WaitActive"":,30"v":0,"name":"crucible","level":30Sep 22 23:14:27.659 INFO Current flush_numbers [0..12]: [0, 0]
,"time":","2023-09-22T23:14:27.65952904Z"time":","hostname":"2023-09-22T23:14:27.65953469Z","ip-10-150-1-74.us-west-2.compute.internal"hostname",":pid"":4301ip-10-150-1-74.us-west-2.compute.internal"},
"pid":4301}
{"{msg":""msg":"[1] downstairs client at 127.0.0.1:49337 has UUID 16a7ce45-bb2e-4212-a780-53352bc92fab","v":0[2] downstairs client at 127.0.0.1:38320 has UUID 24249434-76be-4dd4-92c1-0907ffa4ae22",",name""v":":crucible0",","namelevel":"":crucible"30,"level":30,"time":","2023-09-22T23:14:27.659596485Z"time":","hostname":2023-09-22T23:14:27.659598351Z"",Sep 22 23:14:27.659 INFO Current flush_numbers [0..12]: [0, 0]
"hostname":"ip-10-150-1-74.us-west-2.compute.internal"Sep 22 23:14:27.659 INFO current number of open files limit 65536 is already the maximum
,"ip-10-150-1-74.us-west-2.compute.internalpid":"4301,"pid}"
:4301}
{"msg{"Sep 22 23:14:27.659 INFO Opened existing region file "/tmp/downstairs-9O3BNKFH/region.json"
:""msg":"Sep 22 23:14:27.659 INFO Downstairs has completed Negotiation, task: proc
[2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 24249434-76be-4dd4-92c1-0907ffa4ae22, encrypted: true, database_read_version: 1, database_write_version: 1 }"[1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 16a7ce45-bb2e-4212-a780-53352bc92fab, encrypted: true, database_read_version: 1, database_write_version: 1 }Sep 22 23:14:27.659 INFO Database read version 1
",,""v"v"::00,","name":"name"crucibleSep 22 23:14:27.659 INFO Database write version 1
:""crucible",","level"level"::3030,"time":"2023-09-22T23:14:27.659694452Z",,""time"hostname"::""2023-09-22T23:14:27.659694239Z"ip-10-150-1-74.us-west-2.compute.internal",,"Sep 22 23:14:27.659 INFO Downstairs has completed Negotiation, task: proc
pid":4301}
{"msg":"f3c3fd03-7420-43e1-a477-c989612ec069 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30"hostname":"ip-10-150-1-74.us-west-2.compute.internal",,""pid":time":"43012023-09-22T23:14:27.659820448Z","}hostname":"
ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{{""msg":"msg":"6478de22-510f-412e-afa6-6a9c47f1d3b8 WaitActive WaitActive WaitActive[2] downstairs client at 127.0.0.1:61853 has UUID 3a5e7f4a-366b-4efc-b559-fdf518e087fb"","v":,"0v",":name":"0crucible",",name":""crucible"level":,"30level":30,"time":"2023-09-22T23:14:27.659873806Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
,"time":"2023-09-22T23:14:27.659872602Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 3a5e7f4a-366b-4efc-b559-fdf518e087fb, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.659914251Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"f3c3fd03-7420-43e1-a477-c989612ec069 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.659942689Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
Sep 22 23:14:27.660 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.660 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.660 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.660 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.660 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.660 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.660 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.660 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.660 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.660 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.660 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.660 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.660 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.660 INFO UUID: fa1fe4a9-5c4a-42db-ac50-6ea8a2af770b
Sep 22 23:14:27.660 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.660 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.660 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.660 INFO Using address: 127.0.0.1:54668, task: main
{"msg":"Sep 22 23:14:27.660 INFO Current flush_numbers [0..12]: [0, 0]
[0] 44f3bfa1-82bf-4e34-9a0d-b9e0db308303 (bc6a14b0-7427-4a2e-9f5b-e7289d18983f) WaitActive WaitActive WaitActive ds_transition to WaitQuorumSep 22 23:14:27.660 INFO [0] a2292a1d-0eb4-4a70-b0af-896ace339068 (6eb51b2f-ab23-44be-a134-dc7d267c1b0b) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
","v":0,"name":"crucible","level":30Sep 22 23:14:27.660 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.660 INFO [0] Transition from WaitActive to WaitQuorum
,"time":"2023-09-22T23:14:27.660835843Z"Sep 22 23:14:27.660 WARN [0] new RM replaced this: None
,"hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.660891728Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:27.66092935Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30Sep 22 23:14:27.660 INFO [0] Starts reconcile loop
,"time":"2023-09-22T23:14:27.660968778Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[1] 44f3bfa1-82bf-4e34-9a0d-b9e0db308303 (bc6a14b0-7427-4a2e-9f5b-e7289d18983f) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30Sep 22 23:14:27.660 INFO Downstairs has completed Negotiation, task: proc
,"time":"2023-09-22T23:14:27.661010469Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30Sep 22 23:14:27.661 INFO current number of open files limit 65536 is already the maximum
,"time":"2023-09-22T23:14:27.661043683Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"Sep 22 23:14:27.661 INFO [1] a2292a1d-0eb4-4a70-b0af-896ace339068 (6eb51b2f-ab23-44be-a134-dc7d267c1b0b) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.661 INFO Opened existing region file "/tmp/downstairs-8awGcOg4/region.json"
[1] new RM replaced this: None",Sep 22 23:14:27.661 INFO Repair listens on 127.0.0.1:0, task: repair
"v":0,"name":"crucible","level":40Sep 22 23:14:27.661 INFO Database read version 1
Sep 22 23:14:27.661 INFO [1] Transition from WaitActive to WaitQuorum
,"time":"2023-09-22T23:14:27.661098482Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
Sep 22 23:14:27.661 INFO Database write version 1
Sep 22 23:14:27.661 WARN [1] new RM replaced this: None
{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.661136477Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301Sep 22 23:14:27.661 INFO [1] Starts reconcile loop
}
{"msg":"[2] 44f3bfa1-82bf-4e34-9a0d-b9e0db308303 (bc6a14b0-7427-4a2e-9f5b-e7289d18983f) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30Sep 22 23:14:27.661 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:44247, task: repair
,"time":"2023-09-22T23:14:27.661183502Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"Sep 22 23:14:27.661 INFO [2] a2292a1d-0eb4-4a70-b0af-896ace339068 (6eb51b2f-ab23-44be-a134-dc7d267c1b0b) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
2023-09-22T23:14:27.661224965Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{Sep 22 23:14:27.661 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:44247, task: repair
"Sep 22 23:14:27.661 INFO [2] Transition from WaitActive to WaitQuorum
msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40Sep 22 23:14:27.661 WARN [2] new RM replaced this: None
,"time":"2023-09-22T23:14:27.661278328Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30Sep 22 23:14:27.661 INFO [2] Starts reconcile loop
Sep 22 23:14:27.661 INFO listening, local_addr: 127.0.0.1:44247, task: repair
,"time":"2023-09-22T23:14:27.661311568Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[0] 127.0.0.1:47517 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.661349779Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"44f3bfa1-82bf-4e34-9a0d-b9e0db308303 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30Sep 22 23:14:27.661 INFO [0] 127.0.0.1:36461 task reports connection:true
,"time":"2023-09-22T23:14:27.661382869Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[0]R  flush_numbers: [0, 0]","v":0,"name":"crucible","level":30Sep 22 23:14:27.661 INFO a2292a1d-0eb4-4a70-b0af-896ace339068 WaitQuorum WaitQuorum WaitQuorum
,"time":"2023-09-22T23:14:27.661417597Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[0]R  generation: [0, 0]","v":0,"name":"Sep 22 23:14:27.661 INFO [0]R  flush_numbers: [0, 0]
crucible","level":30,"time":"Sep 22 23:14:27.661 INFO [0]R  generation: [0, 0]
2023-09-22T23:14:27.661461953Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":Sep 22 23:14:27.661 INFO [0]R  dirty: [false, false]
"[0]R  dirty: [false, false]","v":0,"name":"crucible","level":30Sep 22 23:14:27.661 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:44247, task: repair
Sep 22 23:14:27.661 INFO [1]R  flush_numbers: [0, 0]
,"time":"2023-09-22T23:14:27.661503049Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
Sep 22 23:14:27.661 INFO [1]R  generation: [0, 0]
{"msg":"[1]R  flush_numbers: [0, 0]","v":0,"name":"crucible","level":30Sep 22 23:14:27.661 INFO Using repair address: 127.0.0.1:44247, task: main
Sep 22 23:14:27.661 INFO [1]R  dirty: [false, false]
,"time":"2023-09-22T23:14:27.661547214Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"Sep 22 23:14:27.661 INFO [2]R  flush_numbers: [0, 0]
[1]R  generation: [0, 0]"Sep 22 23:14:27.661 INFO No SSL acceptor configured, task: main
,"v":0,"name":"crucible","level":30Sep 22 23:14:27.661 INFO [2]R  generation: [0, 0]
,"time":"2023-09-22T23:14:27.66159248Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
Sep 22 23:14:27.661 INFO [2]R  dirty: [false, false]
{"msg":"[1]R  dirty: [false, false]","v":0,"name":"crucible","level":30Sep 22 23:14:27.661 INFO Max found gen is 1
,"time":"2023-09-22T23:14:27.661627568Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[2]R  flush_numbers: [0, 0]",Sep 22 23:14:27.661 INFO Generation requested: 1 >= found:1
"v":0,"name":"crucible","level":30The guest has finished waiting for activation
Sep 22 23:14:27.661 INFO Next flush: 1
,"time":"2023-09-22T23:14:27.661664928Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[2]R  generation: [0, 0]","v":0,"name":"crucible","level":30Sep 22 23:14:27.661 INFO All extents match
,"time":"2023-09-22T23:14:27.661705216Z","hostname":The guest has finished waiting for activation
"Sep 22 23:14:27.661 INFO No downstairs repair required
ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[2]R  dirty: [false, false]","v":0,"name":"crucible","level":30Sep 22 23:14:27.661 INFO No initial repair work was required
,"time":"2023-09-22T23:14:27.661745572Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","Sep 22 23:14:27.661 INFO Set Downstairs and Upstairs active
pid":4301}
{"msg":"Max found gen is 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.661786075Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal",Sep 22 23:14:27.661 INFO a2292a1d-0eb4-4a70-b0af-896ace339068 is now active with session: 6eb51b2f-ab23-44be-a134-dc7d267c1b0b
"pid":4301}
{"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible","level":30Sep 22 23:14:27.661 INFO a2292a1d-0eb4-4a70-b0af-896ace339068 Set Active after no repair
,"time":"2023-09-22T23:14:27.6618255Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"Sep 22 23:14:27.661 INFO Notify all downstairs, region set compare is done.
msg":"Next flush: 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.661860884Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"All extents match"Sep 22 23:14:27.661 INFO Set check for repair
,"v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.661898903Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.661930359Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30The guest has finished waiting for activation
,"time":"2023-09-22T23:14:27.661961543Z","Sep 22 23:14:27.661 INFO [1] 127.0.0.1:48269 task reports connection:true
hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level"Sep 22 23:14:27.661 INFO Upstairs starts
:30Sep 22 23:14:27.662 INFO a2292a1d-0eb4-4a70-b0af-896ace339068 Active Active Active
,"time":"2023-09-22T23:14:27.662015326Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{Sep 22 23:14:27.662 INFO Set check for repair
"msg":"44f3bfa1-82bf-4e34-9a0d-b9e0db308303 is now active with session: bc6a14b0-7427-4a2e-9f5b-e7289d18983f","v":0,"name":"crucibleSep 22 23:14:27.662 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
","level":30,"time":"2023-09-22T23:14:27.662065939Z","hostname":"Sep 22 23:14:27.662 INFO Upstairs <-> Downstairs Message Version: 4
ip-10-150-1-74.us-west-2.compute.internal"Sep 22 23:14:27.662 INFO [2] 127.0.0.1:38443 task reports connection:true
,"pid":4301}
{"msg":"44f3bfa1-82bf-4e34-9a0d-b9e0db308303 Set Active after no repair","v":0,"name":"crucible","level":30Sep 22 23:14:27.662 INFO a2292a1d-0eb4-4a70-b0af-896ace339068 Active Active Active
,"Sep 22 23:14:27.662 INFO Crucible stats registered with UUID: 0cb33d2c-901d-4ca7-884a-5b8b12a112c2
time":"2023-09-22T23:14:27.662112672Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
Sep 22 23:14:27.662 INFO Set check for repair
{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30Sep 22 23:14:27.662 INFO Crucible 0cb33d2c-901d-4ca7-884a-5b8b12a112c2 has session id: 40298b4a-cb20-4101-85d3-564bfb0231b1
,"time":"2023-09-22T23:14:27.662159875Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.662191489Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[1] 127.0.0.1:64593 task reports connection:true","v":0,"name":"crucible","level":30Sep 22 23:14:27.662 INFO [0] received reconcile message
,"time":"2023-09-22T23:14:27.662223391Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"44f3bfa1-82bf-4e34-9a0d-b9e0db308303 Active Active Active","v":0,Sep 22 23:14:27.662 INFO [0] All repairs completed, exit
"name":"crucible","level":30,"time":"2023-09-22T23:14:27.662261504Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":Sep 22 23:14:27.662 INFO [0] Starts cmd_loop
4301}
{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.662298418Z"{,"hostname":"ip-10-150-1-74.us-west-2.compute.internal",""pid":4301msg"}:
"{"msg":"[0] 6478de22-510f-412e-afa6-6a9c47f1d3b8 (2614894d-39a8-4be9-92fd-93de9a377555) WaitActive WaitActive WaitActive ds_transition to WaitQuorum"[2] 127.0.0.1:46072 task reports connection:true",","vv""::00,",name"":"name"crucible":,""cruciblelevel"":30,"level":30,"time":"2023-09-22T23:14:27.662345695Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
,"{time":""msg":"2023-09-22T23:14:27.662348487Z","hostname":"44f3bfa1-82bf-4e34-9a0d-b9e0db308303 Active Active Active"Sep 22 23:14:27.662 INFO listening on 127.0.0.1:0, task: main
,ip-10-150-1-74.us-west-2.compute.internal""v":,0",pid"":name"4301:"crucible"},
"level":30{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,","time"name"::""crucible"2023-09-22T23:14:27.66239733Z,"",level"":hostname"30:"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"Set check for repair","Sep 22 23:14:27.662 INFO listening on 127.0.0.1:0, task: main
,v"":time"0:,""name":"2023-09-22T23:14:27.662418216Z"crucible",","hostname"level":":30ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
,"time{":""2023-09-22T23:14:27.662448997Z"msg",:""hostname":"[0] new RM replaced this: None","v":ip-10-150-1-74.us-west-2.compute.internal"0,",pid"":name4301":"}
crucible","level"{Sep 22 23:14:27.662 INFO listening on 127.0.0.1:0, task: main
:"40msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.662491224Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal",,""pid"time"::"4301}
Sep 22 23:14:27.662 INFO UUID: f00f8ef1-bf2e-4a59-9818-84a4661ee77c
2023-09-22T23:14:27.662501059Z"{",msg":""hostname":[0] Starts reconcile loop"","v":0,"ip-10-150-1-74.us-west-2.compute.internal"name",":pid"":crucible"4301,"level"}:
Sep 22 23:14:27.662 INFO Blocks per extent:5 Total Extents: 2
30{Sep 22 23:14:27.662 INFO [0] connecting to 127.0.0.1:51538, looper: 0
"msg":"[0] All repairs completed, exit",",time"":v"":0,"2023-09-22T23:14:27.662550694Z"name":,""crucible"hostname,"":"level":30ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":","time":"2023-09-22T23:14:27.662575384Z","hostname":"[1] 6478de22-510f-412e-afa6-6a9c47f1d3b8 (2614894d-39a8-4be9-92fd-93de9a377555) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum","v":ip-10-150-1-74.us-west-2.compute.internal"0,","pid"name:"4301:"}crucible
","level":{30"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.662608215Z",,""hostnametime"":":"Sep 22 23:14:27.662 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
2023-09-22T23:14:27.662616756Z"ip-10-150-1-74.us-west-2.compute.internal,"","hostname"pid"::"4301}
ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{{""msgmsg""::Sep 22 23:14:27.662 INFO Upstairs <-> Downstairs Message Version: 4, task: main
""[1] Transition from WaitActive to WaitQuorum"[1] received reconcile message",,""v"v"::00,","namename":"":crucible""crucible,"",levelSep 22 23:14:27.662 INFO [1] connecting to 127.0.0.1:38499, looper: 1
""level:Sep 22 23:14:27.662 INFO Using address: 127.0.0.1:63966, task: main
30":30,"time":"2023-09-22T23:14:27.662700006Z","hostname":","time":"ip-10-150-1-74.us-west-2.compute.internal","pid"2023-09-22T23:14:27.662697809Z":4301,"}
hostname":"{ip-10-150-1-74.us-west-2.compute.internal"","msgpid""::4301"}
{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40Sep 22 23:14:27.662 INFO [2] connecting to 127.0.0.1:54668, looper: 2
Sep 22 23:14:27.662 INFO [1] received reconcile message
,"time":"2023-09-22T23:14:27.662830106Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"Sep 22 23:14:27.662 INFO [1] All repairs completed, exit
[1] Starts reconcile loop","v":0,"name":"crucible","level":30Sep 22 23:14:27.662 INFO [1] Starts cmd_loop
Sep 22 23:14:27.662 INFO [2] received reconcile message
Sep 22 23:14:27.662 INFO up_listen starts, task: up_listen
Sep 22 23:14:27.662 INFO [2] All repairs completed, exit
Sep 22 23:14:27.662 INFO [2] Starts cmd_loop
Sep 22 23:14:27.662 INFO Wait for all three downstairs to come online
,"time":"2023-09-22T23:14:27.662880474Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
Sep 22 23:14:27.663 INFO Flush timeout: 0.5
{"msg":"[2] 6478de22-510f-412e-afa6-6a9c47f1d3b8 (2614894d-39a8-4be9-92fd-93de9a377555) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0Sep 22 23:14:27.663 INFO Repair listens on 127.0.0.1:0, task: repair
,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.663064681Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal"The guest has finished waiting for activation
,"pid":4301}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.663113964Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301Sep 22 23:14:27.663 INFO current number of open files limit 65536 is already the maximum
}
Sep 22 23:14:27.663 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:36339, task: repair
{"msg":"{[2] new RM replaced this: None","v":0,"name":"crucible"","msglevel""::40"[0] f3c3fd03-7420-43e1-a477-c989612ec069 (fe77f777-565a-4e57-8434-1353313e888f) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level,"":time":"302023-09-22T23:14:27.663183004Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level,"":time"30:"Sep 22 23:14:27.663 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:36339, task: repair
2023-09-22T23:14:27.663199942Z","hostname":","time"ip-10-150-1-74.us-west-2.compute.internal":","pid":2023-09-22T23:14:27.663224553Z"4301,"hostname":"}
ip-10-150-1-74.us-west-2.compute.internal{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","Sep 22 23:14:27.663 INFO Created new region file "/tmp/downstairs-vsXtRvAk/region.json"
level":30Sep 22 23:14:27.663 INFO listening, local_addr: 127.0.0.1:36339, task: repair
","pid":4301}
{,"time"":msg"":"2023-09-22T23:14:27.663276982Z","[0] 127.0.0.1:49699 task reports connection:true"hostname":","v":0,ip-10-150-1-74.us-west-2.compute.internal"","namepid":4301}
":"crucible","level":{30"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:27.663317164Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301[1] All repairs completed, exit}
","time":",{"v"2023-09-22T23:14:27.663327324Z:"0,,""hostnamename"":":"crucible"","msg"level"::"ip-10-150-1-74.us-west-2.compute.internal"30,"pid":6478de22-510f-412e-afa6-6a9c47f1d3b8 WaitQuorum WaitQuorum WaitQuorum"4301,"v"}:
0,"name":"crucible"Sep 22 23:14:27.663 INFO [2] 0cb33d2c-901d-4ca7-884a-5b8b12a112c2 looper connected, looper: 2
{,,"""timelevel""::"30msg":"2023-09-22T23:14:27.663359456Z"[0] Starts reconcile loop",","hostnamev""::0","name":"crucible",ip-10-150-1-74.us-west-2.compute.internal""time",:""pid",:2023-09-22T23:14:27.663380304Z4301"",}"
hostname":"level":30ip-10-150-1-74.us-west-2.compute.internal"{,"pid":4301}
"msg":"{"msg":"[0]R  flush_numbers: [0, 0]","v":0,"name":"crucible"[1] Starts cmd_loop,"",level,"":v30"":time"0:Sep 22 23:14:27.663 INFO [2] Proc runs for 127.0.0.1:54668 in state New
"2023-09-22T23:14:27.663404514Z",",name""time:"":,crucible""","2023-09-22T23:14:27.663422775Zlevel""hostname",:"30:hostname"":"ip-10-150-1-74.us-west-2.compute.internal"ip-10-150-1-74.us-west-2.compute.internal",,""pid"pid"::43014301}
,}"
time{":""2023-09-22T23:14:27.663442661Zmsg"":","{hostname[0]R  generation: [0, 0]"":","v":0",ip-10-150-1-74.us-west-2.compute.internal""name,"":pid""Sep 22 23:14:27.663 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:36339, task: repair
:crucible4301"msg},
""{level"::""30msg":"[2] received reconcile message","v":0,"name":"[1] f3c3fd03-7420-43e1-a477-c989612ec069 (fe77f777-565a-4e57-8434-1353313e888f) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum"crucible","level,"":,30v"":time":0","name":"2023-09-22T23:14:27.663489559Z"crucible",","hostname"level":,"":time":"30ip-10-150-1-74.us-west-2.compute.internal",2023-09-22T23:14:27.663499846Z""pid",:"4301hostname":}"
ip-10-150-1-74.us-west-2.compute.internal","pid":{4301}
"msg":"{Sep 22 23:14:27.663 INFO Using repair address: 127.0.0.1:36339, task: main
"[0]R  dirty: [false, false]"msg":",",v"[2] All repairs completed, exit:"0,,""v":name0",:""name":crucible"""Sep 22 23:14:27.663 INFO accepted connection from 127.0.0.1:41168, task: main
time":"crucible,""2023-09-22T23:14:27.66351222Z"level,"":level"30:,30Sep 22 23:14:27.663 INFO No SSL acceptor configured, task: main
"hostname":"ip-10-150-1-74.us-west-2.compute.internal",,","timetime""::"""pid":2023-09-22T23:14:27.663551622Z2023-09-22T23:14:27.663553388Z""4301,,""hostnamehostname"}:"
:""ip-10-150-1-74.us-west-2.compute.internalip-10-150-1-74.us-west-2.compute.internal"",,""pidpid""::43014301{}}

{"msg{""":msgmsg""::"""[2] Starts cmd_loop"[1]R  flush_numbers: [0, 0]",",v""[1] Transition from WaitActive to WaitQuorum":v"0:,0,,"""namename""::""v"cruciblecrucible"",:",0"levellevel""::3030,"name":"crucible","level":30Sep 22 23:14:27.663 INFO accepted connection from 127.0.0.1:53809, task: main
,,""timetime""::"",2023-09-22T23:14:27.663610259Z2023-09-22T23:14:27.663610163Z""",,"time"hostname""hostname:"":":"ip-10-150-1-74.us-west-2.compute.internal"2023-09-22T23:14:27.663614674Z,ip-10-150-1-74.us-west-2.compute.internal""pid,"":4301,"}
hostname":""pidip-10-150-1-74.us-west-2.compute.internal"",:"4301pid":}4301
}
{"msg":"{[1]R  generation: [0, 0]","v":0","msg"name"::""crucible"[1] new RM replaced this: None",",level"":v"30:0,"name":"crucible","level":40,"time":"2023-09-22T23:14:27.663678835Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
,"time":"2023-09-22T23:14:27.66368312Z"{,"hostname":""ip-10-150-1-74.us-west-2.compute.internal","pid":4301msg":"}
[1]R  dirty: [false, false]","v":{0,"name":""msg":"crucible"[1] Starts reconcile loop",",level""v"::300,"name":"crucible","level":30,"time":","2023-09-22T23:14:27.66372369Z"time":","hostname":"2023-09-22T23:14:27.663726236Z","hostname"ip-10-150-1-74.us-west-2.compute.internal":","pid":4301ip-10-150-1-74.us-west-2.compute.internal"},"
pid":4301}
{"msg":"[2]R  flush_numbers: [0, 0]","v":0,"name":"crucible","level":30{"msg":","time":"2023-09-22T23:14:27.663755066Z","[2] f3c3fd03-7420-43e1-a477-c989612ec069 (fe77f777-565a-4e57-8434-1353313e888f) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum"hostname":","v":0,"ip-10-150-1-74.us-west-2.compute.internal"name",":"pid":crucible"4301,"}
level":30{"msg":"[2]R  generation: [0, 0]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.663782161Z","hostname":","timeip-10-150-1-74.us-west-2.compute.internal"":,""pid":4301}
2023-09-22T23:14:27.663778084Z"{,""hostnamemsg":"":"[2]R  dirty: [false, false]","v":0,"ip-10-150-1-74.us-west-2.compute.internal"name":,""crucible"pid,""level"::304301}
,"time":"2023-09-22T23:14:27.663811524Z","hostname"{:"ip-10-150-1-74.us-west-2.compute.internal","pid":4301"}
msg":"{"msg":"[2] Transition from WaitActive to WaitQuorum"Max found gen is 1",","v":v"0:,0",name":""crucible"name",":level"":30crucible","level":30,"time":"2023-09-22T23:14:27.663842118Z","hostname":"Sep 22 23:14:27.663 INFO accepted connection from 127.0.0.1:51241, task: main
ip-10-150-1-74.us-west-2.compute.internal",,""timepid""::4301"}
2023-09-22T23:14:27.663844931Z","{hostname":""msgip-10-150-1-74.us-west-2.compute.internal":"","pid":Generation requested: 1 >= found:14301","}v"
{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40:0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.663984039Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
,"time":"{"2023-09-22T23:14:27.663994283Z"msg":","[2] Starts reconcile loop"hostname":","v":0,"name":"ip-10-150-1-74.us-west-2.compute.internalcrucible"",",level":"30pid":4301}
,"time":"2023-09-22T23:14:27.664025309Z","{hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":"4301msg":}"
Next flush: 1"{,"v":0,"name""msg":":"crucible"[0] 127.0.0.1:53663 task reports connection:true",","levelv":"0:,"30name":"crucible","level":30,"time":"2023-09-22T23:14:27.664064987Z",",hostname":""time":"ip-10-150-1-74.us-west-2.compute.internal","pid":43012023-09-22T23:14:27.664063158Z"}
,"hostname":"{"msg":"ip-10-150-1-74.us-west-2.compute.internal","pid":f3c3fd03-7420-43e1-a477-c989612ec069 WaitQuorum WaitQuorum WaitQuorum"4301,"v":0},"
name":"crucible","level":30{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.664100369Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
,"time":"{2023-09-22T23:14:27.664109623Z",""hostnamemsg""::""[0]R  flush_numbers: [0, 0]"ip-10-150-1-74.us-west-2.compute.internal",,""v":pid":04301,"name":"}crucible"
,"level":30{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.664148387Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
Sep 22 23:14:27.664 INFO [0] 0cb33d2c-901d-4ca7-884a-5b8b12a112c2 looper connected, looper: 0
{"msg":"[0]R  generation: [0, 0]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.664159204Z","hostname":","ip-10-150-1-74.us-west-2.compute.internal"time",":"pid":43012023-09-22T23:14:27.664193186Z",}"
hostname":"ip-10-150-1-74.us-west-2.compute.internal","{pid":4301}
Sep 22 23:14:27.664 INFO [0] Proc runs for 127.0.0.1:51538 in state New
"{msg":""msg":"No initial repair work was required"[0]R  dirty: [false, false]",","v"v:"0:,"0name",":name":"Sep 22 23:14:27.664 INFO Upstairs starts
crucible"","crucible"level",:"30level":30,"time":"2023-09-22T23:14:27.664257297Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
,"time":"Sep 22 23:14:27.664 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
{2023-09-22T23:14:27.664258694Z""msg",":"hostname":"[1]R  flush_numbers: [0, 0]","ip-10-150-1-74.us-west-2.compute.internal"v",:"0pid",Sep 22 23:14:27.664 INFO Upstairs <-> Downstairs Message Version: 4
:"4301name":"}crucible"
,"level":30{"msg":"Sep 22 23:14:27.664 INFO [1] 0cb33d2c-901d-4ca7-884a-5b8b12a112c2 looper connected, looper: 1
Set Downstairs and Upstairs active","v":0,,""time":name"":"crucible2023-09-22T23:14:27.66431883Z"",,""level"Sep 22 23:14:27.664 INFO Crucible stats registered with UUID: 6e979958-69ff-4fc0-a35a-d73365a539fc
hostname"::"30ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{,"time":""msg":2023-09-22T23:14:27.664348283Z"","Sep 22 23:14:27.664 INFO Crucible 6e979958-69ff-4fc0-a35a-d73365a539fc has session id: 5b9ef146-85d4-44fa-b242-97d4416ee69a
hostname":"Sep 22 23:14:27.664 INFO [1] Proc runs for 127.0.0.1:38499 in state New
ip-10-150-1-74.us-west-2.compute.internal","[1]R  generation: [0, 0]pid"":4301},
"v":0,"{name":"crucible",""msg"level"::"306478de22-510f-412e-afa6-6a9c47f1d3b8 is now active with session: 2614894d-39a8-4be9-92fd-93de9a377555","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.664394724Z","hostname":,""time":"ip-10-150-1-74.us-west-2.compute.internal"2023-09-22T23:14:27.664403891Z",","pid"hostname"::4301"}
ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{{"msg":""msg":"[1]R  dirty: [false, false]","v":06478de22-510f-412e-afa6-6a9c47f1d3b8 Set Active after no repair,""name":","crucible"v",:"0level",:"30name":"crucible","level":30,"time",:""time":2023-09-22T23:14:27.664458488Z"","hostname"2023-09-22T23:14:27.664455612Z":","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":ip-10-150-1-74.us-west-2.compute.internal"4301,"pid}"
:4301}
{{"msg":""[2]R  flush_numbers: [0, 0]msg"":","v":0,"Notify all downstairs, region set compare is done.name"":","crucible"v",:"0level",:"30name":"crucible","level":30,"time":",2023-09-22T23:14:27.664516162Z""time",:Sep 22 23:14:27.664 INFO listening on 127.0.0.1:0, task: main
""hostname":"2023-09-22T23:14:27.66451958Z","ip-10-150-1-74.us-west-2.compute.internal"hostname":","pid":4301ip-10-150-1-74.us-west-2.compute.internal","pid}"
:4301}
{"{msg":"[2]R  generation: [0, 0]"",msg"":v"":0Set check for repair",",name"":"v"crucible":,0","level"name"::30"crucible","level":30Sep 22 23:14:27.664 INFO listening on 127.0.0.1:0, task: main
,,""time":"time":"2023-09-22T23:14:27.664593054Z"2023-09-22T23:14:27.664590144Z","hostname":","hostnameip-10-150-1-74.us-west-2.compute.internal"",:""pid":4301ip-10-150-1-74.us-west-2.compute.internal",}"
pid":4301}{
"msg":"[1] 127.0.0.1:39986 task reports connection:true"{,"v":0,"name":""cruciblemsg"":,""level":30[2]R  dirty: [false, false]Sep 22 23:14:27.664 INFO listening on 127.0.0.1:0, task: main
","v":0,"name":"crucible",","time":level"":2023-09-22T23:14:27.664659907Z"30,"hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":","time":"6478de22-510f-412e-afa6-6a9c47f1d3b8 Active Active Active","2023-09-22T23:14:27.664675877Z"v":,0","hostname":name":""crucible","level"ip-10-150-1-74.us-west-2.compute.internal":,"30pid":4301}
{"msg":","Max found gen is 1time":"","2023-09-22T23:14:27.664707797Z"v"Sep 22 23:14:27.664 INFO [0] connecting to 127.0.0.1:42286, looper: 0
,:"0hostname",:""name":"crucible"ip-10-150-1-74.us-west-2.compute.internal",",level"":pid30":4301}
{"msg":"Set check for repair","v":0,"name":"crucible","level",:"30time":"2023-09-22T23:14:27.664745288Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
Sep 22 23:14:27.664 INFO Connection request from 0cb33d2c-901d-4ca7-884a-5b8b12a112c2 with version 4, task: proc
{,""time"msg:"":"2023-09-22T23:14:27.664762557Z"Generation requested: 1 >= found:1,""hostname":,""v":0,"ip-10-150-1-74.us-west-2.compute.internalname"":,""pid":crucible"4301,"level"}:
Sep 22 23:14:27.664 INFO upstairs UpstairsConnection { upstairs_id: 0cb33d2c-901d-4ca7-884a-5b8b12a112c2, session_id: 7c8684a1-73e7-46fa-b1fc-40fcde8c5854, gen: 1 } connected, version 4, task: proc
30{"msg":"[2] 127.0.0.1:38320 task reports connection:true","v":0,"name":"crucible","level":,30"time":"2023-09-22T23:14:27.664807121Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"Next flush: 1","v":0,"name",":"time"crucible:"","level2023-09-22T23:14:27.664820657Z"":,"30hostname":"ip-10-150-1-74.us-west-2.compute.internal","Sep 22 23:14:27.664 INFO [1] connecting to 127.0.0.1:52962, looper: 1
pid":4301}
,"time":"{2023-09-22T23:14:27.66485084Z","hostname""msg:":""ip-10-150-1-74.us-west-2.compute.internal"6478de22-510f-412e-afa6-6a9c47f1d3b8 Active Active Active,""pid":,"4301v":0},
"name":"crucible","level"{:30"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.664893057Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid,"":time4301":"}
Sep 22 23:14:27.664 INFO Connection request from 0cb33d2c-901d-4ca7-884a-5b8b12a112c2 with version 4, task: proc
2023-09-22T23:14:27.664901902Z"{"msg":"Set check for repair",","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
Sep 22 23:14:27.664 INFO upstairs UpstairsConnection { upstairs_id: 0cb33d2c-901d-4ca7-884a-5b8b12a112c2, session_id: 7c8684a1-73e7-46fa-b1fc-40fcde8c5854, gen: 1 } connected, version 4, task: proc
{"vmsg":"":0No downstairs repair required",","name"v"::"0crucible,"",name":""crucible"level",:"30level":30,"time":"2023-09-22T23:14:27.66497969Z","hostname":","time":"ip-10-150-1-74.us-west-2.compute.internal","pid2023-09-22T23:14:27.664981011Z"":4301,"hostname":}"
ip-10-150-1-74.us-west-2.compute.internal","pid":4301{}
"msg":"[0] received reconcile message"{,"v":0,"name"":"msg":crucible"","level"No initial repair work was required":30,"v":0,"name":"crucible","level":30Sep 22 23:14:27.665 INFO [2] connecting to 127.0.0.1:63966, looper: 2
,,Sep 22 23:14:27.665 INFO Connection request from 0cb33d2c-901d-4ca7-884a-5b8b12a112c2 with version 4, task: proc
"time"":"time":"2023-09-22T23:14:27.665035625Z"2023-09-22T23:14:27.665030198Z",","hostname"hostname:"":"ip-10-150-1-74.us-west-2.compute.internal"ip-10-150-1-74.us-west-2.compute.internal",,""pid"pid"::43014301}}Sep 22 23:14:27.665 INFO upstairs UpstairsConnection { upstairs_id: 0cb33d2c-901d-4ca7-884a-5b8b12a112c2, session_id: 7c8684a1-73e7-46fa-b1fc-40fcde8c5854, gen: 1 } connected, version 4, task: proc


{"{msg":""msgSet Downstairs and Upstairs active"":","v":0[0] All repairs completed, exit",",name":""crucible"v",:"0level",:"30name":"crucible","level":30,,""time"time":":"2023-09-22T23:14:27.665105579Z"2023-09-22T23:14:27.665108854Z",","hostname":hostname"":"ip-10-150-1-74.us-west-2.compute.internal"ip-10-150-1-74.us-west-2.compute.internal,"","pid"pid"::43014301}}

{"{msg":""msg":"[0] Starts cmd_loop","v":0,"f3c3fd03-7420-43e1-a477-c989612ec069 is now active with session: fe77f777-565a-4e57-8434-1353313e888f"name":"crucible",,""v"level:"0:,30"name":"crucible","level":30,Sep 22 23:14:27.665 INFO up_listen starts, task: up_listen
","time"time"::""2023-09-22T23:14:27.665171897Z"2023-09-22T23:14:27.665167893Z",","hostname":"hostname":"ip-10-150-1-74.us-west-2.compute.internal","ip-10-150-1-74.us-west-2.compute.internalpid"":,"4301pid":4301}
Sep 22 23:14:27.665 INFO Wait for all three downstairs to come online
}
{{"msg":""msg":"f3c3fd03-7420-43e1-a477-c989612ec069 Set Active after no repair"[1] received reconcile message",",v""Sep 22 23:14:27.665 INFO Flush timeout: 0.5
:0,"vname""::"0crucible",","name":"level"crucible":,"30level":30,"time":"2023-09-22T23:14:27.665252337Z","hostname":","timeip-10-150-1-74.us-west-2.compute.internal"",":pid":"4301}
2023-09-22T23:14:27.665251324Z","{hostname":""msg":"ip-10-150-1-74.us-west-2.compute.internal"[1] All repairs completed, exit",","pid"v":0:,"4301name":"crucible","}level":
30{"msg":"Notify all downstairs, region set compare is done.","v":0,",name"":"time":"crucible",2023-09-22T23:14:27.665297265Z"","level"hostname":":30ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":","time":"2023-09-22T23:14:27.665318577Z"2023-09-22T23:14:27.665329829Z","hostname":","ip-10-150-1-74.us-west-2.compute.internal"hostname",":pid":"4301}
ip-10-150-1-74.us-west-2.compute.internal",{"pid"":msg":"4301[2] received reconcile message","v"}:
0,"name":"crucible","level":30{"msg":","time":"Set check for repair"2023-09-22T23:14:27.665370684Z",","v"hostname":":0ip-10-150-1-74.us-west-2.compute.internal",,""name":"pid":crucible"4301,"}level"
:30{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":","2023-09-22T23:14:27.665397252Z"time":","hostname"2023-09-22T23:14:27.665407229Z:"","hostname"ip-10-150-1-74.us-west-2.compute.internal":,""pid":4301ip-10-150-1-74.us-west-2.compute.internal","pid"}:
4301}
{{"Sep 22 23:14:27.665 INFO [2] 0cb33d2c-901d-4ca7-884a-5b8b12a112c2 (7c8684a1-73e7-46fa-b1fc-40fcde8c5854) New New New ds_transition to WaitActive
msg"":"msg":"[2] Starts cmd_loop"[1] 127.0.0.1:49337 task reports connection:true",",v"":v"0:,"0name",:""name"crucible"Sep 22 23:14:27.665 INFO [2] Transition from New to WaitActive
:"Sep 22 23:14:27.665 INFO [0] 6e979958-69ff-4fc0-a35a-d73365a539fc looper connected, looper: 0
,"crucible"level",:"30level":30,"time":"2023-09-22T23:14:27.66548Z","hostname":","time":ip-10-150-1-74.us-west-2.compute.internal"","pid"2023-09-22T23:14:27.665478704Z":4301,"hostname}"
:"ip-10-150-1-74.us-west-2.compute.internal","Sep 22 23:14:27.665 INFO [0] Proc runs for 127.0.0.1:42286 in state New
{pid":"4301msg":"}
f3c3fd03-7420-43e1-a477-c989612ec069 Active Active Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.665535593Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"Set check for repair","v":0,"name":"crucible","level":30Sep 22 23:14:27.665 INFO [0] 0cb33d2c-901d-4ca7-884a-5b8b12a112c2 (7c8684a1-73e7-46fa-b1fc-40fcde8c5854) New New WaitActive ds_transition to WaitActive
,"time":"2023-09-22T23:14:27.665575592Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"Sep 22 23:14:27.665 INFO [0] Transition from New to WaitActive
msg":"[2] 127.0.0.1:61853 task reports connection:true","v":0,"name":"crucible","level":Sep 22 23:14:27.665 INFO [1] 6e979958-69ff-4fc0-a35a-d73365a539fc looper connected, looper: 1
30,"time":"2023-09-22T23:14:27.665635173Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"Sep 22 23:14:27.665 INFO [1] Proc runs for 127.0.0.1:52962 in state New
f3c3fd03-7420-43e1-a477-c989612ec069 Active Active Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.665684497Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
Sep 22 23:14:27.665 INFO [1] 0cb33d2c-901d-4ca7-884a-5b8b12a112c2 (7c8684a1-73e7-46fa-b1fc-40fcde8c5854) WaitActive New WaitActive ds_transition to WaitActive
{"msg":"Set check for repair","v":0,"name":"crucible","level":30Sep 22 23:14:27.665 INFO [1] Transition from New to WaitActive
,"time":"2023-09-22T23:14:27.665723244Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{Sep 22 23:14:27.665 INFO [2] 6e979958-69ff-4fc0-a35a-d73365a539fc looper connected, looper: 2
"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.665770524Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.665823274Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","Sep 22 23:14:27.665 INFO [2] Proc runs for 127.0.0.1:63966 in state New
pid":4301}
{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30The guest has requested activation
,"time":"2023-09-22T23:14:27.665877784Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.665921332Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
Sep 22 23:14:27.665 INFO 0cb33d2c-901d-4ca7-884a-5b8b12a112c2 active request set
{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.665959964Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.665997834Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.666047323Z","Sep 22 23:14:27.666 INFO [0] received activate with gen 1
hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[2] All repairs completed, exit","v":Sep 22 23:14:27.666 INFO [0] client got ds_active_rx, promote! session 7c8684a1-73e7-46fa-b1fc-40fcde8c5854
0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.666094111Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal"Sep 22 23:14:27.666 INFO accepted connection from 127.0.0.1:44001, task: main
,"pid":4301}
{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.666138503Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
Sep 22 23:14:27.666 INFO [1] received activate with gen 1
Sep 22 23:14:27.666 INFO accepted connection from 127.0.0.1:36027, task: main
Sep 22 23:14:27.666 INFO [1] client got ds_active_rx, promote! session 7c8684a1-73e7-46fa-b1fc-40fcde8c5854
Sep 22 23:14:27.666 INFO [2] received activate with gen 1
Sep 22 23:14:27.666 INFO [2] client got ds_active_rx, promote! session 7c8684a1-73e7-46fa-b1fc-40fcde8c5854
Sep 22 23:14:27.666 INFO accepted connection from 127.0.0.1:57952, task: main
Sep 22 23:14:27.666 INFO Connection request from 6e979958-69ff-4fc0-a35a-d73365a539fc with version 4, task: proc
Sep 22 23:14:27.666 INFO [2] downstairs client at 127.0.0.1:54668 has UUID fa1fe4a9-5c4a-42db-ac50-6ea8a2af770b
Sep 22 23:14:27.666 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: fa1fe4a9-5c4a-42db-ac50-6ea8a2af770b, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.666 INFO upstairs UpstairsConnection { upstairs_id: 6e979958-69ff-4fc0-a35a-d73365a539fc, session_id: 9abbc7cc-1c99-4a3a-9498-4633122695a8, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.666 INFO 0cb33d2c-901d-4ca7-884a-5b8b12a112c2 WaitActive WaitActive WaitActive
Sep 22 23:14:27.666 INFO [0] downstairs client at 127.0.0.1:51538 has UUID 5ebd7851-0a07-43d6-ba5e-89e1c5b26e32
test test::integration_test_guest_zero_length_io ... ok
Sep 22 23:14:27.666 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 5ebd7851-0a07-43d6-ba5e-89e1c5b26e32, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.666 INFO Connection request from 6e979958-69ff-4fc0-a35a-d73365a539fc with version 4, task: proc
Sep 22 23:14:27.667 INFO upstairs UpstairsConnection { upstairs_id: 6e979958-69ff-4fc0-a35a-d73365a539fc, session_id: 9abbc7cc-1c99-4a3a-9498-4633122695a8, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.667 INFO 0cb33d2c-901d-4ca7-884a-5b8b12a112c2 WaitActive WaitActive WaitActive
Sep 22 23:14:27.667 INFO Connection request from 6e979958-69ff-4fc0-a35a-d73365a539fc with version 4, task: proc
Sep 22 23:14:27.667 INFO upstairs UpstairsConnection { upstairs_id: 6e979958-69ff-4fc0-a35a-d73365a539fc, session_id: 9abbc7cc-1c99-4a3a-9498-4633122695a8, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.667 INFO [1] downstairs client at 127.0.0.1:38499 has UUID 189bf2c8-c3ff-478c-9ad5-fadafbe3e365
Sep 22 23:14:27.667 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 189bf2c8-c3ff-478c-9ad5-fadafbe3e365, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.667 INFO 0cb33d2c-901d-4ca7-884a-5b8b12a112c2 WaitActive WaitActive WaitActive
Sep 22 23:14:27.667 INFO [0] 6e979958-69ff-4fc0-a35a-d73365a539fc (9abbc7cc-1c99-4a3a-9498-4633122695a8) New New New ds_transition to WaitActive
Sep 22 23:14:27.667 INFO [0] Transition from New to WaitActive
Sep 22 23:14:27.667 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.667 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.667 INFO [1] 6e979958-69ff-4fc0-a35a-d73365a539fc (9abbc7cc-1c99-4a3a-9498-4633122695a8) WaitActive New New ds_transition to WaitActive
Sep 22 23:14:27.667 INFO [1] Transition from New to WaitActive
Sep 22 23:14:27.667 INFO Created new region file "/tmp/downstairs-n8rm3T5A/region.json"
Sep 22 23:14:27.667 INFO [2] 6e979958-69ff-4fc0-a35a-d73365a539fc (9abbc7cc-1c99-4a3a-9498-4633122695a8) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:14:27.667 INFO [2] Transition from New to WaitActive
Sep 22 23:14:27.667 INFO Downstairs has completed Negotiation, task: proc
The guest has requested activation
Sep 22 23:14:27.667 INFO 6e979958-69ff-4fc0-a35a-d73365a539fc active request set
Sep 22 23:14:27.667 INFO [0] received activate with gen 1
Sep 22 23:14:27.667 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.668 INFO [0] client got ds_active_rx, promote! session 9abbc7cc-1c99-4a3a-9498-4633122695a8
test test::integration_test_io_span_out_of_range ... ok
Sep 22 23:14:27.668 INFO [1] received activate with gen 1
test test::integration_test_io_out_of_range ... ok
Sep 22 23:14:27.668 INFO [1] client got ds_active_rx, promote! session 9abbc7cc-1c99-4a3a-9498-4633122695a8
Sep 22 23:14:27.668 INFO [2] received activate with gen 1
Sep 22 23:14:27.668 INFO [2] client got ds_active_rx, promote! session 9abbc7cc-1c99-4a3a-9498-4633122695a8
Sep 22 23:14:27.668 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.668 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.668 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.668 INFO Created new region file "/tmp/downstairs-IYCeuQqy/region.json"
Sep 22 23:14:27.668 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.668 INFO Created new region file "/tmp/downstairs-IS2nyztY/region.json"
Sep 22 23:14:27.668 INFO [0] downstairs client at 127.0.0.1:42286 has UUID 892b80dc-b0fe-4fd6-9a73-88aa72d6d43b
Sep 22 23:14:27.668 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.668 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 892b80dc-b0fe-4fd6-9a73-88aa72d6d43b, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.669 INFO 6e979958-69ff-4fc0-a35a-d73365a539fc WaitActive WaitActive WaitActive
Sep 22 23:14:27.669 INFO [1] downstairs client at 127.0.0.1:52962 has UUID 41cf1bdf-fb18-466b-8427-1ff781e7de45
Sep 22 23:14:27.669 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 41cf1bdf-fb18-466b-8427-1ff781e7de45, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.669 INFO [2] 0cb33d2c-901d-4ca7-884a-5b8b12a112c2 (7c8684a1-73e7-46fa-b1fc-40fcde8c5854) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.669 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.669 WARN [2] new RM replaced this: None
Sep 22 23:14:27.669 INFO 6e979958-69ff-4fc0-a35a-d73365a539fc WaitActive WaitActive WaitActive
Sep 22 23:14:27.669 INFO [2] Starts reconcile loop
Sep 22 23:14:27.669 INFO [2] downstairs client at 127.0.0.1:63966 has UUID f00f8ef1-bf2e-4a59-9818-84a4661ee77c
Sep 22 23:14:27.669 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: f00f8ef1-bf2e-4a59-9818-84a4661ee77c, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.669 INFO [0] 0cb33d2c-901d-4ca7-884a-5b8b12a112c2 (7c8684a1-73e7-46fa-b1fc-40fcde8c5854) WaitActive WaitActive WaitQuorum ds_transition to WaitQuorum
Sep 22 23:14:27.669 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.669 INFO 6e979958-69ff-4fc0-a35a-d73365a539fc WaitActive WaitActive WaitActive
Sep 22 23:14:27.669 WARN [0] new RM replaced this: None
Sep 22 23:14:27.669 INFO [0] Starts reconcile loop
Sep 22 23:14:27.669 INFO [1] 0cb33d2c-901d-4ca7-884a-5b8b12a112c2 (7c8684a1-73e7-46fa-b1fc-40fcde8c5854) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum
Sep 22 23:14:27.669 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.669 WARN [1] new RM replaced this: None
Sep 22 23:14:27.669 INFO [1] Starts reconcile loop
Sep 22 23:14:27.669 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.669 INFO [2] 127.0.0.1:54668 task reports connection:true
Sep 22 23:14:27.669 INFO 0cb33d2c-901d-4ca7-884a-5b8b12a112c2 WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:14:27.669 INFO [0]R  flush_numbers: [0, 0]
Sep 22 23:14:27.669 INFO [0]R  generation: [0, 0]
Sep 22 23:14:27.669 INFO [0]R  dirty: [false, false]
Sep 22 23:14:27.669 INFO [1]R  flush_numbers: [0, 0]
Sep 22 23:14:27.669 INFO [1]R  generation: [0, 0]
Sep 22 23:14:27.669 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.669 INFO [1]R  dirty: [false, false]
Sep 22 23:14:27.669 INFO [2]R  flush_numbers: [0, 0]
Sep 22 23:14:27.669 INFO [2]R  generation: [0, 0]
Sep 22 23:14:27.669 INFO [2]R  dirty: [false, false]
Sep 22 23:14:27.669 INFO Max found gen is 1
Sep 22 23:14:27.669 INFO Generation requested: 1 >= found:1
Sep 22 23:14:27.669 INFO Next flush: 1
Sep 22 23:14:27.669 INFO All extents match
Sep 22 23:14:27.669 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.669 INFO No downstairs repair required
Sep 22 23:14:27.669 INFO No initial repair work was required
Sep 22 23:14:27.669 INFO Opened existing region file "/tmp/downstairs-vsXtRvAk/region.json"
Sep 22 23:14:27.669 INFO Set Downstairs and Upstairs active
Sep 22 23:14:27.669 INFO Database read version 1
Sep 22 23:14:27.669 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.669 INFO Database write version 1
Sep 22 23:14:27.669 INFO 0cb33d2c-901d-4ca7-884a-5b8b12a112c2 is now active with session: 7c8684a1-73e7-46fa-b1fc-40fcde8c5854
Sep 22 23:14:27.670 INFO 0cb33d2c-901d-4ca7-884a-5b8b12a112c2 Set Active after no repair
Sep 22 23:14:27.670 INFO Notify all downstairs, region set compare is done.
Sep 22 23:14:27.670 INFO Set check for repair
Sep 22 23:14:27.670 INFO [0] 127.0.0.1:51538 task reports connection:true
Sep 22 23:14:27.670 INFO 0cb33d2c-901d-4ca7-884a-5b8b12a112c2 Active Active Active
Sep 22 23:14:27.670 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.670 INFO Set check for repair
Sep 22 23:14:27.670 INFO [1] 127.0.0.1:38499 task reports connection:true
Sep 22 23:14:27.670 INFO 0cb33d2c-901d-4ca7-884a-5b8b12a112c2 Active Active Active
Sep 22 23:14:27.670 INFO Set check for repair
Sep 22 23:14:27.670 INFO [0] received reconcile message
Sep 22 23:14:27.670 INFO [0] All repairs completed, exit
Sep 22 23:14:27.670 INFO [0] Starts cmd_loop
Sep 22 23:14:27.670 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.670 INFO [1] received reconcile message
Sep 22 23:14:27.670 INFO [1] All repairs completed, exit
Sep 22 23:14:27.670 INFO [1] Starts cmd_loop
Sep 22 23:14:27.670 INFO [2] received reconcile message
Sep 22 23:14:27.670 INFO [2] All repairs completed, exit
Sep 22 23:14:27.670 INFO [2] Starts cmd_loop
Sep 22 23:14:27.670 INFO Downstairs has completed Negotiation, task: proc
The guest has finished waiting for activation
Sep 22 23:14:27.670 INFO [0] 6e979958-69ff-4fc0-a35a-d73365a539fc (9abbc7cc-1c99-4a3a-9498-4633122695a8) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.670 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.670 WARN [0] new RM replaced this: None
Sep 22 23:14:27.670 INFO [0] Starts reconcile loop
Sep 22 23:14:27.670 DEBG IO Read  1000 has deps []
Sep 22 23:14:27.670 INFO [1] 6e979958-69ff-4fc0-a35a-d73365a539fc (9abbc7cc-1c99-4a3a-9498-4633122695a8) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.670 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.670 WARN [1] new RM replaced this: None
Sep 22 23:14:27.670 INFO [1] Starts reconcile loop
Sep 22 23:14:27.670 DEBG Write     :1004 deps:[JobId(1003), JobId(1002), JobId(1001), JobId(1000)] res:true
Sep 22 23:14:27.671 INFO [2] 6e979958-69ff-4fc0-a35a-d73365a539fc (9abbc7cc-1c99-4a3a-9498-4633122695a8) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.671 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.671 WARN [2] new RM replaced this: None
Sep 22 23:14:27.671 INFO [2] Starts reconcile loop
Sep 22 23:14:27.671 INFO [0] 127.0.0.1:42286 task reports connection:true
Sep 22 23:14:27.671 INFO 6e979958-69ff-4fc0-a35a-d73365a539fc WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:14:27.671 INFO [0]R  flush_numbers: [0, 0]
Sep 22 23:14:27.671 INFO [0]R  generation: [0, 0]
Sep 22 23:14:27.671 INFO [0]R  dirty: [false, false]
Sep 22 23:14:27.671 INFO [1]R  flush_numbers: [0, 0]
Sep 22 23:14:27.671 INFO [1]R  generation: [0, 0]
Sep 22 23:14:27.671 INFO [1]R  dirty: [false, false]
Sep 22 23:14:27.671 INFO [2]R  flush_numbers: [0, 0]
Sep 22 23:14:27.671 INFO [2]R  generation: [0, 0]
Sep 22 23:14:27.671 INFO [2]R  dirty: [false, false]
Sep 22 23:14:27.671 INFO Max found gen is 1
Sep 22 23:14:27.671 INFO Generation requested: 1 >= found:1
Sep 22 23:14:27.671 INFO UUID: 9de06cf3-fc58-4960-af68-27205e75c2e5
Sep 22 23:14:27.671 INFO Next flush: 1
Sep 22 23:14:27.671 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.671 INFO All extents match
Sep 22 23:14:27.671 DEBG Read      :1000 deps:[] res:true
Sep 22 23:14:27.671 INFO No downstairs repair required
Sep 22 23:14:27.671 INFO No initial repair work was required
Sep 22 23:14:27.671 INFO Set Downstairs and Upstairs active
Sep 22 23:14:27.671 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.671 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.671 INFO 6e979958-69ff-4fc0-a35a-d73365a539fc is now active with session: 9abbc7cc-1c99-4a3a-9498-4633122695a8
Sep 22 23:14:27.671 INFO Using address: 127.0.0.1:49734, task: main
Sep 22 23:14:27.671 INFO 6e979958-69ff-4fc0-a35a-d73365a539fc Set Active after no repair
Sep 22 23:14:27.671 INFO Notify all downstairs, region set compare is done.
Sep 22 23:14:27.671 INFO Set check for repair
Sep 22 23:14:27.671 DEBG Read      :1000 deps:[] res:true
Sep 22 23:14:27.671 INFO [1] 127.0.0.1:52962 task reports connection:true
Sep 22 23:14:27.671 INFO 6e979958-69ff-4fc0-a35a-d73365a539fc Active Active Active
Sep 22 23:14:27.671 INFO Set check for repair
Sep 22 23:14:27.671 DEBG Write     :1004 deps:[JobId(1003), JobId(1002), JobId(1001), JobId(1000)] res:true
Sep 22 23:14:27.671 INFO [2] 127.0.0.1:63966 task reports connection:true
Sep 22 23:14:27.671 INFO 6e979958-69ff-4fc0-a35a-d73365a539fc Active Active Active
Sep 22 23:14:27.671 DEBG Read      :1000 deps:[] res:true
Sep 22 23:14:27.671 INFO Set check for repair
Sep 22 23:14:27.671 INFO [0] received reconcile message
Sep 22 23:14:27.671 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.671 INFO [0] All repairs completed, exit
Sep 22 23:14:27.671 INFO [0] Starts cmd_loop
Sep 22 23:14:27.671 INFO [1] received reconcile message
Sep 22 23:14:27.671 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:43773, task: repair
Sep 22 23:14:27.671 INFO [1] All repairs completed, exit
Sep 22 23:14:27.671 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:43773, task: repair
Sep 22 23:14:27.671 INFO [1] Starts cmd_loop
Sep 22 23:14:27.672 INFO listening, local_addr: 127.0.0.1:43773, task: repair
Sep 22 23:14:27.672 DEBG [2] Read AckReady 1000, : downstairs
Sep 22 23:14:27.672 INFO [2] received reconcile message
Sep 22 23:14:27.672 INFO [2] All repairs completed, exit
Sep 22 23:14:27.672 INFO [2] Starts cmd_loop
Sep 22 23:14:27.672 DEBG [0] Read already AckReady 1000, : downstairs
The guest has finished waiting for activation
Sep 22 23:14:27.672 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:43773, task: repair
Sep 22 23:14:27.672 DEBG [1] Read already AckReady 1000, : downstairs
Sep 22 23:14:27.672 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.672 INFO Using repair address: 127.0.0.1:43773, task: main
Sep 22 23:14:27.672 INFO Opened existing region file "/tmp/downstairs-IYCeuQqy/region.json"
Sep 22 23:14:27.672 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.672 DEBG up_ds_listen was notified
Sep 22 23:14:27.672 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.672 INFO Database read version 1
Sep 22 23:14:27.672 DEBG Write     :1004 deps:[JobId(1003), JobId(1002), JobId(1001), JobId(1000)] res:true
Sep 22 23:14:27.672 INFO Database write version 1
Sep 22 23:14:27.672 INFO Opened existing region file "/tmp/downstairs-IS2nyztY/region.json"
Sep 22 23:14:27.672 DEBG up_ds_listen process 1000
Sep 22 23:14:27.672 INFO Database read version 1
Sep 22 23:14:27.672 INFO Database write version 1
Sep 22 23:14:27.672 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:14:27.672 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.672 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.672 INFO Upstairs starts
Sep 22 23:14:27.672 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:14:27.672 INFO Created new region file "/tmp/downstairs-LvUMdXVd/region.json"
Sep 22 23:14:27.672 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:14:27.672 INFO Crucible stats registered with UUID: 432c17f3-ca99-4afa-891c-25e95b3f728b
Sep 22 23:14:27.672 INFO Crucible 432c17f3-ca99-4afa-891c-25e95b3f728b has session id: 364115f9-176c-480d-86ab-3fccc63267b3
Sep 22 23:14:27.672 INFO [0] connecting to 127.0.0.1:42286, looper: 0
Sep 22 23:14:27.672 INFO [1] connecting to 127.0.0.1:52962, looper: 1
Sep 22 23:14:27.673 INFO [2] connecting to 127.0.0.1:63966, looper: 2
Sep 22 23:14:27.673 INFO up_listen starts, task: up_listen
Sep 22 23:14:27.673 INFO Wait for all three downstairs to come online
Sep 22 23:14:27.673 INFO Flush timeout: 0.5
Sep 22 23:14:27.673 INFO UUID: c160bb67-b4be-4c85-a112-7924757e1ad4
Sep 22 23:14:27.673 INFO UUID: a57d7c84-72de-4915-b41c-7b731748ffdd
Sep 22 23:14:27.673 INFO [1] 432c17f3-ca99-4afa-891c-25e95b3f728b looper connected, looper: 1
Sep 22 23:14:27.673 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.673 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.673 INFO [1] Proc runs for 127.0.0.1:52962 in state New
Sep 22 23:14:27.673 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.673 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.673 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.673 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.673 INFO [0] 432c17f3-ca99-4afa-891c-25e95b3f728b looper connected, looper: 0
Sep 22 23:14:27.673 INFO Using address: 127.0.0.1:63965, task: main
Sep 22 23:14:27.673 INFO Using address: 127.0.0.1:48094, task: main
Sep 22 23:14:27.673 INFO [0] Proc runs for 127.0.0.1:42286 in state New
Sep 22 23:14:27.673 INFO [2] 432c17f3-ca99-4afa-891c-25e95b3f728b looper connected, looper: 2
Sep 22 23:14:27.673 INFO [2] Proc runs for 127.0.0.1:63966 in state New
Sep 22 23:14:27.673 INFO accepted connection from 127.0.0.1:48180, task: main
Sep 22 23:14:27.673 DEBG Read      :1005 deps:[JobId(1004), JobId(1002), JobId(1000)] res:true
Sep 22 23:14:27.674 INFO accepted connection from 127.0.0.1:39314, task: main
Sep 22 23:14:27.674 INFO accepted connection from 127.0.0.1:43895, task: main
Sep 22 23:14:27.674 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.674 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.674 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:62668, task: repair
Sep 22 23:14:27.674 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:54063, task: repair
Sep 22 23:14:27.674 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:54063, task: repair
Sep 22 23:14:27.674 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:62668, task: repair
Sep 22 23:14:27.674 INFO listening, local_addr: 127.0.0.1:62668, task: repair
Sep 22 23:14:27.674 INFO listening, local_addr: 127.0.0.1:54063, task: repair
Sep 22 23:14:27.674 INFO Connection request from 432c17f3-ca99-4afa-891c-25e95b3f728b with version 4, task: proc
Sep 22 23:14:27.674 INFO upstairs UpstairsConnection { upstairs_id: 432c17f3-ca99-4afa-891c-25e95b3f728b, session_id: dfdb5e67-2a1d-4dc7-96ad-46ab7166dc54, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.674 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:62668, task: repair
Sep 22 23:14:27.674 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:54063, task: repair
Sep 22 23:14:27.674 INFO Using repair address: 127.0.0.1:62668, task: main
Sep 22 23:14:27.674 DEBG Read      :1005 deps:[JobId(1004), JobId(1002), JobId(1000)] res:true
Sep 22 23:14:27.674 INFO Using repair address: 127.0.0.1:54063, task: main
Sep 22 23:14:27.674 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.674 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.674 INFO Connection request from 432c17f3-ca99-4afa-891c-25e95b3f728b with version 4, task: proc
Sep 22 23:14:27.674 INFO upstairs UpstairsConnection { upstairs_id: 432c17f3-ca99-4afa-891c-25e95b3f728b, session_id: dfdb5e67-2a1d-4dc7-96ad-46ab7166dc54, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.674 INFO Connection request from 432c17f3-ca99-4afa-891c-25e95b3f728b with version 4, task: proc
Sep 22 23:14:27.674 INFO upstairs UpstairsConnection { upstairs_id: 432c17f3-ca99-4afa-891c-25e95b3f728b, session_id: dfdb5e67-2a1d-4dc7-96ad-46ab7166dc54, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.675 INFO [1] 432c17f3-ca99-4afa-891c-25e95b3f728b (dfdb5e67-2a1d-4dc7-96ad-46ab7166dc54) New New New ds_transition to WaitActive
Sep 22 23:14:27.675 INFO [1] Transition from New to WaitActive
Sep 22 23:14:27.675 INFO [0] 432c17f3-ca99-4afa-891c-25e95b3f728b (dfdb5e67-2a1d-4dc7-96ad-46ab7166dc54) New WaitActive New ds_transition to WaitActive
Sep 22 23:14:27.675 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.675 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.675 INFO [0] Transition from New to WaitActive
Sep 22 23:14:27.675 INFO [2] 432c17f3-ca99-4afa-891c-25e95b3f728b (dfdb5e67-2a1d-4dc7-96ad-46ab7166dc54) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:14:27.675 INFO [2] Transition from New to WaitActive
Sep 22 23:14:27.675 INFO Created new region file "/tmp/downstairs-nP07tIRt/region.json"
Sep 22 23:14:27.675 INFO Created new region file "/tmp/downstairs-7rayLe79/region.json"
The guest has requested activation
Sep 22 23:14:27.675 INFO 432c17f3-ca99-4afa-891c-25e95b3f728b active request set
Sep 22 23:14:27.675 INFO [0] received activate with gen 1
Sep 22 23:14:27.675 INFO [0] client got ds_active_rx, promote! session dfdb5e67-2a1d-4dc7-96ad-46ab7166dc54
Sep 22 23:14:27.675 INFO [1] received activate with gen 1
Sep 22 23:14:27.675 INFO [1] client got ds_active_rx, promote! session dfdb5e67-2a1d-4dc7-96ad-46ab7166dc54
Sep 22 23:14:27.675 INFO [2] received activate with gen 1
Sep 22 23:14:27.675 INFO [2] client got ds_active_rx, promote! session dfdb5e67-2a1d-4dc7-96ad-46ab7166dc54
test test::integration_test_just_read ... ok
Sep 22 23:14:27.676 INFO [1] downstairs client at 127.0.0.1:52962 has UUID 41cf1bdf-fb18-466b-8427-1ff781e7de45
Sep 22 23:14:27.676 DEBG Read      :1005 deps:[JobId(1004), JobId(1002), JobId(1000)] res:true
Sep 22 23:14:27.676 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 41cf1bdf-fb18-466b-8427-1ff781e7de45, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.676 INFO 432c17f3-ca99-4afa-891c-25e95b3f728b WaitActive WaitActive WaitActive
Sep 22 23:14:27.676 INFO [0] downstairs client at 127.0.0.1:42286 has UUID 892b80dc-b0fe-4fd6-9a73-88aa72d6d43b
Sep 22 23:14:27.676 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 892b80dc-b0fe-4fd6-9a73-88aa72d6d43b, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.676 INFO 432c17f3-ca99-4afa-891c-25e95b3f728b WaitActive WaitActive WaitActive
Sep 22 23:14:27.676 INFO [2] downstairs client at 127.0.0.1:63966 has UUID f00f8ef1-bf2e-4a59-9818-84a4661ee77c
Sep 22 23:14:27.676 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: f00f8ef1-bf2e-4a59-9818-84a4661ee77c, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.676 INFO 432c17f3-ca99-4afa-891c-25e95b3f728b WaitActive WaitActive WaitActive
Sep 22 23:14:27.676 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.676 INFO Created new region file "/tmp/downstairs-ahT0SB6c/region.json"
Sep 22 23:14:27.676 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.677 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.677 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.677 INFO Opened existing region file "/tmp/downstairs-LvUMdXVd/region.json"
Sep 22 23:14:27.677 INFO Database read version 1
Sep 22 23:14:27.677 INFO Database write version 1
Sep 22 23:14:27.677 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.677 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.677 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.678 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.678 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.678 INFO Opened existing region file "/tmp/downstairs-nP07tIRt/region.json"
Sep 22 23:14:27.678 INFO Database read version 1
Sep 22 23:14:27.678 INFO Database write version 1
Sep 22 23:14:27.678 INFO [1] 432c17f3-ca99-4afa-891c-25e95b3f728b (dfdb5e67-2a1d-4dc7-96ad-46ab7166dc54) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.678 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.678 WARN [1] new RM replaced this: None
Sep 22 23:14:27.678 INFO [1] Starts reconcile loop
Sep 22 23:14:27.678 INFO [0] 432c17f3-ca99-4afa-891c-25e95b3f728b (dfdb5e67-2a1d-4dc7-96ad-46ab7166dc54) WaitActive WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.678 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.678 WARN [0] new RM replaced this: None
Sep 22 23:14:27.678 INFO [0] Starts reconcile loop
Sep 22 23:14:27.678 INFO [2] 432c17f3-ca99-4afa-891c-25e95b3f728b (dfdb5e67-2a1d-4dc7-96ad-46ab7166dc54) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.678 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.678 WARN [2] new RM replaced this: None
Sep 22 23:14:27.678 INFO [2] Starts reconcile loop
Sep 22 23:14:27.678 INFO [1] 127.0.0.1:52962 task reports connection:true
Sep 22 23:14:27.678 INFO 432c17f3-ca99-4afa-891c-25e95b3f728b WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:14:27.678 INFO [0]R  flush_numbers: [0, 0]
Sep 22 23:14:27.678 INFO [0]R  generation: [0, 0]
Sep 22 23:14:27.678 INFO [0]R  dirty: [false, false]
Sep 22 23:14:27.678 INFO [1]R  flush_numbers: [0, 0]
Sep 22 23:14:27.678 INFO [1]R  generation: [0, 0]
Sep 22 23:14:27.678 INFO [1]R  dirty: [false, false]
Sep 22 23:14:27.678 INFO [2]R  flush_numbers: [0, 0]
Sep 22 23:14:27.678 INFO [2]R  generation: [0, 0]
Sep 22 23:14:27.678 INFO [2]R  dirty: [false, false]
Sep 22 23:14:27.678 INFO Max found gen is 1
Sep 22 23:14:27.678 INFO Generation requested: 1 >= found:1
Sep 22 23:14:27.678 INFO Next flush: 1
Sep 22 23:14:27.678 INFO All extents match
Sep 22 23:14:27.678 INFO No downstairs repair required
Sep 22 23:14:27.678 INFO No initial repair work was required
Sep 22 23:14:27.678 INFO Set Downstairs and Upstairs active
Sep 22 23:14:27.679 INFO 432c17f3-ca99-4afa-891c-25e95b3f728b is now active with session: dfdb5e67-2a1d-4dc7-96ad-46ab7166dc54
Sep 22 23:14:27.679 INFO 432c17f3-ca99-4afa-891c-25e95b3f728b Set Active after no repair
Sep 22 23:14:27.679 INFO Notify all downstairs, region set compare is done.
Sep 22 23:14:27.679 INFO Set check for repair
Sep 22 23:14:27.679 INFO [0] 127.0.0.1:42286 task reports connection:true
Sep 22 23:14:27.679 INFO 432c17f3-ca99-4afa-891c-25e95b3f728b Active Active Active
Sep 22 23:14:27.679 INFO Set check for repair
Sep 22 23:14:27.679 INFO [2] 127.0.0.1:63966 task reports connection:true
Sep 22 23:14:27.679 INFO 432c17f3-ca99-4afa-891c-25e95b3f728b Active Active Active
Sep 22 23:14:27.679 INFO Set check for repair
Sep 22 23:14:27.679 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.679 INFO [0] received reconcile message
Sep 22 23:14:27.679 INFO Opened existing region file "/tmp/downstairs-7rayLe79/region.json"
Sep 22 23:14:27.679 INFO Database read version 1
Sep 22 23:14:27.679 INFO [0] All repairs completed, exit
Sep 22 23:14:27.679 INFO Database write version 1
Sep 22 23:14:27.679 INFO [0] Starts cmd_loop
Sep 22 23:14:27.679 INFO [1] received reconcile message
Sep 22 23:14:27.679 INFO [1] All repairs completed, exit
Sep 22 23:14:27.679 INFO [1] Starts cmd_loop
Sep 22 23:14:27.679 INFO [2] received reconcile message
Sep 22 23:14:27.679 INFO [2] All repairs completed, exit
Sep 22 23:14:27.679 INFO [2] Starts cmd_loop
The guest has finished waiting for activation
Sep 22 23:14:27.679 INFO UUID: 7296d75f-0499-4277-a0c4-39bde4c94ac5
Sep 22 23:14:27.679 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.679 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.679 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.679 DEBG IO Read  1000 has deps []
Sep 22 23:14:27.679 INFO Using address: 127.0.0.1:46304, task: main
Sep 22 23:14:27.680 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.680 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:64884, task: repair
Sep 22 23:14:27.680 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:64884, task: repair
Sep 22 23:14:27.680 INFO listening, local_addr: 127.0.0.1:64884, task: repair
Sep 22 23:14:27.680 DEBG Read      :1000 deps:[] res:true
Sep 22 23:14:27.680 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:64884, task: repair
Sep 22 23:14:27.680 INFO Using repair address: 127.0.0.1:64884, task: main
Sep 22 23:14:27.680 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.680 INFO UUID: 287a3e0c-04cc-4f00-8245-e0cea8ea9ef0
Sep 22 23:14:27.680 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.680 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.680 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.680 DEBG Read      :1000 deps:[] res:true
Sep 22 23:14:27.680 INFO Using address: 127.0.0.1:41031, task: main
Sep 22 23:14:27.680 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.681 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.681 WARN a2292a1d-0eb4-4a70-b0af-896ace339068 request to replace downstairs 127.0.0.1:36461 with 127.0.0.1:49734
Sep 22 23:14:27.681 INFO a2292a1d-0eb4-4a70-b0af-896ace339068 found old target: 127.0.0.1:36461 at 0
Sep 22 23:14:27.681 INFO a2292a1d-0eb4-4a70-b0af-896ace339068 replacing old: 127.0.0.1:36461 at 0
Sep 22 23:14:27.681 INFO [0] client skip 0 in process jobs because fault, : downstairs
Sep 22 23:14:27.681 INFO [0] changed 0 jobs to fault skipped, : downstairs
Sep 22 23:14:27.681 DEBG Read      :1000 deps:[] res:true
Sep 22 23:14:27.681 INFO [0] a2292a1d-0eb4-4a70-b0af-896ace339068 (6eb51b2f-ab23-44be-a134-dc7d267c1b0b) Active Active Active ds_transition to Replacing
Sep 22 23:14:27.681 INFO [0] Transition from Active to Replacing
Sep 22 23:14:27.681 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.681 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:59163, task: repair
Sep 22 23:14:27.681 WARN a2292a1d-0eb4-4a70-b0af-896ace339068 request to replace downstairs 127.0.0.1:48269 with 127.0.0.1:46304
Sep 22 23:14:27.681 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:59163, task: repair
Sep 22 23:14:27.681 INFO a2292a1d-0eb4-4a70-b0af-896ace339068 found old target: 127.0.0.1:48269 at 1
Sep 22 23:14:27.681 INFO listening, local_addr: 127.0.0.1:59163, task: repair
Sep 22 23:14:27.681 DEBG [0] Read AckReady 1000, : downstairs
Sep 22 23:14:27.681 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:59163, task: repair
Sep 22 23:14:27.681 INFO Using repair address: 127.0.0.1:59163, task: main
Sep 22 23:14:27.681 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.681 DEBG [1] Read already AckReady 1000, : downstairs
Sep 22 23:14:27.681 DEBG [2] Read already AckReady 1000, : downstairs
Sep 22 23:14:27.681 DEBG up_ds_listen was notified
Sep 22 23:14:27.681 DEBG up_ds_listen process 1000
Sep 22 23:14:27.681 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:14:27.681 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.682 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.682 DEBG IO Read  1000 has deps []
Sep 22 23:14:27.682 INFO Created new region file "/tmp/downstairs-lhV1HVSC/region.json"
Sep 22 23:14:27.682 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.682 INFO Opened existing region file "/tmp/downstairs-ahT0SB6c/region.json"
Sep 22 23:14:27.682 INFO Database read version 1
Sep 22 23:14:27.682 INFO Database write version 1
Sep 22 23:14:27.682 DEBG Read      :1000 deps:[] res:true
Sep 22 23:14:27.682 INFO UUID: af190b5e-a849-4b2d-90bf-b82d85ee2543
Sep 22 23:14:27.682 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.683 DEBG Read      :1000 deps:[] res:true
Sep 22 23:14:27.683 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.683 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.683 INFO Using address: 127.0.0.1:34981, task: main
Sep 22 23:14:27.683 DEBG Read      :1000 deps:[] res:true
Sep 22 23:14:27.683 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.683 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:51334, task: repair
Sep 22 23:14:27.683 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:51334, task: repair
Sep 22 23:14:27.683 INFO listening, local_addr: 127.0.0.1:51334, task: repair
Sep 22 23:14:27.683 DEBG [1] Read AckReady 1000, : downstairs
Sep 22 23:14:27.683 DEBG [0] Read already AckReady 1000, : downstairs
Sep 22 23:14:27.683 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:51334, task: repair
Sep 22 23:14:27.683 DEBG [2] Read already AckReady 1000, : downstairs
Sep 22 23:14:27.683 INFO Using repair address: 127.0.0.1:51334, task: main
Sep 22 23:14:27.683 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.683 DEBG up_ds_listen was notified
Sep 22 23:14:27.683 DEBG up_ds_listen process 1000
Sep 22 23:14:27.683 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:14:27.684 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.684 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.684 INFO Created new region file "/tmp/downstairs-9K0Zt4dt/region.json"
Sep 22 23:14:27.685 INFO UUID: e624c1bd-6ca3-47e5-9bce-8c13d262fb0f
Sep 22 23:14:27.685 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.685 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.685 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.685 INFO Using address: 127.0.0.1:41733, task: main
Sep 22 23:14:27.686 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.686 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:50935, task: repair
Sep 22 23:14:27.686 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:50935, task: repair
Sep 22 23:14:27.686 INFO listening, local_addr: 127.0.0.1:50935, task: repair
Sep 22 23:14:27.686 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:50935, task: repair
test test::integration_test_guest_downstairs_unwritten ... ok
Sep 22 23:14:27.686 INFO Using repair address: 127.0.0.1:50935, task: main
Sep 22 23:14:27.686 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.687 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.687 INFO Created new region file "/tmp/downstairs-GF5EnqWQ/region.json"
Sep 22 23:14:27.687 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.688 INFO Created new region file "/tmp/downstairs-lYcgzWeT/region.json"
Sep 22 23:14:27.688 INFO current number of open files limit 65536 is already the maximum
test test::integration_test_multi_read_only ... ok
Sep 22 23:14:27.688 INFO Opened existing region file "/tmp/downstairs-lhV1HVSC/region.json"
Sep 22 23:14:27.688 INFO Database read version 1
Sep 22 23:14:27.688 INFO Database write version 1
Sep 22 23:14:27.689 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.689 INFO Created new region file "/tmp/downstairs-VxCr6wAp/region.json"
test test::integration_test_guest_replace_many_downstairs ... ok
Sep 22 23:14:27.690 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.690 INFO Created new region file "/tmp/downstairs-w7fd116Q/region.json"
Sep 22 23:14:27.690 INFO UUID: 3e5c0f56-3bd3-4ecd-befe-102504a91370
Sep 22 23:14:27.690 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.690 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.690 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.690 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.690 INFO Using address: 127.0.0.1:40988, task: main
Sep 22 23:14:27.690 INFO Opened existing region file "/tmp/downstairs-9K0Zt4dt/region.json"
Sep 22 23:14:27.690 INFO Database read version 1
Sep 22 23:14:27.690 INFO Database write version 1
Sep 22 23:14:27.691 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.691 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:40770, task: repair
Sep 22 23:14:27.691 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:40770, task: repair
Sep 22 23:14:27.691 INFO listening, local_addr: 127.0.0.1:40770, task: repair
Sep 22 23:14:27.691 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:40770, task: repair
Sep 22 23:14:27.691 INFO Using repair address: 127.0.0.1:40770, task: main
Sep 22 23:14:27.691 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.691 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.691 INFO Opened existing region file "/tmp/downstairs-lYcgzWeT/region.json"
Sep 22 23:14:27.691 INFO Database read version 1
Sep 22 23:14:27.691 INFO Database write version 1
Sep 22 23:14:27.691 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.691 INFO Opened existing region file "/tmp/downstairs-GF5EnqWQ/region.json"
Sep 22 23:14:27.692 INFO Database read version 1
Sep 22 23:14:27.692 INFO Database write version 1
Sep 22 23:14:27.692 INFO UUID: 12d88dad-09b4-4e8c-bc38-a95145cf057c
Sep 22 23:14:27.692 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.692 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.692 INFO Upstairs starts
Sep 22 23:14:27.692 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.692 INFO Using address: 127.0.0.1:39833, task: main
Sep 22 23:14:27.692 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:14:27.692 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:14:27.692 INFO Crucible stats registered with UUID: ce9a8163-4db1-4374-b6d0-604bca1e70ae
Sep 22 23:14:27.692 INFO Crucible ce9a8163-4db1-4374-b6d0-604bca1e70ae has session id: bdade29d-3285-4c6a-be66-c4bc0b33d01d
Sep 22 23:14:27.693 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.693 INFO UUID: c415f140-2b86-4861-a661-215ad49ee0b4
Sep 22 23:14:27.693 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.693 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.693 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.693 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.693 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.693 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.693 INFO Using address: 127.0.0.1:49579, task: main
Sep 22 23:14:27.693 INFO [0] connecting to 127.0.0.1:48094, looper: 0
Sep 22 23:14:27.693 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:55042, task: repair
Sep 22 23:14:27.693 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:55042, task: repair
Sep 22 23:14:27.693 INFO listening, local_addr: 127.0.0.1:55042, task: repair
Sep 22 23:14:27.693 INFO [1] connecting to 127.0.0.1:41031, looper: 1
Sep 22 23:14:27.693 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.693 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.693 INFO Opened existing region file "/tmp/downstairs-VxCr6wAp/region.json"
Sep 22 23:14:27.693 INFO Database read version 1
Sep 22 23:14:27.693 INFO Database write version 1
Sep 22 23:14:27.693 INFO [2] connecting to 127.0.0.1:40988, looper: 2
Sep 22 23:14:27.693 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:64895, task: repair
Sep 22 23:14:27.693 INFO up_listen starts, task: up_listen
Sep 22 23:14:27.693 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:55042, task: repair
Sep 22 23:14:27.693 INFO Wait for all three downstairs to come online
Sep 22 23:14:27.693 INFO Flush timeout: 0.5
Sep 22 23:14:27.693 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:64895, task: repair
Sep 22 23:14:27.693 INFO Using repair address: 127.0.0.1:55042, task: main
Sep 22 23:14:27.693 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.693 INFO listening, local_addr: 127.0.0.1:64895, task: repair
Sep 22 23:14:27.693 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:64895, task: repair
Sep 22 23:14:27.693 INFO accepted connection from 127.0.0.1:50420, task: main
Sep 22 23:14:27.693 INFO Using repair address: 127.0.0.1:64895, task: main
Sep 22 23:14:27.693 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.694 INFO accepted connection from 127.0.0.1:38244, task: main
Sep 22 23:14:27.694 INFO accepted connection from 127.0.0.1:45738, task: main
Sep 22 23:14:27.694 INFO [0] ce9a8163-4db1-4374-b6d0-604bca1e70ae looper connected, looper: 0
Sep 22 23:14:27.694 INFO UUID: d7e92243-263d-4c5e-b8c0-b3bf2546a302
Sep 22 23:14:27.694 INFO [0] Proc runs for 127.0.0.1:48094 in state New
Sep 22 23:14:27.694 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.694 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.694 INFO [1] ce9a8163-4db1-4374-b6d0-604bca1e70ae looper connected, looper: 1
Sep 22 23:14:27.694 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.694 INFO Using address: 127.0.0.1:59250, task: main
Sep 22 23:14:27.694 INFO [1] Proc runs for 127.0.0.1:41031 in state New
Sep 22 23:14:27.694 INFO [2] ce9a8163-4db1-4374-b6d0-604bca1e70ae looper connected, looper: 2
Sep 22 23:14:27.694 INFO [2] Proc runs for 127.0.0.1:40988 in state New
Sep 22 23:14:27.694 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.694 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.694 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:44662, task: repair
Sep 22 23:14:27.694 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:44662, task: repair
Sep 22 23:14:27.694 INFO listening, local_addr: 127.0.0.1:44662, task: repair
Sep 22 23:14:27.694 INFO Created new region file "/tmp/downstairs-6CnpwJQF/region.json"
Sep 22 23:14:27.694 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.694 INFO Opened existing region file "/tmp/downstairs-w7fd116Q/region.json"
Sep 22 23:14:27.694 INFO Database read version 1
Sep 22 23:14:27.694 INFO Database write version 1
Sep 22 23:14:27.694 INFO Connection request from ce9a8163-4db1-4374-b6d0-604bca1e70ae with version 4, task: proc
Sep 22 23:14:27.694 INFO Upstairs starts
Sep 22 23:14:27.694 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:44662, task: repair
Sep 22 23:14:27.694 INFO upstairs UpstairsConnection { upstairs_id: ce9a8163-4db1-4374-b6d0-604bca1e70ae, session_id: 7f38a194-f08c-4be4-9249-679cb6202428, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.694 INFO Using repair address: 127.0.0.1:44662, task: main
Sep 22 23:14:27.694 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:14:27.694 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.694 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:14:27.694 INFO Crucible stats registered with UUID: acc4dcd4-ace9-4660-952a-342aafe19661
Sep 22 23:14:27.694 INFO Connection request from ce9a8163-4db1-4374-b6d0-604bca1e70ae with version 4, task: proc
Sep 22 23:14:27.694 INFO Crucible acc4dcd4-ace9-4660-952a-342aafe19661 has session id: eee6ab7f-065d-44a7-b1e0-38dcf266c152
Sep 22 23:14:27.694 INFO upstairs UpstairsConnection { upstairs_id: ce9a8163-4db1-4374-b6d0-604bca1e70ae, session_id: 7f38a194-f08c-4be4-9249-679cb6202428, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.695 INFO Connection request from ce9a8163-4db1-4374-b6d0-604bca1e70ae with version 4, task: proc
Sep 22 23:14:27.695 INFO upstairs UpstairsConnection { upstairs_id: ce9a8163-4db1-4374-b6d0-604bca1e70ae, session_id: 7f38a194-f08c-4be4-9249-679cb6202428, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.695 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.695 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.695 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.695 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.695 INFO [0] connecting to 127.0.0.1:63965, looper: 0
The guest has requested activation
Sep 22 23:14:27.695 INFO Created new region file "/tmp/downstairs-bF0aQCxJ/region.json"
Sep 22 23:14:27.695 INFO [0] ce9a8163-4db1-4374-b6d0-604bca1e70ae (7f38a194-f08c-4be4-9249-679cb6202428) New New New ds_transition to WaitActive
Sep 22 23:14:27.695 INFO [0] Transition from New to WaitActive
Sep 22 23:14:27.695 INFO [1] connecting to 127.0.0.1:34981, looper: 1
Sep 22 23:14:27.695 INFO [1] ce9a8163-4db1-4374-b6d0-604bca1e70ae (7f38a194-f08c-4be4-9249-679cb6202428) WaitActive New New ds_transition to WaitActive
Sep 22 23:14:27.695 INFO [1] Transition from New to WaitActive
Sep 22 23:14:27.695 INFO [2] connecting to 127.0.0.1:39833, looper: 2
Sep 22 23:14:27.695 INFO [2] ce9a8163-4db1-4374-b6d0-604bca1e70ae (7f38a194-f08c-4be4-9249-679cb6202428) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:14:27.695 INFO UUID: 61ba2500-0b28-4437-bb9f-9b380dfcca32
Sep 22 23:14:27.695 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.695 INFO [2] Transition from New to WaitActive
Sep 22 23:14:27.695 INFO up_listen starts, task: up_listen
Sep 22 23:14:27.695 INFO ce9a8163-4db1-4374-b6d0-604bca1e70ae active request set
Sep 22 23:14:27.695 INFO Wait for all three downstairs to come online
Sep 22 23:14:27.695 INFO Flush timeout: 0.5
Sep 22 23:14:27.695 INFO [0] received activate with gen 1
Sep 22 23:14:27.695 INFO [0] client got ds_active_rx, promote! session 7f38a194-f08c-4be4-9249-679cb6202428
Sep 22 23:14:27.695 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.695 INFO [1] received activate with gen 1
Sep 22 23:14:27.695 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.695 INFO [1] client got ds_active_rx, promote! session 7f38a194-f08c-4be4-9249-679cb6202428
Sep 22 23:14:27.695 INFO Using address: 127.0.0.1:45743, task: main
Sep 22 23:14:27.695 INFO [2] received activate with gen 1
Sep 22 23:14:27.695 INFO [0] acc4dcd4-ace9-4660-952a-342aafe19661 looper connected, looper: 0
Sep 22 23:14:27.695 INFO [2] client got ds_active_rx, promote! session 7f38a194-f08c-4be4-9249-679cb6202428
Sep 22 23:14:27.695 INFO [0] Proc runs for 127.0.0.1:63965 in state New
Sep 22 23:14:27.696 INFO UpstairsConnection { upstairs_id: ce9a8163-4db1-4374-b6d0-604bca1e70ae, session_id: 7f38a194-f08c-4be4-9249-679cb6202428, gen: 1 } is now active (read-write)
Sep 22 23:14:27.696 INFO accepted connection from 127.0.0.1:56306, task: main
Sep 22 23:14:27.696 INFO UpstairsConnection { upstairs_id: ce9a8163-4db1-4374-b6d0-604bca1e70ae, session_id: 7f38a194-f08c-4be4-9249-679cb6202428, gen: 1 } is now active (read-write)
Sep 22 23:14:27.696 INFO [1] acc4dcd4-ace9-4660-952a-342aafe19661 looper connected, looper: 1
Sep 22 23:14:27.696 INFO [1] Proc runs for 127.0.0.1:34981 in state New
Sep 22 23:14:27.696 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.696 INFO UpstairsConnection { upstairs_id: ce9a8163-4db1-4374-b6d0-604bca1e70ae, session_id: 7f38a194-f08c-4be4-9249-679cb6202428, gen: 1 } is now active (read-write)
Sep 22 23:14:27.696 INFO accepted connection from 127.0.0.1:52527, task: main
Sep 22 23:14:27.696 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:42007, task: repair
Sep 22 23:14:27.696 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:42007, task: repair
Sep 22 23:14:27.696 INFO accepted connection from 127.0.0.1:62168, task: main
Sep 22 23:14:27.696 INFO listening, local_addr: 127.0.0.1:42007, task: repair
Sep 22 23:14:27.696 INFO [2] acc4dcd4-ace9-4660-952a-342aafe19661 looper connected, looper: 2
Sep 22 23:14:27.696 INFO [2] Proc runs for 127.0.0.1:39833 in state New
Sep 22 23:14:27.696 INFO [0] downstairs client at 127.0.0.1:48094 has UUID a57d7c84-72de-4915-b41c-7b731748ffdd
Sep 22 23:14:27.696 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:42007, task: repair
Sep 22 23:14:27.696 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: a57d7c84-72de-4915-b41c-7b731748ffdd, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.696 INFO Using repair address: 127.0.0.1:42007, task: main
Sep 22 23:14:27.696 INFO ce9a8163-4db1-4374-b6d0-604bca1e70ae WaitActive WaitActive WaitActive
Sep 22 23:14:27.696 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.696 INFO [1] downstairs client at 127.0.0.1:41031 has UUID 287a3e0c-04cc-4f00-8245-e0cea8ea9ef0
Sep 22 23:14:27.696 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 287a3e0c-04cc-4f00-8245-e0cea8ea9ef0, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.696 INFO ce9a8163-4db1-4374-b6d0-604bca1e70ae WaitActive WaitActive WaitActive
Sep 22 23:14:27.696 INFO [2] downstairs client at 127.0.0.1:40988 has UUID 3e5c0f56-3bd3-4ecd-befe-102504a91370
Sep 22 23:14:27.696 INFO Connection request from acc4dcd4-ace9-4660-952a-342aafe19661 with version 4, task: proc
Sep 22 23:14:27.696 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 3e5c0f56-3bd3-4ecd-befe-102504a91370, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.696 INFO upstairs UpstairsConnection { upstairs_id: acc4dcd4-ace9-4660-952a-342aafe19661, session_id: f9138689-ed0d-4ebb-bf00-e5ccee3ed000, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.696 INFO ce9a8163-4db1-4374-b6d0-604bca1e70ae WaitActive WaitActive WaitActive
Sep 22 23:14:27.696 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.696 INFO Connection request from acc4dcd4-ace9-4660-952a-342aafe19661 with version 4, task: proc
Sep 22 23:14:27.696 INFO upstairs UpstairsConnection { upstairs_id: acc4dcd4-ace9-4660-952a-342aafe19661, session_id: f9138689-ed0d-4ebb-bf00-e5ccee3ed000, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.696 INFO Created new region file "/tmp/downstairs-sXZCeOZr/region.json"
Sep 22 23:14:27.697 INFO Connection request from acc4dcd4-ace9-4660-952a-342aafe19661 with version 4, task: proc
Sep 22 23:14:27.697 INFO upstairs UpstairsConnection { upstairs_id: acc4dcd4-ace9-4660-952a-342aafe19661, session_id: f9138689-ed0d-4ebb-bf00-e5ccee3ed000, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.697 INFO UUID: 1e03bcc8-571c-4b44-a9e5-acc5366adf71
Sep 22 23:14:27.697 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.697 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.697 INFO [0] acc4dcd4-ace9-4660-952a-342aafe19661 (f9138689-ed0d-4ebb-bf00-e5ccee3ed000) New New New ds_transition to WaitActive
Sep 22 23:14:27.697 INFO [0] Transition from New to WaitActive
Sep 22 23:14:27.697 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.697 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.697 INFO Using address: 127.0.0.1:41704, task: main
Sep 22 23:14:27.697 INFO [1] acc4dcd4-ace9-4660-952a-342aafe19661 (f9138689-ed0d-4ebb-bf00-e5ccee3ed000) WaitActive New New ds_transition to WaitActive
Sep 22 23:14:27.697 INFO [1] Transition from New to WaitActive
Sep 22 23:14:27.697 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.697 INFO [2] acc4dcd4-ace9-4660-952a-342aafe19661 (f9138689-ed0d-4ebb-bf00-e5ccee3ed000) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:14:27.697 INFO [2] Transition from New to WaitActive
Sep 22 23:14:27.697 INFO Current flush_numbers [0..12]: [0, 0]
The guest has requested activation
Sep 22 23:14:27.697 INFO acc4dcd4-ace9-4660-952a-342aafe19661 active request set
Sep 22 23:14:27.697 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.697 INFO [0] received activate with gen 1
Sep 22 23:14:27.697 INFO [0] client got ds_active_rx, promote! session f9138689-ed0d-4ebb-bf00-e5ccee3ed000
Sep 22 23:14:27.697 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:57699, task: repair
Sep 22 23:14:27.697 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.697 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:57699, task: repair
Sep 22 23:14:27.697 INFO [1] received activate with gen 1
Sep 22 23:14:27.697 INFO [1] client got ds_active_rx, promote! session f9138689-ed0d-4ebb-bf00-e5ccee3ed000
Sep 22 23:14:27.697 INFO listening, local_addr: 127.0.0.1:57699, task: repair
Sep 22 23:14:27.698 INFO [2] received activate with gen 1
Sep 22 23:14:27.698 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.698 INFO [2] client got ds_active_rx, promote! session f9138689-ed0d-4ebb-bf00-e5ccee3ed000
Sep 22 23:14:27.698 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:57699, task: repair
Sep 22 23:14:27.698 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.698 INFO Using repair address: 127.0.0.1:57699, task: main
Sep 22 23:14:27.698 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.698 INFO UpstairsConnection { upstairs_id: acc4dcd4-ace9-4660-952a-342aafe19661, session_id: f9138689-ed0d-4ebb-bf00-e5ccee3ed000, gen: 1 } is now active (read-write)
Sep 22 23:14:27.698 INFO [0] ce9a8163-4db1-4374-b6d0-604bca1e70ae (7f38a194-f08c-4be4-9249-679cb6202428) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.698 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.698 WARN [0] new RM replaced this: None
Sep 22 23:14:27.698 INFO UpstairsConnection { upstairs_id: acc4dcd4-ace9-4660-952a-342aafe19661, session_id: f9138689-ed0d-4ebb-bf00-e5ccee3ed000, gen: 1 } is now active (read-write)
Sep 22 23:14:27.698 INFO [0] Starts reconcile loop
Sep 22 23:14:27.698 INFO [1] ce9a8163-4db1-4374-b6d0-604bca1e70ae (7f38a194-f08c-4be4-9249-679cb6202428) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.698 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.698 WARN [1] new RM replaced this: None
Sep 22 23:14:27.698 INFO UpstairsConnection { upstairs_id: acc4dcd4-ace9-4660-952a-342aafe19661, session_id: f9138689-ed0d-4ebb-bf00-e5ccee3ed000, gen: 1 } is now active (read-write)
Sep 22 23:14:27.698 INFO [1] Starts reconcile loop
Sep 22 23:14:27.698 INFO [2] ce9a8163-4db1-4374-b6d0-604bca1e70ae (7f38a194-f08c-4be4-9249-679cb6202428) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.698 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.698 WARN [2] new RM replaced this: None
Sep 22 23:14:27.698 INFO [2] Starts reconcile loop
Sep 22 23:14:27.698 INFO [0] 127.0.0.1:48094 task reports connection:true
Sep 22 23:14:27.698 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.698 INFO ce9a8163-4db1-4374-b6d0-604bca1e70ae WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:14:27.698 INFO [0]R  flush_numbers: [0, 0]
Sep 22 23:14:27.698 INFO [0]R  generation: [0, 0]
Sep 22 23:14:27.698 INFO [0]R  dirty: [false, false]
Sep 22 23:14:27.698 INFO [1]R  flush_numbers: [0, 0]
Sep 22 23:14:27.698 INFO [1]R  generation: [0, 0]
Sep 22 23:14:27.698 INFO [1]R  dirty: [false, false]
Sep 22 23:14:27.698 INFO [2]R  flush_numbers: [0, 0]
Sep 22 23:14:27.698 INFO [2]R  generation: [0, 0]
Sep 22 23:14:27.698 INFO [2]R  dirty: [false, false]
Sep 22 23:14:27.698 INFO Created new region file "/tmp/downstairs-cvtdcEe4/region.json"
Sep 22 23:14:27.698 INFO Max found gen is 1
Sep 22 23:14:27.698 INFO Generation requested: 1 >= found:1
Sep 22 23:14:27.698 INFO Next flush: 1
Sep 22 23:14:27.698 INFO All extents match
Sep 22 23:14:27.698 INFO No downstairs repair required
Sep 22 23:14:27.698 INFO No initial repair work was required
Sep 22 23:14:27.698 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.698 INFO Set Downstairs and Upstairs active
Sep 22 23:14:27.698 INFO Opened existing region file "/tmp/downstairs-bF0aQCxJ/region.json"
Sep 22 23:14:27.698 INFO Database read version 1
Sep 22 23:14:27.698 INFO ce9a8163-4db1-4374-b6d0-604bca1e70ae is now active with session: 7f38a194-f08c-4be4-9249-679cb6202428
Sep 22 23:14:27.698 INFO Database write version 1
Sep 22 23:14:27.698 INFO ce9a8163-4db1-4374-b6d0-604bca1e70ae Set Active after no repair
Sep 22 23:14:27.698 INFO Notify all downstairs, region set compare is done.
Sep 22 23:14:27.698 INFO Set check for repair
Sep 22 23:14:27.698 INFO [1] 127.0.0.1:41031 task reports connection:true
Sep 22 23:14:27.698 INFO ce9a8163-4db1-4374-b6d0-604bca1e70ae Active Active Active
Sep 22 23:14:27.698 INFO [0] downstairs client at 127.0.0.1:63965 has UUID c160bb67-b4be-4c85-a112-7924757e1ad4
Sep 22 23:14:27.699 INFO Set check for repair
Sep 22 23:14:27.699 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: c160bb67-b4be-4c85-a112-7924757e1ad4, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.699 INFO [2] 127.0.0.1:40988 task reports connection:true
Sep 22 23:14:27.699 INFO ce9a8163-4db1-4374-b6d0-604bca1e70ae Active Active Active
Sep 22 23:14:27.699 INFO acc4dcd4-ace9-4660-952a-342aafe19661 WaitActive WaitActive WaitActive
Sep 22 23:14:27.699 INFO Set check for repair
Sep 22 23:14:27.699 INFO [0] received reconcile message
Sep 22 23:14:27.699 INFO [1] downstairs client at 127.0.0.1:34981 has UUID af190b5e-a849-4b2d-90bf-b82d85ee2543
Sep 22 23:14:27.699 INFO [0] All repairs completed, exit
Sep 22 23:14:27.699 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: af190b5e-a849-4b2d-90bf-b82d85ee2543, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.699 INFO [0] Starts cmd_loop
Sep 22 23:14:27.699 INFO acc4dcd4-ace9-4660-952a-342aafe19661 WaitActive WaitActive WaitActive
Sep 22 23:14:27.699 INFO [1] received reconcile message
Sep 22 23:14:27.699 INFO [1] All repairs completed, exit
Sep 22 23:14:27.699 INFO [1] Starts cmd_loop
Sep 22 23:14:27.699 INFO [2] downstairs client at 127.0.0.1:39833 has UUID 12d88dad-09b4-4e8c-bc38-a95145cf057c
Sep 22 23:14:27.699 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.699 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 12d88dad-09b4-4e8c-bc38-a95145cf057c, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.699 INFO Opened existing region file "/tmp/downstairs-6CnpwJQF/region.json"
Sep 22 23:14:27.699 INFO Database read version 1
Sep 22 23:14:27.699 INFO Database write version 1
Sep 22 23:14:27.699 INFO [2] received reconcile message
Sep 22 23:14:27.699 INFO acc4dcd4-ace9-4660-952a-342aafe19661 WaitActive WaitActive WaitActive
Sep 22 23:14:27.699 INFO [2] All repairs completed, exit
Sep 22 23:14:27.699 INFO [2] Starts cmd_loop
The guest has finished waiting for activation
Sep 22 23:14:27.699 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.699 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.699 DEBG IO Read  1000 has deps []
Sep 22 23:14:27.699 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.700 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.700 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.700 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.700 DEBG Read      :1000 deps:[] res:true
Sep 22 23:14:27.700 INFO [0] acc4dcd4-ace9-4660-952a-342aafe19661 (f9138689-ed0d-4ebb-bf00-e5ccee3ed000) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.700 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.700 WARN [0] new RM replaced this: None
Sep 22 23:14:27.700 INFO [0] Starts reconcile loop
Sep 22 23:14:27.700 INFO [1] acc4dcd4-ace9-4660-952a-342aafe19661 (f9138689-ed0d-4ebb-bf00-e5ccee3ed000) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.701 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.701 WARN [1] new RM replaced this: None
Sep 22 23:14:27.701 INFO [1] Starts reconcile loop
Sep 22 23:14:27.701 INFO [2] acc4dcd4-ace9-4660-952a-342aafe19661 (f9138689-ed0d-4ebb-bf00-e5ccee3ed000) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.701 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.701 DEBG Read      :1000 deps:[] res:true
Sep 22 23:14:27.701 WARN [2] new RM replaced this: None
Sep 22 23:14:27.701 INFO [2] Starts reconcile loop
Sep 22 23:14:27.701 INFO [0] 127.0.0.1:63965 task reports connection:true
Sep 22 23:14:27.701 INFO acc4dcd4-ace9-4660-952a-342aafe19661 WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:14:27.701 INFO [0]R  flush_numbers: [0, 0]
Sep 22 23:14:27.701 INFO [0]R  generation: [0, 0]
Sep 22 23:14:27.701 INFO [0]R  dirty: [false, false]
Sep 22 23:14:27.701 INFO [1]R  flush_numbers: [0, 0]
Sep 22 23:14:27.701 INFO [1]R  generation: [0, 0]
Sep 22 23:14:27.701 INFO [1]R  dirty: [false, false]
Sep 22 23:14:27.701 INFO [2]R  flush_numbers: [0, 0]
Sep 22 23:14:27.701 INFO [2]R  generation: [0, 0]
Sep 22 23:14:27.701 INFO [2]R  dirty: [false, false]
Sep 22 23:14:27.701 INFO Max found gen is 1
Sep 22 23:14:27.701 INFO Generation requested: 1 >= found:1
Sep 22 23:14:27.701 INFO Next flush: 1
Sep 22 23:14:27.701 INFO All extents match
Sep 22 23:14:27.701 INFO No downstairs repair required
Sep 22 23:14:27.701 INFO No initial repair work was required
Sep 22 23:14:27.701 INFO Set Downstairs and Upstairs active
Sep 22 23:14:27.701 INFO acc4dcd4-ace9-4660-952a-342aafe19661 is now active with session: f9138689-ed0d-4ebb-bf00-e5ccee3ed000
Sep 22 23:14:27.701 INFO acc4dcd4-ace9-4660-952a-342aafe19661 Set Active after no repair
Sep 22 23:14:27.701 INFO Notify all downstairs, region set compare is done.
Sep 22 23:14:27.701 INFO Set check for repair
Sep 22 23:14:27.701 INFO [1] 127.0.0.1:34981 task reports connection:true
Sep 22 23:14:27.701 INFO acc4dcd4-ace9-4660-952a-342aafe19661 Active Active Active
Sep 22 23:14:27.701 INFO Set check for repair
Sep 22 23:14:27.701 DEBG Read      :1000 deps:[] res:true
Sep 22 23:14:27.701 INFO [2] 127.0.0.1:39833 task reports connection:true
Sep 22 23:14:27.701 INFO acc4dcd4-ace9-4660-952a-342aafe19661 Active Active Active
Sep 22 23:14:27.701 INFO Set check for repair
Sep 22 23:14:27.701 INFO [0] received reconcile message
Sep 22 23:14:27.701 INFO [0] All repairs completed, exit
Sep 22 23:14:27.701 INFO [0] Starts cmd_loop
Sep 22 23:14:27.701 INFO UUID: 6fdc0eed-5ffb-4659-a839-2533f07ab2b0
Sep 22 23:14:27.701 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.701 INFO [1] received reconcile message
Sep 22 23:14:27.702 INFO [1] All repairs completed, exit
Sep 22 23:14:27.702 INFO [1] Starts cmd_loop
Sep 22 23:14:27.702 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.702 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.702 INFO Using address: 127.0.0.1:37634, task: main
Sep 22 23:14:27.702 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.702 INFO [2] received reconcile message
Sep 22 23:14:27.702 INFO Opened existing region file "/tmp/downstairs-sXZCeOZr/region.json"
Sep 22 23:14:27.702 INFO Database read version 1
Sep 22 23:14:27.702 INFO [2] All repairs completed, exit
Sep 22 23:14:27.702 INFO Database write version 1
Sep 22 23:14:27.702 INFO [2] Starts cmd_loop
The guest has finished waiting for activation
Sep 22 23:14:27.702 INFO UUID: 7879f4e7-8bf5-479b-b3e8-105e87c0dd1b
Sep 22 23:14:27.702 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.702 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.702 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.702 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.702 INFO Using address: 127.0.0.1:43456, task: main
Sep 22 23:14:27.702 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:41491, task: repair
Sep 22 23:14:27.702 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:41491, task: repair
Sep 22 23:14:27.702 INFO listening, local_addr: 127.0.0.1:41491, task: repair
Sep 22 23:14:27.702 DEBG IO Read  1000 has deps []
Sep 22 23:14:27.702 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.702 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:41491, task: repair
Sep 22 23:14:27.702 INFO Using repair address: 127.0.0.1:41491, task: main
Sep 22 23:14:27.702 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.702 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:56401, task: repair
Sep 22 23:14:27.702 DEBG [0] Read AckReady 1000, : downstairs
Sep 22 23:14:27.702 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:56401, task: repair
Sep 22 23:14:27.703 INFO listening, local_addr: 127.0.0.1:56401, task: repair
Sep 22 23:14:27.703 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.703 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:56401, task: repair
Sep 22 23:14:27.703 DEBG [1] Read already AckReady 1000, : downstairs
Sep 22 23:14:27.703 INFO Using repair address: 127.0.0.1:56401, task: main
Sep 22 23:14:27.703 INFO Created new region file "/tmp/downstairs-AYTaLROs/region.json"
Sep 22 23:14:27.703 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.703 DEBG [2] Read already AckReady 1000, : downstairs
Sep 22 23:14:27.703 DEBG up_ds_listen was notified
Sep 22 23:14:27.703 DEBG up_ds_listen process 1000
Sep 22 23:14:27.703 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:14:27.703 DEBG Read      :1000 deps:[] res:true
Sep 22 23:14:27.704 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.704 INFO Upstairs starts
Sep 22 23:14:27.704 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:14:27.704 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:14:27.704 INFO Crucible stats registered with UUID: 07dcd1dc-bd26-48c4-9d8c-b1cefc78367c
Sep 22 23:14:27.704 INFO Crucible 07dcd1dc-bd26-48c4-9d8c-b1cefc78367c has session id: ea05610f-f165-4ac5-bd8c-225d17886309
Sep 22 23:14:27.704 DEBG Read      :1000 deps:[] res:true
Sep 22 23:14:27.704 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.704 INFO Opened existing region file "/tmp/downstairs-cvtdcEe4/region.json"
Sep 22 23:14:27.704 INFO Database read version 1
Sep 22 23:14:27.704 INFO Database write version 1
Sep 22 23:14:27.704 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.704 INFO UUID: d2794c71-5659-445d-8650-5f7a1461a2ed
Sep 22 23:14:27.704 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.704 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.704 DEBG Read      :1000 deps:[] res:true
Sep 22 23:14:27.704 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.704 INFO [0] connecting to 127.0.0.1:41733, looper: 0
Sep 22 23:14:27.704 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.704 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.704 INFO Using address: 127.0.0.1:33459, task: main
Sep 22 23:14:27.705 INFO [1] connecting to 127.0.0.1:49579, looper: 1
Sep 22 23:14:27.705 INFO [2] connecting to 127.0.0.1:43456, looper: 2
Sep 22 23:14:27.705 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.705 INFO up_listen starts, task: up_listen
Sep 22 23:14:27.705 INFO Wait for all three downstairs to come online
Sep 22 23:14:27.705 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:35294, task: repair
Sep 22 23:14:27.705 INFO Flush timeout: 0.5
Sep 22 23:14:27.705 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:35294, task: repair
Sep 22 23:14:27.705 INFO Scrub check for 1965c3d3-3628-4d17-8184-b7c7fb3d1c3b
Sep 22 23:14:27.705 INFO listening, local_addr: 127.0.0.1:35294, task: repair
Sep 22 23:14:27.705 INFO Scrub for 1965c3d3-3628-4d17-8184-b7c7fb3d1c3b begins
Sep 22 23:14:27.705 INFO Scrub with total_size:5120 block_size:512
Sep 22 23:14:27.705 INFO Scrubs from block 0 to 10 in (256) 131072 size IOs pm:0
Sep 22 23:14:27.705 INFO Adjust block_count to 10 at offset 0
Sep 22 23:14:27.705 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:35294, task: repair
Sep 22 23:14:27.705 INFO Using repair address: 127.0.0.1:35294, task: main
Sep 22 23:14:27.705 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.705 INFO accepted connection from 127.0.0.1:56638, task: main
Sep 22 23:14:27.705 INFO accepted connection from 127.0.0.1:51008, task: main
Sep 22 23:14:27.705 INFO accepted connection from 127.0.0.1:44546, task: main
Sep 22 23:14:27.705 DEBG [0] Read AckReady 1000, : downstairs
Sep 22 23:14:27.705 INFO [0] 07dcd1dc-bd26-48c4-9d8c-b1cefc78367c looper connected, looper: 0
Sep 22 23:14:27.706 INFO [0] Proc runs for 127.0.0.1:41733 in state New
Sep 22 23:14:27.706 DEBG [1] Read already AckReady 1000, : downstairs
Sep 22 23:14:27.706 INFO [2] 07dcd1dc-bd26-48c4-9d8c-b1cefc78367c looper connected, looper: 2
Sep 22 23:14:27.706 INFO [2] Proc runs for 127.0.0.1:43456 in state New
Sep 22 23:14:27.706 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.706 INFO [1] 07dcd1dc-bd26-48c4-9d8c-b1cefc78367c looper connected, looper: 1
Sep 22 23:14:27.706 INFO [1] Proc runs for 127.0.0.1:49579 in state New
Sep 22 23:14:27.706 DEBG [2] Read already AckReady 1000, : downstairs
Sep 22 23:14:27.706 INFO Created new region file "/tmp/downstairs-dlM6zFZd/region.json"
Sep 22 23:14:27.706 DEBG up_ds_listen was notified
Sep 22 23:14:27.706 DEBG up_ds_listen process 1000
Sep 22 23:14:27.706 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:14:27.706 INFO Connection request from 07dcd1dc-bd26-48c4-9d8c-b1cefc78367c with version 4, task: proc
Sep 22 23:14:27.706 INFO upstairs UpstairsConnection { upstairs_id: 07dcd1dc-bd26-48c4-9d8c-b1cefc78367c, session_id: d464775d-265a-4fbf-a35a-512bcc66cbcc, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.706 INFO Connection request from 07dcd1dc-bd26-48c4-9d8c-b1cefc78367c with version 4, task: proc
Sep 22 23:14:27.706 INFO upstairs UpstairsConnection { upstairs_id: 07dcd1dc-bd26-48c4-9d8c-b1cefc78367c, session_id: d464775d-265a-4fbf-a35a-512bcc66cbcc, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.706 INFO Connection request from 07dcd1dc-bd26-48c4-9d8c-b1cefc78367c with version 4, task: proc
Sep 22 23:14:27.706 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.706 INFO upstairs UpstairsConnection { upstairs_id: 07dcd1dc-bd26-48c4-9d8c-b1cefc78367c, session_id: d464775d-265a-4fbf-a35a-512bcc66cbcc, gen: 1 } connected, version 4, task: proc
The guest has requested activation
Sep 22 23:14:27.707 INFO [0] 07dcd1dc-bd26-48c4-9d8c-b1cefc78367c (d464775d-265a-4fbf-a35a-512bcc66cbcc) New New New ds_transition to WaitActive
Sep 22 23:14:27.707 INFO [0] Transition from New to WaitActive
Sep 22 23:14:27.707 INFO UUID: f4dd1f9e-c133-4738-8e4e-8aad0cee7c95
Sep 22 23:14:27.707 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.707 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.707 INFO Opened existing region file "/tmp/downstairs-AYTaLROs/region.json"
Sep 22 23:14:27.707 INFO [2] 07dcd1dc-bd26-48c4-9d8c-b1cefc78367c (d464775d-265a-4fbf-a35a-512bcc66cbcc) WaitActive New New ds_transition to WaitActive
Sep 22 23:14:27.707 INFO Database read version 1
Sep 22 23:14:27.707 INFO Database write version 1
Sep 22 23:14:27.707 INFO [2] Transition from New to WaitActive
Sep 22 23:14:27.707 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.707 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.707 INFO Using address: 127.0.0.1:58860, task: main
Sep 22 23:14:27.707 INFO [1] 07dcd1dc-bd26-48c4-9d8c-b1cefc78367c (d464775d-265a-4fbf-a35a-512bcc66cbcc) WaitActive New WaitActive ds_transition to WaitActive
Sep 22 23:14:27.707 INFO [1] Transition from New to WaitActive
Sep 22 23:14:27.707 INFO 07dcd1dc-bd26-48c4-9d8c-b1cefc78367c active request set
Sep 22 23:14:27.707 INFO [0] received activate with gen 1
Sep 22 23:14:27.707 INFO [0] client got ds_active_rx, promote! session d464775d-265a-4fbf-a35a-512bcc66cbcc
Sep 22 23:14:27.707 INFO [1] received activate with gen 1
Sep 22 23:14:27.707 INFO [1] client got ds_active_rx, promote! session d464775d-265a-4fbf-a35a-512bcc66cbcc
Sep 22 23:14:27.707 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.707 INFO [2] received activate with gen 1
Sep 22 23:14:27.707 DEBG IO Write 1001 has deps [JobId(1000)]
Sep 22 23:14:27.707 INFO [2] client got ds_active_rx, promote! session d464775d-265a-4fbf-a35a-512bcc66cbcc
Sep 22 23:14:27.707 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:41513, task: repair
Sep 22 23:14:27.707 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:41513, task: repair
Sep 22 23:14:27.707 INFO listening, local_addr: 127.0.0.1:41513, task: repair
Sep 22 23:14:27.707 INFO UpstairsConnection { upstairs_id: 07dcd1dc-bd26-48c4-9d8c-b1cefc78367c, session_id: d464775d-265a-4fbf-a35a-512bcc66cbcc, gen: 1 } is now active (read-write)
Sep 22 23:14:27.708 INFO UpstairsConnection { upstairs_id: 07dcd1dc-bd26-48c4-9d8c-b1cefc78367c, session_id: d464775d-265a-4fbf-a35a-512bcc66cbcc, gen: 1 } is now active (read-write)
Sep 22 23:14:27.708 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:41513, task: repair
Sep 22 23:14:27.708 INFO Using repair address: 127.0.0.1:41513, task: main
Sep 22 23:14:27.708 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.708 INFO UpstairsConnection { upstairs_id: 07dcd1dc-bd26-48c4-9d8c-b1cefc78367c, session_id: d464775d-265a-4fbf-a35a-512bcc66cbcc, gen: 1 } is now active (read-write)
Sep 22 23:14:27.708 INFO [0] downstairs client at 127.0.0.1:41733 has UUID e624c1bd-6ca3-47e5-9bce-8c13d262fb0f
Sep 22 23:14:27.708 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: e624c1bd-6ca3-47e5-9bce-8c13d262fb0f, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.708 INFO 07dcd1dc-bd26-48c4-9d8c-b1cefc78367c WaitActive WaitActive WaitActive
Sep 22 23:14:27.708 INFO [2] downstairs client at 127.0.0.1:43456 has UUID 7879f4e7-8bf5-479b-b3e8-105e87c0dd1b
Sep 22 23:14:27.708 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 7879f4e7-8bf5-479b-b3e8-105e87c0dd1b, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.708 INFO 07dcd1dc-bd26-48c4-9d8c-b1cefc78367c WaitActive WaitActive WaitActive
Sep 22 23:14:27.708 INFO [1] downstairs client at 127.0.0.1:49579 has UUID c415f140-2b86-4861-a661-215ad49ee0b4
Sep 22 23:14:27.708 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: c415f140-2b86-4861-a661-215ad49ee0b4, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.709 INFO 07dcd1dc-bd26-48c4-9d8c-b1cefc78367c WaitActive WaitActive WaitActive
Sep 22 23:14:27.709 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.709 INFO Created new region file "/tmp/downstairs-wC55dsN5/region.json"
Sep 22 23:14:27.709 INFO UUID: b556d7b2-2d5c-488b-aa20-a0b62e7ef0f0
Sep 22 23:14:27.709 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.709 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.709 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.709 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.709 INFO Using address: 127.0.0.1:62359, task: main
Sep 22 23:14:27.709 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.709 DEBG IO Write 1001 has deps [JobId(1000)]
Sep 22 23:14:27.709 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.709 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.709 DEBG up_ds_listen was notified
Sep 22 23:14:27.709 DEBG up_ds_listen process 1001
Sep 22 23:14:27.709 DEBG [A] ack job 1001:2, : downstairs
Sep 22 23:14:27.709 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:65391, task: repair
Sep 22 23:14:27.709 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.709 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:65391, task: repair
Sep 22 23:14:27.709 INFO listening, local_addr: 127.0.0.1:65391, task: repair
Sep 22 23:14:27.709 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.710 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:65391, task: repair
Sep 22 23:14:27.710 INFO Using repair address: 127.0.0.1:65391, task: main
Sep 22 23:14:27.710 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.710 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.710 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.710 INFO [0] 07dcd1dc-bd26-48c4-9d8c-b1cefc78367c (d464775d-265a-4fbf-a35a-512bcc66cbcc) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.710 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.710 WARN [0] new RM replaced this: None
Sep 22 23:14:27.710 INFO [0] Starts reconcile loop
Sep 22 23:14:27.710 INFO [2] 07dcd1dc-bd26-48c4-9d8c-b1cefc78367c (d464775d-265a-4fbf-a35a-512bcc66cbcc) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.710 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.710 WARN [2] new RM replaced this: None
Sep 22 23:14:27.710 INFO [2] Starts reconcile loop
Sep 22 23:14:27.710 INFO Upstairs starts
Sep 22 23:14:27.710 INFO [1] 07dcd1dc-bd26-48c4-9d8c-b1cefc78367c (d464775d-265a-4fbf-a35a-512bcc66cbcc) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum
Sep 22 23:14:27.710 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:14:27.710 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.710 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:14:27.710 WARN [1] new RM replaced this: None
Sep 22 23:14:27.711 INFO [1] Starts reconcile loop
Sep 22 23:14:27.711 INFO Crucible stats registered with UUID: adfa7dbe-ca19-48c5-8ab5-25a64ef3a728
Sep 22 23:14:27.711 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.711 INFO Crucible adfa7dbe-ca19-48c5-8ab5-25a64ef3a728 has session id: 28156900-b93f-495a-b8e4-f44cac9dd065
Sep 22 23:14:27.711 INFO Opened existing region file "/tmp/downstairs-dlM6zFZd/region.json"
Sep 22 23:14:27.711 INFO [0] 127.0.0.1:41733 task reports connection:true
Sep 22 23:14:27.711 INFO Database read version 1
Sep 22 23:14:27.711 INFO 07dcd1dc-bd26-48c4-9d8c-b1cefc78367c WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:14:27.711 INFO Database write version 1
Sep 22 23:14:27.711 INFO [0]R  flush_numbers: [0, 0]
Sep 22 23:14:27.711 INFO [0]R  generation: [0, 0]
Sep 22 23:14:27.711 INFO [0]R  dirty: [false, false]
Sep 22 23:14:27.711 INFO [1]R  flush_numbers: [0, 0]
Sep 22 23:14:27.711 INFO [1]R  generation: [0, 0]
Sep 22 23:14:27.711 INFO [1]R  dirty: [false, false]
Sep 22 23:14:27.711 INFO [2]R  flush_numbers: [0, 0]
Sep 22 23:14:27.711 INFO [2]R  generation: [0, 0]
Sep 22 23:14:27.711 INFO [2]R  dirty: [false, false]
Sep 22 23:14:27.711 INFO Max found gen is 1
Sep 22 23:14:27.711 INFO Generation requested: 1 >= found:1
Sep 22 23:14:27.711 INFO Next flush: 1
Sep 22 23:14:27.711 INFO All extents match
Sep 22 23:14:27.711 INFO No downstairs repair required
Sep 22 23:14:27.711 INFO No initial repair work was required
Sep 22 23:14:27.711 INFO Set Downstairs and Upstairs active
Sep 22 23:14:27.711 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.711 INFO 07dcd1dc-bd26-48c4-9d8c-b1cefc78367c is now active with session: d464775d-265a-4fbf-a35a-512bcc66cbcc
Sep 22 23:14:27.711 INFO 07dcd1dc-bd26-48c4-9d8c-b1cefc78367c Set Active after no repair
Sep 22 23:14:27.711 INFO Notify all downstairs, region set compare is done.
Sep 22 23:14:27.711 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.711 INFO Set check for repair
Sep 22 23:14:27.711 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.711 INFO [2] 127.0.0.1:43456 task reports connection:true
Sep 22 23:14:27.711 INFO 07dcd1dc-bd26-48c4-9d8c-b1cefc78367c Active Active Active
Sep 22 23:14:27.711 INFO [0] connecting to 127.0.0.1:59250, looper: 0
Sep 22 23:14:27.711 INFO Set check for repair
Sep 22 23:14:27.711 INFO [1] 127.0.0.1:49579 task reports connection:true
Sep 22 23:14:27.711 INFO 07dcd1dc-bd26-48c4-9d8c-b1cefc78367c Active Active Active
Sep 22 23:14:27.711 INFO Set check for repair
Sep 22 23:14:27.711 INFO [1] connecting to 127.0.0.1:37634, looper: 1
Sep 22 23:14:27.711 INFO [0] received reconcile message
Sep 22 23:14:27.711 INFO [0] All repairs completed, exit
Sep 22 23:14:27.711 INFO [0] Starts cmd_loop
Sep 22 23:14:27.711 INFO [2] connecting to 127.0.0.1:62359, looper: 2
Sep 22 23:14:27.711 INFO [1] received reconcile message
Sep 22 23:14:27.711 INFO [1] All repairs completed, exit
Sep 22 23:14:27.712 INFO up_listen starts, task: up_listen
Sep 22 23:14:27.712 INFO Wait for all three downstairs to come online
Sep 22 23:14:27.712 INFO Flush timeout: 0.5
Sep 22 23:14:27.712 INFO [1] Starts cmd_loop
Sep 22 23:14:27.712 INFO [2] received reconcile message
Sep 22 23:14:27.712 INFO [2] All repairs completed, exit
Sep 22 23:14:27.712 INFO [2] Starts cmd_loop
Sep 22 23:14:27.712 DEBG up_ds_listen was notified
The guest has finished waiting for activation
Sep 22 23:14:27.712 DEBG up_ds_listen process 1001
Sep 22 23:14:27.712 INFO accepted connection from 127.0.0.1:47782, task: main
Sep 22 23:14:27.712 DEBG [A] ack job 1001:2, : downstairs
Sep 22 23:14:27.712 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.712 INFO accepted connection from 127.0.0.1:51530, task: main
Sep 22 23:14:27.712 INFO accepted connection from 127.0.0.1:64754, task: main
Sep 22 23:14:27.712 INFO Scrub at offset 10/10 sp:10
Sep 22 23:14:27.712 INFO [0] adfa7dbe-ca19-48c5-8ab5-25a64ef3a728 looper connected, looper: 0
Sep 22 23:14:27.712 INFO [0] Proc runs for 127.0.0.1:59250 in state New
Sep 22 23:14:27.712 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.712 INFO Opened existing region file "/tmp/downstairs-wC55dsN5/region.json"
Sep 22 23:14:27.712 INFO Database read version 1
Sep 22 23:14:27.712 INFO Database write version 1
Sep 22 23:14:27.712 INFO [1] adfa7dbe-ca19-48c5-8ab5-25a64ef3a728 looper connected, looper: 1
Sep 22 23:14:27.712 DEBG IO Read  1000 has deps []
Sep 22 23:14:27.712 INFO [1] Proc runs for 127.0.0.1:37634 in state New
Sep 22 23:14:27.712 INFO [2] adfa7dbe-ca19-48c5-8ab5-25a64ef3a728 looper connected, looper: 2
Sep 22 23:14:27.712 INFO [2] Proc runs for 127.0.0.1:62359 in state New
Sep 22 23:14:27.713 INFO Connection request from adfa7dbe-ca19-48c5-8ab5-25a64ef3a728 with version 4, task: proc
Sep 22 23:14:27.713 INFO upstairs UpstairsConnection { upstairs_id: adfa7dbe-ca19-48c5-8ab5-25a64ef3a728, session_id: 2dd2b5b0-82b1-4771-854f-f648a24d146f, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.713 INFO Connection request from adfa7dbe-ca19-48c5-8ab5-25a64ef3a728 with version 4, task: proc
Sep 22 23:14:27.713 INFO upstairs UpstairsConnection { upstairs_id: adfa7dbe-ca19-48c5-8ab5-25a64ef3a728, session_id: 2dd2b5b0-82b1-4771-854f-f648a24d146f, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.713 DEBG Write     :1001 deps:[JobId(1000)] res:true
Sep 22 23:14:27.713 INFO Connection request from adfa7dbe-ca19-48c5-8ab5-25a64ef3a728 with version 4, task: proc
Sep 22 23:14:27.713 INFO upstairs UpstairsConnection { upstairs_id: adfa7dbe-ca19-48c5-8ab5-25a64ef3a728, session_id: 2dd2b5b0-82b1-4771-854f-f648a24d146f, gen: 1 } connected, version 4, task: proc
The guest has requested activation
Sep 22 23:14:27.713 DEBG Read      :1000 deps:[] res:true
Sep 22 23:14:27.713 INFO Scrub 1965c3d3-3628-4d17-8184-b7c7fb3d1c3b done in 0 seconds. Retries:0 scrub_size:5120 size:10 pause_milli:0
Sep 22 23:14:27.713 INFO UUID: a9ddcb17-2090-44c0-b9dd-e7df3a8e01e7
Sep 22 23:14:27.713 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.713 INFO [0] adfa7dbe-ca19-48c5-8ab5-25a64ef3a728 (2dd2b5b0-82b1-4771-854f-f648a24d146f) New New New ds_transition to WaitActive
Sep 22 23:14:27.713 INFO [0] Transition from New to WaitActive
Sep 22 23:14:27.713 DEBG IO Flush 1002 has deps [JobId(1001), JobId(1000)]
Sep 22 23:14:27.713 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.713 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.713 INFO Using address: 127.0.0.1:60290, task: main
Sep 22 23:14:27.713 INFO [1] adfa7dbe-ca19-48c5-8ab5-25a64ef3a728 (2dd2b5b0-82b1-4771-854f-f648a24d146f) WaitActive New New ds_transition to WaitActive
Sep 22 23:14:27.713 INFO [1] Transition from New to WaitActive
Sep 22 23:14:27.713 INFO [2] adfa7dbe-ca19-48c5-8ab5-25a64ef3a728 (2dd2b5b0-82b1-4771-854f-f648a24d146f) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:14:27.713 INFO [2] Transition from New to WaitActive
Sep 22 23:14:27.714 INFO adfa7dbe-ca19-48c5-8ab5-25a64ef3a728 active request set
Sep 22 23:14:27.714 INFO [0] received activate with gen 1
Sep 22 23:14:27.714 INFO [0] client got ds_active_rx, promote! session 2dd2b5b0-82b1-4771-854f-f648a24d146f
Sep 22 23:14:27.714 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.714 DEBG Read      :1000 deps:[] res:true
Sep 22 23:14:27.714 INFO [1] received activate with gen 1
Sep 22 23:14:27.714 INFO [1] client got ds_active_rx, promote! session 2dd2b5b0-82b1-4771-854f-f648a24d146f
Sep 22 23:14:27.714 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:63842, task: repair
Sep 22 23:14:27.714 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:63842, task: repair
Sep 22 23:14:27.714 INFO [2] received activate with gen 1
Sep 22 23:14:27.714 DEBG Write     :1001 deps:[JobId(1000)] res:true
Sep 22 23:14:27.714 INFO [2] client got ds_active_rx, promote! session 2dd2b5b0-82b1-4771-854f-f648a24d146f
Sep 22 23:14:27.714 INFO listening, local_addr: 127.0.0.1:63842, task: repair
Sep 22 23:14:27.714 INFO UpstairsConnection { upstairs_id: adfa7dbe-ca19-48c5-8ab5-25a64ef3a728, session_id: 2dd2b5b0-82b1-4771-854f-f648a24d146f, gen: 1 } is now active (read-write)
Sep 22 23:14:27.714 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:63842, task: repair
Sep 22 23:14:27.714 INFO Using repair address: 127.0.0.1:63842, task: main
Sep 22 23:14:27.714 DEBG Read      :1000 deps:[] res:true
Sep 22 23:14:27.714 INFO UpstairsConnection { upstairs_id: adfa7dbe-ca19-48c5-8ab5-25a64ef3a728, session_id: 2dd2b5b0-82b1-4771-854f-f648a24d146f, gen: 1 } is now active (read-write)
Sep 22 23:14:27.714 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.714 INFO UUID: e9a979b4-d43f-404b-9e3f-1f402040beec
Sep 22 23:14:27.714 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.714 INFO UpstairsConnection { upstairs_id: adfa7dbe-ca19-48c5-8ab5-25a64ef3a728, session_id: 2dd2b5b0-82b1-4771-854f-f648a24d146f, gen: 1 } is now active (read-write)
Sep 22 23:14:27.714 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.714 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.714 INFO Using address: 127.0.0.1:37936, task: main
Sep 22 23:14:27.715 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.715 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:60163, task: repair
Sep 22 23:14:27.715 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:60163, task: repair
Sep 22 23:14:27.715 INFO listening, local_addr: 127.0.0.1:60163, task: repair
Sep 22 23:14:27.715 INFO [0] downstairs client at 127.0.0.1:59250 has UUID d7e92243-263d-4c5e-b8c0-b3bf2546a302
Sep 22 23:14:27.715 DEBG Write     :1001 deps:[JobId(1000)] res:true
Sep 22 23:14:27.715 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: d7e92243-263d-4c5e-b8c0-b3bf2546a302, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.715 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:60163, task: repair
Sep 22 23:14:27.715 INFO adfa7dbe-ca19-48c5-8ab5-25a64ef3a728 WaitActive WaitActive WaitActive
Sep 22 23:14:27.715 INFO Using repair address: 127.0.0.1:60163, task: main
Sep 22 23:14:27.715 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.715 INFO [1] downstairs client at 127.0.0.1:37634 has UUID 6fdc0eed-5ffb-4659-a839-2533f07ab2b0
Sep 22 23:14:27.715 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 6fdc0eed-5ffb-4659-a839-2533f07ab2b0, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.715 INFO adfa7dbe-ca19-48c5-8ab5-25a64ef3a728 WaitActive WaitActive WaitActive
Sep 22 23:14:27.715 INFO Upstairs starts
Sep 22 23:14:27.715 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:14:27.715 INFO [2] downstairs client at 127.0.0.1:62359 has UUID b556d7b2-2d5c-488b-aa20-a0b62e7ef0f0
Sep 22 23:14:27.715 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:14:27.715 DEBG IO Read  1002 has deps [JobId(1001)]
Sep 22 23:14:27.715 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: b556d7b2-2d5c-488b-aa20-a0b62e7ef0f0, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.715 INFO Crucible stats registered with UUID: aafec2ff-66c0-4ac7-aa5b-9f14a05b68e7
Sep 22 23:14:27.715 INFO Crucible aafec2ff-66c0-4ac7-aa5b-9f14a05b68e7 has session id: c6d467f8-efa1-4cc2-8786-807418938ca1
Sep 22 23:14:27.715 INFO adfa7dbe-ca19-48c5-8ab5-25a64ef3a728 WaitActive WaitActive WaitActive
Sep 22 23:14:27.715 DEBG [0] Read AckReady 1000, : downstairs
Sep 22 23:14:27.715 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.715 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.715 DEBG Flush     :1002 extent_limit None deps:[JobId(1001), JobId(1000)] res:true f:1 g:1
Sep 22 23:14:27.715 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.716 DEBG [2] Read already AckReady 1000, : downstairs
Sep 22 23:14:27.716 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.716 DEBG Flush     :1002 extent_limit None deps:[JobId(1001), JobId(1000)] res:true f:1 g:1
Sep 22 23:14:27.716 INFO [0] connecting to 127.0.0.1:45743, looper: 0
Sep 22 23:14:27.716 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.716 DEBG Flush     :1002 extent_limit None deps:[JobId(1001), JobId(1000)] res:true f:1 g:1
Sep 22 23:14:27.716 DEBG [1] Read already AckReady 1000, : downstairs
Sep 22 23:14:27.716 DEBG up_ds_listen was notified
Sep 22 23:14:27.716 INFO [1] connecting to 127.0.0.1:33459, looper: 1
Sep 22 23:14:27.716 DEBG up_ds_listen process 1000
Sep 22 23:14:27.716 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:14:27.716 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.716 INFO [2] connecting to 127.0.0.1:60290, looper: 2
Sep 22 23:14:27.716 INFO Upstairs starts
Sep 22 23:14:27.716 DEBG Read      :1002 deps:[JobId(1001)] res:true
Sep 22 23:14:27.716 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:14:27.716 INFO up_listen starts, task: up_listen
Sep 22 23:14:27.716 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:14:27.716 INFO Wait for all three downstairs to come online
Sep 22 23:14:27.716 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.716 INFO Flush timeout: 0.5
Sep 22 23:14:27.716 DEBG up_ds_listen was notified
Sep 22 23:14:27.716 INFO Crucible stats registered with UUID: f134b04d-a090-4ba5-ab19-3a0aa121e3dd
Sep 22 23:14:27.716 DEBG up_ds_listen process 1002
Sep 22 23:14:27.716 INFO Crucible f134b04d-a090-4ba5-ab19-3a0aa121e3dd has session id: a7c9e24d-671e-41c9-86fd-611b200865be
Sep 22 23:14:27.716 DEBG [A] ack job 1002:3, : downstairs
Sep 22 23:14:27.716 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.716 DEBG [rc] retire 1002 clears [JobId(1000), JobId(1001), JobId(1002)], : downstairs
Sep 22 23:14:27.716 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.716 INFO accepted connection from 127.0.0.1:60164, task: main
Sep 22 23:14:27.716 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.716 DEBG Read      :1002 deps:[JobId(1001)] res:true
Sep 22 23:14:27.716 INFO accepted connection from 127.0.0.1:54301, task: main
Sep 22 23:14:27.717 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.717 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.717 INFO [0] aafec2ff-66c0-4ac7-aa5b-9f14a05b68e7 looper connected, looper: 0
Sep 22 23:14:27.717 INFO [0] Proc runs for 127.0.0.1:45743 in state New
Sep 22 23:14:27.717 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.717 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.717 INFO [0] connecting to 127.0.0.1:41704, looper: 0
Sep 22 23:14:27.717 INFO accepted connection from 127.0.0.1:59457, task: main
Sep 22 23:14:27.717 INFO [1] connecting to 127.0.0.1:58860, looper: 1
Sep 22 23:14:27.717 INFO [0] adfa7dbe-ca19-48c5-8ab5-25a64ef3a728 (2dd2b5b0-82b1-4771-854f-f648a24d146f) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.717 INFO [1] aafec2ff-66c0-4ac7-aa5b-9f14a05b68e7 looper connected, looper: 1
Sep 22 23:14:27.717 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.717 WARN [0] new RM replaced this: None
Sep 22 23:14:27.717 DEBG Read      :1002 deps:[JobId(1001)] res:true
Sep 22 23:14:27.717 INFO [1] Proc runs for 127.0.0.1:33459 in state New
Sep 22 23:14:27.717 INFO [0] Starts reconcile loop
Sep 22 23:14:27.717 INFO [2] aafec2ff-66c0-4ac7-aa5b-9f14a05b68e7 looper connected, looper: 2
Sep 22 23:14:27.717 INFO [2] connecting to 127.0.0.1:37936, looper: 2
Sep 22 23:14:27.717 INFO [1] adfa7dbe-ca19-48c5-8ab5-25a64ef3a728 (2dd2b5b0-82b1-4771-854f-f648a24d146f) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.717 INFO [2] Proc runs for 127.0.0.1:60290 in state New
Sep 22 23:14:27.717 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.717 INFO up_listen starts, task: up_listen
Sep 22 23:14:27.717 WARN [1] new RM replaced this: None
Sep 22 23:14:27.717 INFO Wait for all three downstairs to come online
Sep 22 23:14:27.717 INFO Flush timeout: 0.5
Sep 22 23:14:27.717 INFO [1] Starts reconcile loop
Sep 22 23:14:27.717 INFO [2] adfa7dbe-ca19-48c5-8ab5-25a64ef3a728 (2dd2b5b0-82b1-4771-854f-f648a24d146f) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.717 DEBG IO Write 1001 has deps [JobId(1000)]
Sep 22 23:14:27.717 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.717 WARN [2] new RM replaced this: None
Sep 22 23:14:27.717 INFO [2] Starts reconcile loop
Sep 22 23:14:27.717 DEBG up_ds_listen was notified
Sep 22 23:14:27.717 INFO [0] 127.0.0.1:59250 task reports connection:true
Sep 22 23:14:27.717 DEBG up_ds_listen process 1001
Sep 22 23:14:27.717 INFO adfa7dbe-ca19-48c5-8ab5-25a64ef3a728 WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:14:27.718 DEBG [A] ack job 1001:2, : downstairs
Sep 22 23:14:27.718 INFO [0]R  flush_numbers: [0, 0]
Sep 22 23:14:27.718 INFO accepted connection from 127.0.0.1:64265, task: main
Sep 22 23:14:27.718 INFO [0]R  generation: [0, 0]
Sep 22 23:14:27.718 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.718 INFO [0]R  dirty: [false, false]
Sep 22 23:14:27.718 INFO Connection request from aafec2ff-66c0-4ac7-aa5b-9f14a05b68e7 with version 4, task: proc
Sep 22 23:14:27.718 INFO [1]R  flush_numbers: [0, 0]
Sep 22 23:14:27.718 INFO [1]R  generation: [0, 0]
Sep 22 23:14:27.718 INFO upstairs UpstairsConnection { upstairs_id: aafec2ff-66c0-4ac7-aa5b-9f14a05b68e7, session_id: 37685993-17b9-4e43-bd95-ffd5fc39e9bd, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.718 INFO [1]R  dirty: [false, false]
Sep 22 23:14:27.718 INFO accepted connection from 127.0.0.1:39421, task: main
Sep 22 23:14:27.718 INFO [2]R  flush_numbers: [0, 0]
Sep 22 23:14:27.718 INFO [2]R  generation: [0, 0]
Sep 22 23:14:27.718 INFO [2]R  dirty: [false, false]
Sep 22 23:14:27.718 INFO Max found gen is 1
Sep 22 23:14:27.718 INFO accepted connection from 127.0.0.1:65012, task: main
Sep 22 23:14:27.718 INFO Connection request from aafec2ff-66c0-4ac7-aa5b-9f14a05b68e7 with version 4, task: proc
Sep 22 23:14:27.718 INFO Generation requested: 1 >= found:1
Sep 22 23:14:27.718 INFO Next flush: 1
Sep 22 23:14:27.718 INFO upstairs UpstairsConnection { upstairs_id: aafec2ff-66c0-4ac7-aa5b-9f14a05b68e7, session_id: 37685993-17b9-4e43-bd95-ffd5fc39e9bd, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.718 INFO All extents match
Sep 22 23:14:27.718 INFO No downstairs repair required
Sep 22 23:14:27.718 INFO [0] f134b04d-a090-4ba5-ab19-3a0aa121e3dd looper connected, looper: 0
Sep 22 23:14:27.718 INFO No initial repair work was required
Sep 22 23:14:27.718 INFO Set Downstairs and Upstairs active
Sep 22 23:14:27.718 INFO Connection request from aafec2ff-66c0-4ac7-aa5b-9f14a05b68e7 with version 4, task: proc
Sep 22 23:14:27.718 INFO [0] Proc runs for 127.0.0.1:41704 in state New
Sep 22 23:14:27.718 INFO adfa7dbe-ca19-48c5-8ab5-25a64ef3a728 is now active with session: 2dd2b5b0-82b1-4771-854f-f648a24d146f
Sep 22 23:14:27.718 INFO upstairs UpstairsConnection { upstairs_id: aafec2ff-66c0-4ac7-aa5b-9f14a05b68e7, session_id: 37685993-17b9-4e43-bd95-ffd5fc39e9bd, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.718 INFO adfa7dbe-ca19-48c5-8ab5-25a64ef3a728 Set Active after no repair
Sep 22 23:14:27.718 INFO Notify all downstairs, region set compare is done.
Sep 22 23:14:27.718 INFO Set check for repair
Sep 22 23:14:27.718 INFO [1] f134b04d-a090-4ba5-ab19-3a0aa121e3dd looper connected, looper: 1
Sep 22 23:14:27.718 INFO [1] Proc runs for 127.0.0.1:58860 in state New
Sep 22 23:14:27.718 INFO [1] 127.0.0.1:37634 task reports connection:true
The guest has requested activation
Sep 22 23:14:27.718 INFO adfa7dbe-ca19-48c5-8ab5-25a64ef3a728 Active Active Active
Sep 22 23:14:27.718 INFO Set check for repair
Sep 22 23:14:27.718 INFO [2] f134b04d-a090-4ba5-ab19-3a0aa121e3dd looper connected, looper: 2
Sep 22 23:14:27.718 INFO [2] 127.0.0.1:62359 task reports connection:true
Sep 22 23:14:27.718 INFO [0] aafec2ff-66c0-4ac7-aa5b-9f14a05b68e7 (37685993-17b9-4e43-bd95-ffd5fc39e9bd) New New New ds_transition to WaitActive
Sep 22 23:14:27.718 INFO [2] Proc runs for 127.0.0.1:37936 in state New
Sep 22 23:14:27.718 INFO [0] Transition from New to WaitActive
Sep 22 23:14:27.718 INFO adfa7dbe-ca19-48c5-8ab5-25a64ef3a728 Active Active Active
Sep 22 23:14:27.718 INFO Set check for repair
Sep 22 23:14:27.718 INFO Scrub check for a80ca6f9-770a-4008-b246-89463d10a8c7
Sep 22 23:14:27.718 INFO [0] received reconcile message
Sep 22 23:14:27.718 INFO [1] aafec2ff-66c0-4ac7-aa5b-9f14a05b68e7 (37685993-17b9-4e43-bd95-ffd5fc39e9bd) WaitActive New New ds_transition to WaitActive
Sep 22 23:14:27.718 INFO Scrub for a80ca6f9-770a-4008-b246-89463d10a8c7 not required
Sep 22 23:14:27.718 INFO [1] Transition from New to WaitActive
Sep 22 23:14:27.718 INFO [0] All repairs completed, exit
Sep 22 23:14:27.718 INFO [0] Starts cmd_loop
Sep 22 23:14:27.718 INFO [2] aafec2ff-66c0-4ac7-aa5b-9f14a05b68e7 (37685993-17b9-4e43-bd95-ffd5fc39e9bd) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:14:27.718 INFO [2] Transition from New to WaitActive
Sep 22 23:14:27.718 INFO aafec2ff-66c0-4ac7-aa5b-9f14a05b68e7 active request set
Sep 22 23:14:27.718 INFO [1] received reconcile message
Sep 22 23:14:27.718 INFO [1] All repairs completed, exit
Sep 22 23:14:27.718 INFO [0] received activate with gen 1
Sep 22 23:14:27.718 INFO [0] client got ds_active_rx, promote! session 37685993-17b9-4e43-bd95-ffd5fc39e9bd
Sep 22 23:14:27.718 INFO [1] Starts cmd_loop
Sep 22 23:14:27.719 INFO [1] received activate with gen 1
Sep 22 23:14:27.719 INFO [2] received reconcile message
Sep 22 23:14:27.719 INFO Connection request from f134b04d-a090-4ba5-ab19-3a0aa121e3dd with version 4, task: proc
Sep 22 23:14:27.719 INFO [1] client got ds_active_rx, promote! session 37685993-17b9-4e43-bd95-ffd5fc39e9bd
Sep 22 23:14:27.719 INFO [2] All repairs completed, exit
Sep 22 23:14:27.719 INFO upstairs UpstairsConnection { upstairs_id: f134b04d-a090-4ba5-ab19-3a0aa121e3dd, session_id: 9d45ed58-1efb-4b07-81f4-53421dc0e429, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.719 INFO [2] Starts cmd_loop
Sep 22 23:14:27.719 INFO [2] received activate with gen 1
Sep 22 23:14:27.719 INFO [2] client got ds_active_rx, promote! session 37685993-17b9-4e43-bd95-ffd5fc39e9bd
Sep 22 23:14:27.719 INFO Connection request from f134b04d-a090-4ba5-ab19-3a0aa121e3dd with version 4, task: proc
The guest has finished waiting for activation
Sep 22 23:14:27.719 INFO upstairs UpstairsConnection { upstairs_id: f134b04d-a090-4ba5-ab19-3a0aa121e3dd, session_id: 9d45ed58-1efb-4b07-81f4-53421dc0e429, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.719 INFO UpstairsConnection { upstairs_id: aafec2ff-66c0-4ac7-aa5b-9f14a05b68e7, session_id: 37685993-17b9-4e43-bd95-ffd5fc39e9bd, gen: 1 } is now active (read-write)
Sep 22 23:14:27.719 INFO Connection request from f134b04d-a090-4ba5-ab19-3a0aa121e3dd with version 4, task: proc
Sep 22 23:14:27.719 INFO upstairs UpstairsConnection { upstairs_id: f134b04d-a090-4ba5-ab19-3a0aa121e3dd, session_id: 9d45ed58-1efb-4b07-81f4-53421dc0e429, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.719 INFO UpstairsConnection { upstairs_id: aafec2ff-66c0-4ac7-aa5b-9f14a05b68e7, session_id: 37685993-17b9-4e43-bd95-ffd5fc39e9bd, gen: 1 } is now active (read-write)
The guest has requested activation
Sep 22 23:14:27.719 INFO UpstairsConnection { upstairs_id: aafec2ff-66c0-4ac7-aa5b-9f14a05b68e7, session_id: 37685993-17b9-4e43-bd95-ffd5fc39e9bd, gen: 1 } is now active (read-write)
Sep 22 23:14:27.719 INFO [0] f134b04d-a090-4ba5-ab19-3a0aa121e3dd (9d45ed58-1efb-4b07-81f4-53421dc0e429) New New New ds_transition to WaitActive
Sep 22 23:14:27.719 INFO [0] Transition from New to WaitActive
Sep 22 23:14:27.719 DEBG IO Read  1000 has deps []
Sep 22 23:14:27.719 DEBG IO Write 1003 has deps []
Sep 22 23:14:27.719 INFO [1] f134b04d-a090-4ba5-ab19-3a0aa121e3dd (9d45ed58-1efb-4b07-81f4-53421dc0e429) WaitActive New New ds_transition to WaitActive
Sep 22 23:14:27.719 INFO [1] Transition from New to WaitActive
Sep 22 23:14:27.719 INFO [2] f134b04d-a090-4ba5-ab19-3a0aa121e3dd (9d45ed58-1efb-4b07-81f4-53421dc0e429) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:14:27.719 INFO [2] Transition from New to WaitActive
Sep 22 23:14:27.719 INFO f134b04d-a090-4ba5-ab19-3a0aa121e3dd active request set
Sep 22 23:14:27.720 INFO [0] received activate with gen 1
Sep 22 23:14:27.720 INFO [0] client got ds_active_rx, promote! session 9d45ed58-1efb-4b07-81f4-53421dc0e429
Sep 22 23:14:27.720 INFO [0] downstairs client at 127.0.0.1:45743 has UUID 61ba2500-0b28-4437-bb9f-9b380dfcca32
Sep 22 23:14:27.720 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 61ba2500-0b28-4437-bb9f-9b380dfcca32, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.720 INFO [1] received activate with gen 1
Sep 22 23:14:27.720 INFO [1] client got ds_active_rx, promote! session 9d45ed58-1efb-4b07-81f4-53421dc0e429
Sep 22 23:14:27.720 INFO aafec2ff-66c0-4ac7-aa5b-9f14a05b68e7 WaitActive WaitActive WaitActive
Sep 22 23:14:27.720 DEBG Write     :1001 deps:[JobId(1000)] res:true
Sep 22 23:14:27.720 INFO [2] received activate with gen 1
Sep 22 23:14:27.720 INFO [1] downstairs client at 127.0.0.1:33459 has UUID d2794c71-5659-445d-8650-5f7a1461a2ed
Sep 22 23:14:27.720 INFO [2] client got ds_active_rx, promote! session 9d45ed58-1efb-4b07-81f4-53421dc0e429
Sep 22 23:14:27.720 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: d2794c71-5659-445d-8650-5f7a1461a2ed, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.720 DEBG [0] Read AckReady 1002, : downstairs
Sep 22 23:14:27.720 INFO aafec2ff-66c0-4ac7-aa5b-9f14a05b68e7 WaitActive WaitActive WaitActive
Sep 22 23:14:27.720 INFO UpstairsConnection { upstairs_id: f134b04d-a090-4ba5-ab19-3a0aa121e3dd, session_id: 9d45ed58-1efb-4b07-81f4-53421dc0e429, gen: 1 } is now active (read-write)
Sep 22 23:14:27.720 INFO [2] downstairs client at 127.0.0.1:60290 has UUID a9ddcb17-2090-44c0-b9dd-e7df3a8e01e7
Sep 22 23:14:27.720 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: a9ddcb17-2090-44c0-b9dd-e7df3a8e01e7, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.720 INFO aafec2ff-66c0-4ac7-aa5b-9f14a05b68e7 WaitActive WaitActive WaitActive
Sep 22 23:14:27.720 INFO UpstairsConnection { upstairs_id: f134b04d-a090-4ba5-ab19-3a0aa121e3dd, session_id: 9d45ed58-1efb-4b07-81f4-53421dc0e429, gen: 1 } is now active (read-write)
Sep 22 23:14:27.720 DEBG Read      :1000 deps:[] res:true
Sep 22 23:14:27.720 INFO UpstairsConnection { upstairs_id: f134b04d-a090-4ba5-ab19-3a0aa121e3dd, session_id: 9d45ed58-1efb-4b07-81f4-53421dc0e429, gen: 1 } is now active (read-write)
Sep 22 23:14:27.720 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.720 DEBG Read      :1000 deps:[] res:true
Sep 22 23:14:27.720 DEBG Write     :1001 deps:[JobId(1000)] res:true
Sep 22 23:14:27.720 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.720 DEBG Read      :1000 deps:[] res:true
Sep 22 23:14:27.721 INFO [0] downstairs client at 127.0.0.1:41704 has UUID 1e03bcc8-571c-4b44-a9e5-acc5366adf71
Sep 22 23:14:27.721 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 1e03bcc8-571c-4b44-a9e5-acc5366adf71, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.721 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.721 INFO f134b04d-a090-4ba5-ab19-3a0aa121e3dd WaitActive WaitActive WaitActive
Sep 22 23:14:27.721 INFO [1] downstairs client at 127.0.0.1:58860 has UUID f4dd1f9e-c133-4738-8e4e-8aad0cee7c95
Sep 22 23:14:27.721 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: f4dd1f9e-c133-4738-8e4e-8aad0cee7c95, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.721 INFO f134b04d-a090-4ba5-ab19-3a0aa121e3dd WaitActive WaitActive WaitActive
Sep 22 23:14:27.721 INFO [2] downstairs client at 127.0.0.1:37936 has UUID e9a979b4-d43f-404b-9e3f-1f402040beec
Sep 22 23:14:27.721 DEBG Write     :1001 deps:[JobId(1000)] res:true
Sep 22 23:14:27.721 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: e9a979b4-d43f-404b-9e3f-1f402040beec, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.721 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.721 INFO f134b04d-a090-4ba5-ab19-3a0aa121e3dd WaitActive WaitActive WaitActive
Sep 22 23:14:27.721 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.721 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.721 DEBG [0] Read AckReady 1000, : downstairs
Sep 22 23:14:27.721 DEBG IO Read  1002 has deps [JobId(1001)]
Sep 22 23:14:27.721 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.721 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.721 DEBG [1] Read already AckReady 1000, : downstairs
Sep 22 23:14:27.721 DEBG [1] Read already AckReady 1002, : downstairs
Sep 22 23:14:27.722 DEBG [2] Read already AckReady 1000, : downstairs
Sep 22 23:14:27.722 DEBG up_ds_listen was notified
Sep 22 23:14:27.722 INFO [0] aafec2ff-66c0-4ac7-aa5b-9f14a05b68e7 (37685993-17b9-4e43-bd95-ffd5fc39e9bd) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.722 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.722 DEBG up_ds_listen process 1000
Sep 22 23:14:27.722 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.722 WARN [0] new RM replaced this: None
Sep 22 23:14:27.722 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:14:27.722 INFO [0] Starts reconcile loop
Sep 22 23:14:27.722 INFO [1] aafec2ff-66c0-4ac7-aa5b-9f14a05b68e7 (37685993-17b9-4e43-bd95-ffd5fc39e9bd) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.722 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.722 WARN [1] new RM replaced this: None
Sep 22 23:14:27.722 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.722 INFO [1] Starts reconcile loop
Sep 22 23:14:27.722 INFO [2] aafec2ff-66c0-4ac7-aa5b-9f14a05b68e7 (37685993-17b9-4e43-bd95-ffd5fc39e9bd) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.722 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.722 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.722 WARN [2] new RM replaced this: None
Sep 22 23:14:27.722 INFO [2] Starts reconcile loop
Sep 22 23:14:27.722 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.722 INFO [0] 127.0.0.1:45743 task reports connection:true
Sep 22 23:14:27.722 INFO aafec2ff-66c0-4ac7-aa5b-9f14a05b68e7 WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:14:27.722 INFO [0]R  flush_numbers: [0, 0]
Sep 22 23:14:27.722 INFO [0]R  generation: [0, 0]
Sep 22 23:14:27.722 INFO [0]R  dirty: [false, false]
Sep 22 23:14:27.722 INFO [1]R  flush_numbers: [0, 0]
Sep 22 23:14:27.722 DEBG Read      :1002 deps:[JobId(1001)] res:true
Sep 22 23:14:27.722 INFO [1]R  generation: [0, 0]
Sep 22 23:14:27.722 INFO [1]R  dirty: [false, false]
Sep 22 23:14:27.722 INFO [2]R  flush_numbers: [0, 0]
Sep 22 23:14:27.722 INFO [2]R  generation: [0, 0]
Sep 22 23:14:27.722 INFO [2]R  dirty: [false, false]
Sep 22 23:14:27.722 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.722 INFO Max found gen is 1
Sep 22 23:14:27.722 INFO Generation requested: 1 >= found:1
Sep 22 23:14:27.722 INFO Next flush: 1
Sep 22 23:14:27.722 INFO All extents match
Sep 22 23:14:27.722 INFO No downstairs repair required
Sep 22 23:14:27.722 INFO No initial repair work was required
Sep 22 23:14:27.722 INFO Set Downstairs and Upstairs active
Sep 22 23:14:27.722 INFO aafec2ff-66c0-4ac7-aa5b-9f14a05b68e7 is now active with session: 37685993-17b9-4e43-bd95-ffd5fc39e9bd
Sep 22 23:14:27.722 INFO aafec2ff-66c0-4ac7-aa5b-9f14a05b68e7 Set Active after no repair
Sep 22 23:14:27.722 INFO Notify all downstairs, region set compare is done.
Sep 22 23:14:27.722 INFO [0] f134b04d-a090-4ba5-ab19-3a0aa121e3dd (9d45ed58-1efb-4b07-81f4-53421dc0e429) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.722 INFO Set check for repair
Sep 22 23:14:27.722 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.723 WARN [0] new RM replaced this: None
Sep 22 23:14:27.723 INFO [1] 127.0.0.1:33459 task reports connection:true
Sep 22 23:14:27.723 INFO aafec2ff-66c0-4ac7-aa5b-9f14a05b68e7 Active Active Active
Sep 22 23:14:27.723 DEBG Read      :1002 deps:[JobId(1001)] res:true
Sep 22 23:14:27.723 INFO Set check for repair
Sep 22 23:14:27.723 INFO [0] Starts reconcile loop
Sep 22 23:14:27.723 INFO [2] 127.0.0.1:60290 task reports connection:true
Sep 22 23:14:27.723 INFO aafec2ff-66c0-4ac7-aa5b-9f14a05b68e7 Active Active Active
Sep 22 23:14:27.723 INFO Set check for repair
Sep 22 23:14:27.723 DEBG up_ds_listen was notified
Sep 22 23:14:27.723 INFO [1] f134b04d-a090-4ba5-ab19-3a0aa121e3dd (9d45ed58-1efb-4b07-81f4-53421dc0e429) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.723 INFO [0] received reconcile message
Sep 22 23:14:27.723 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.723 DEBG up_ds_listen process 1003
Sep 22 23:14:27.723 WARN [1] new RM replaced this: None
Sep 22 23:14:27.723 INFO [0] All repairs completed, exit
Sep 22 23:14:27.723 DEBG [A] ack job 1003:4, : downstairs
Sep 22 23:14:27.723 INFO [1] Starts reconcile loop
Sep 22 23:14:27.723 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.723 INFO [0] Starts cmd_loop
Sep 22 23:14:27.723 INFO [2] f134b04d-a090-4ba5-ab19-3a0aa121e3dd (9d45ed58-1efb-4b07-81f4-53421dc0e429) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.723 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.723 WARN [2] new RM replaced this: None
Sep 22 23:14:27.723 DEBG Read      :1002 deps:[JobId(1001)] res:true
Sep 22 23:14:27.723 DEBG IO Read  1001 has deps []
Sep 22 23:14:27.723 INFO [2] Starts reconcile loop
Sep 22 23:14:27.723 INFO [1] received reconcile message
Sep 22 23:14:27.723 INFO [1] All repairs completed, exit
Sep 22 23:14:27.723 INFO [0] 127.0.0.1:41704 task reports connection:true
Sep 22 23:14:27.723 INFO [1] Starts cmd_loop
Sep 22 23:14:27.723 INFO f134b04d-a090-4ba5-ab19-3a0aa121e3dd WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:14:27.723 INFO [0]R  flush_numbers: [0, 0]
Sep 22 23:14:27.723 INFO [0]R  generation: [0, 0]
Sep 22 23:14:27.723 INFO [2] received reconcile message
Sep 22 23:14:27.723 INFO [0]R  dirty: [false, false]
Sep 22 23:14:27.723 DEBG IO Read  1004 has deps [JobId(1003)]
Sep 22 23:14:27.723 INFO [1]R  flush_numbers: [0, 0]
Sep 22 23:14:27.723 INFO [2] All repairs completed, exit
Sep 22 23:14:27.723 INFO [1]R  generation: [0, 0]
Sep 22 23:14:27.723 INFO [1]R  dirty: [false, false]
Sep 22 23:14:27.723 INFO [2] Starts cmd_loop
Sep 22 23:14:27.723 INFO [2]R  flush_numbers: [0, 0]
Sep 22 23:14:27.723 INFO [2]R  generation: [0, 0]
Sep 22 23:14:27.723 DEBG [2] Read already AckReady 1002, : downstairs
Sep 22 23:14:27.723 INFO [2]R  dirty: [false, false]
Sep 22 23:14:27.723 INFO Max found gen is 1
The guest has finished waiting for activation
Sep 22 23:14:27.723 INFO Generation requested: 1 >= found:1
Sep 22 23:14:27.723 DEBG up_ds_listen was notified
Sep 22 23:14:27.723 INFO Next flush: 1
Sep 22 23:14:27.723 DEBG up_ds_listen process 1002
Sep 22 23:14:27.723 INFO All extents match
Sep 22 23:14:27.723 DEBG [A] ack job 1002:3, : downstairs
Sep 22 23:14:27.723 INFO No downstairs repair required
Sep 22 23:14:27.723 INFO No initial repair work was required
Sep 22 23:14:27.723 INFO Set Downstairs and Upstairs active
Sep 22 23:14:27.723 INFO f134b04d-a090-4ba5-ab19-3a0aa121e3dd is now active with session: 9d45ed58-1efb-4b07-81f4-53421dc0e429
Sep 22 23:14:27.723 INFO f134b04d-a090-4ba5-ab19-3a0aa121e3dd Set Active after no repair
Sep 22 23:14:27.723 INFO Notify all downstairs, region set compare is done.
Sep 22 23:14:27.723 INFO Set check for repair
Sep 22 23:14:27.723 INFO [1] 127.0.0.1:58860 task reports connection:true
Sep 22 23:14:27.723 INFO f134b04d-a090-4ba5-ab19-3a0aa121e3dd Active Active Active
Sep 22 23:14:27.724 INFO Set check for repair
Sep 22 23:14:27.724 INFO [2] 127.0.0.1:37936 task reports connection:true
Sep 22 23:14:27.724 INFO f134b04d-a090-4ba5-ab19-3a0aa121e3dd Active Active Active
Sep 22 23:14:27.724 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.724 INFO Set check for repair
Sep 22 23:14:27.724 DEBG Read      :1001 deps:[] res:true
Sep 22 23:14:27.724 INFO [0] received reconcile message
Sep 22 23:14:27.724 INFO [0] All repairs completed, exit
Sep 22 23:14:27.724 INFO [0] Starts cmd_loop
Sep 22 23:14:27.724 DEBG IO Write 1000 has deps []
Sep 22 23:14:27.724 INFO [1] received reconcile message
Sep 22 23:14:27.724 INFO [1] All repairs completed, exit
Sep 22 23:14:27.724 INFO [1] Starts cmd_loop
Sep 22 23:14:27.724 DEBG up_ds_listen was notified
Sep 22 23:14:27.724 DEBG Read      :1004 deps:[JobId(1003)] res:true
Sep 22 23:14:27.724 DEBG Read      :1001 deps:[] res:true
Sep 22 23:14:27.724 INFO [2] received reconcile message
Sep 22 23:14:27.724 DEBG up_ds_listen process 1000
Sep 22 23:14:27.724 INFO [2] All repairs completed, exit
Sep 22 23:14:27.724 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:14:27.724 INFO [2] Starts cmd_loop
Sep 22 23:14:27.724 DEBG up_ds_listen checked 1 jobs, back to waiting
The guest has finished waiting for activation
Sep 22 23:14:27.724 DEBG Read      :1001 deps:[] res:true
Sep 22 23:14:27.724 DEBG IO Read  1000 has deps []
Sep 22 23:14:27.725 DEBG Read      :1004 deps:[JobId(1003)] res:true
Sep 22 23:14:27.725 DEBG [0] Read AckReady 1001, : downstairs
Sep 22 23:14:27.725 DEBG [1] Read already AckReady 1001, : downstairs
Sep 22 23:14:27.725 DEBG [2] Read already AckReady 1001, : downstairs
Sep 22 23:14:27.726 DEBG up_ds_listen was notified
Sep 22 23:14:27.726 DEBG up_ds_listen process 1001
Sep 22 23:14:27.726 DEBG [A] ack job 1001:2, : downstairs
Sep 22 23:14:27.726 DEBG [0] Read AckReady 1002, : downstairs
Sep 22 23:14:27.726 DEBG Write     :1000 deps:[] res:true
Sep 22 23:14:27.726 DEBG Read      :1004 deps:[JobId(1003)] res:true
Sep 22 23:14:27.726 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.726 INFO Scrub check for d2ca4639-e324-4c6c-8e8a-4f9bdf088392
Sep 22 23:14:27.726 INFO Scrub for d2ca4639-e324-4c6c-8e8a-4f9bdf088392 begins
Sep 22 23:14:27.726 INFO Scrub with total_size:2560 block_size:512
Sep 22 23:14:27.726 INFO Scrubs from block 0 to 5 in (256) 131072 size IOs pm:0
Sep 22 23:14:27.726 INFO Adjust block_count to 5 at offset 0
Sep 22 23:14:27.726 DEBG Read      :1000 deps:[] res:true
Sep 22 23:14:27.726 DEBG Write     :1000 deps:[] res:true
Sep 22 23:14:27.727 DEBG Read      :1000 deps:[] res:true
test test::integration_test_region ... ok
Sep 22 23:14:27.727 DEBG Write     :1000 deps:[] res:true
Sep 22 23:14:27.727 DEBG [2] Read already AckReady 1002, : downstairs
Sep 22 23:14:27.727 DEBG Read      :1000 deps:[] res:true
Sep 22 23:14:27.727 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.727 INFO Created new region file "/tmp/downstairs-NzNQXxni/region.json"
Sep 22 23:14:27.727 DEBG IO Write 1001 has deps []
Sep 22 23:14:27.728 DEBG up_ds_listen was notified
Sep 22 23:14:27.728 DEBG up_ds_listen process 1001
Sep 22 23:14:27.728 DEBG [A] ack job 1001:2, : downstairs
Sep 22 23:14:27.728 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.728 DEBG IO Write 1002 has deps [JobId(1000)]
Sep 22 23:14:27.728 INFO Scrub check for 27af30bc-466c-47bd-9576-e5adb1115007
Sep 22 23:14:27.728 INFO Scrub for 27af30bc-466c-47bd-9576-e5adb1115007 begins
Sep 22 23:14:27.728 INFO Scrub with total_size:2560 block_size:512
Sep 22 23:14:27.728 INFO Scrubs from block 0 to 5 in (256) 131072 size IOs pm:0
Sep 22 23:14:27.728 INFO Adjust block_count to 5 at offset 0
Sep 22 23:14:27.728 DEBG [0] Read AckReady 1000, : downstairs
Sep 22 23:14:27.728 DEBG [1] Read already AckReady 1000, : downstairs
Sep 22 23:14:27.729 DEBG [1] Read already AckReady 1002, : downstairs
Sep 22 23:14:27.729 DEBG up_ds_listen was notified
Sep 22 23:14:27.729 DEBG up_ds_listen process 1002
Sep 22 23:14:27.729 DEBG [A] ack job 1002:3, : downstairs
Sep 22 23:14:27.729 DEBG [2] Read already AckReady 1000, : downstairs
Sep 22 23:14:27.729 DEBG up_ds_listen was notified
Sep 22 23:14:27.729 DEBG up_ds_listen process 1000
Sep 22 23:14:27.729 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:14:27.729 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.729 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.729 INFO Opened existing region file "/tmp/downstairs-NzNQXxni/region.json"
Sep 22 23:14:27.729 INFO Database read version 1
Sep 22 23:14:27.729 INFO Database write version 1
Sep 22 23:14:27.729 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.729 DEBG Write     :1001 deps:[] res:true
Sep 22 23:14:27.730 DEBG Write     :1001 deps:[] res:true
Sep 22 23:14:27.730 DEBG [0] Read AckReady 1004, : downstairs
Sep 22 23:14:27.730 DEBG Write     :1001 deps:[] res:true
Sep 22 23:14:27.731 INFO UUID: a0901c3f-7fbf-40b8-85c2-142a27a641c8
Sep 22 23:14:27.731 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.731 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.731 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.731 INFO Using address: 127.0.0.1:35706, task: main
Sep 22 23:14:27.731 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.731 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:53909, task: repair
Sep 22 23:14:27.731 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:53909, task: repair
Sep 22 23:14:27.731 INFO listening, local_addr: 127.0.0.1:53909, task: repair
Sep 22 23:14:27.732 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:53909, task: repair
Sep 22 23:14:27.732 DEBG IO Write 1002 has deps [JobId(1000)]
Sep 22 23:14:27.732 INFO Using repair address: 127.0.0.1:53909, task: main
Sep 22 23:14:27.732 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.732 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.732 INFO Created new region file "/tmp/downstairs-JJG0Y7Ly/region.json"
Sep 22 23:14:27.732 DEBG up_ds_listen was notified
Sep 22 23:14:27.732 DEBG up_ds_listen process 1002
Sep 22 23:14:27.732 DEBG [A] ack job 1002:3, : downstairs
Sep 22 23:14:27.732 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.732 INFO Scrub at offset 5/5 sp:5
Sep 22 23:14:27.732 DEBG [1] Read already AckReady 1004, : downstairs
test test::integration_test_scrub_no_rop ... ok
Sep 22 23:14:27.733 DEBG IO Write 1001 has deps [JobId(1000)]
Sep 22 23:14:27.734 INFO Scrub d2ca4639-e324-4c6c-8e8a-4f9bdf088392 done in 0 seconds. Retries:0 scrub_size:2560 size:5 pause_milli:0
Sep 22 23:14:27.734 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.734 DEBG up_ds_listen was notified
Sep 22 23:14:27.734 DEBG IO Flush 1003 has deps [JobId(1002), JobId(1001), JobId(1000)]
Sep 22 23:14:27.734 DEBG up_ds_listen process 1001
Sep 22 23:14:27.734 INFO Created new region file "/tmp/downstairs-yOcvqWhx/region.json"
Sep 22 23:14:27.734 DEBG [A] ack job 1001:2, : downstairs
Sep 22 23:14:27.734 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.734 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.734 INFO Opened existing region file "/tmp/downstairs-JJG0Y7Ly/region.json"
Sep 22 23:14:27.734 INFO Database read version 1
Sep 22 23:14:27.734 INFO Database write version 1
Sep 22 23:14:27.734 DEBG up_ds_listen was notified
Sep 22 23:14:27.734 DEBG up_ds_listen process 1002
Sep 22 23:14:27.734 DEBG [A] ack job 1002:3, : downstairs
Sep 22 23:14:27.734 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.734 INFO Scrub at offset 5/5 sp:5
Sep 22 23:14:27.735 INFO Scrub check for 510c33b3-4659-4dfe-9ea9-4fd9b3d18480
Sep 22 23:14:27.735 INFO Scrub for 510c33b3-4659-4dfe-9ea9-4fd9b3d18480 begins
Sep 22 23:14:27.735 INFO Scrub with total_size:5120 block_size:512
Sep 22 23:14:27.735 INFO Scrubs from block 0 to 10 in (256) 131072 size IOs pm:0
Sep 22 23:14:27.735 INFO Adjust block_count to 10 at offset 0
Sep 22 23:14:27.735 DEBG Flush     :1003 extent_limit None deps:[JobId(1002), JobId(1001), JobId(1000)] res:true f:1 g:1
Sep 22 23:14:27.735 DEBG Flush     :1003 extent_limit None deps:[JobId(1002), JobId(1001), JobId(1000)] res:true f:1 g:1
Sep 22 23:14:27.735 DEBG [2] Read already AckReady 1004, : downstairs
Sep 22 23:14:27.735 DEBG Flush     :1003 extent_limit None deps:[JobId(1002), JobId(1001), JobId(1000)] res:true f:1 g:1
Sep 22 23:14:27.735 INFO UUID: e45251b5-0344-4afb-bb50-e9e4af673e5f
Sep 22 23:14:27.735 DEBG up_ds_listen was notified
Sep 22 23:14:27.735 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.735 DEBG up_ds_listen process 1004
Sep 22 23:14:27.735 DEBG [A] ack job 1004:5, : downstairs
Sep 22 23:14:27.735 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.735 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.735 INFO Using address: 127.0.0.1:51761, task: main
Sep 22 23:14:27.735 DEBG up_ds_listen was notified
Sep 22 23:14:27.735 DEBG up_ds_listen process 1003
Sep 22 23:14:27.735 DEBG [A] ack job 1003:4, : downstairs
Sep 22 23:14:27.735 DEBG [rc] retire 1003 clears [JobId(1000), JobId(1001), JobId(1002), JobId(1003)], : downstairs
Sep 22 23:14:27.735 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.735 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.735 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:50200, task: repair
Sep 22 23:14:27.735 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:50200, task: repair
Sep 22 23:14:27.735 INFO listening, local_addr: 127.0.0.1:50200, task: repair
Sep 22 23:14:27.736 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.736 INFO Scrub 27af30bc-466c-47bd-9576-e5adb1115007 done in 0 seconds. Retries:0 scrub_size:2560 size:5 pause_milli:0
Sep 22 23:14:27.736 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:50200, task: repair
Sep 22 23:14:27.736 INFO Using repair address: 127.0.0.1:50200, task: main
Sep 22 23:14:27.736 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.736 DEBG IO Flush 1003 has deps [JobId(1002), JobId(1001), JobId(1000)]
Sep 22 23:14:27.736 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.736 INFO Created new region file "/tmp/downstairs-RAGRvo5y/region.json"
Sep 22 23:14:27.737 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.737 INFO Opened existing region file "/tmp/downstairs-yOcvqWhx/region.json"
Sep 22 23:14:27.737 INFO Database read version 1
Sep 22 23:14:27.737 INFO Database write version 1
Sep 22 23:14:27.738 DEBG IO Write 1004 has deps []
Sep 22 23:14:27.738 DEBG Write     :1001 deps:[JobId(1000)] res:true
Sep 22 23:14:27.739 INFO UUID: dcbee2a7-5a05-4339-b188-1d1bfb232386
Sep 22 23:14:27.739 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.739 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.739 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.739 INFO Using address: 127.0.0.1:39658, task: main
Sep 22 23:14:27.739 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.739 DEBG Write     :1001 deps:[JobId(1000)] res:true
Sep 22 23:14:27.739 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:54627, task: repair
Sep 22 23:14:27.739 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:54627, task: repair
Sep 22 23:14:27.740 INFO listening, local_addr: 127.0.0.1:54627, task: repair
Sep 22 23:14:27.740 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.740 INFO Opened existing region file "/tmp/downstairs-RAGRvo5y/region.json"
Sep 22 23:14:27.740 INFO Database read version 1
Sep 22 23:14:27.740 INFO Database write version 1
Sep 22 23:14:27.740 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:54627, task: repair
Sep 22 23:14:27.740 INFO Using repair address: 127.0.0.1:54627, task: main
Sep 22 23:14:27.740 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.740 DEBG Flush     :1003 extent_limit None deps:[JobId(1002), JobId(1001), JobId(1000)] res:true f:1 g:1
Sep 22 23:14:27.740 DEBG Flush     :1003 extent_limit None deps:[JobId(1002), JobId(1001), JobId(1000)] res:true f:1 g:1
Sep 22 23:14:27.740 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.740 DEBG Flush     :1003 extent_limit None deps:[JobId(1002), JobId(1001), JobId(1000)] res:true f:1 g:1
Sep 22 23:14:27.740 INFO Created new region file "/tmp/downstairs-HE4BLZZn/region.json"
test test::integration_test_scrub ... Sep 22 23:14:27.740 DEBG Write     :1001 deps:[JobId(1000)] res:true
ok
Sep 22 23:14:27.740 DEBG up_ds_listen was notified
Sep 22 23:14:27.741 DEBG up_ds_listen process 1003
Sep 22 23:14:27.741 DEBG [A] ack job 1003:4, : downstairs
Sep 22 23:14:27.741 DEBG [rc] retire 1003 clears [JobId(1000), JobId(1001), JobId(1002), JobId(1003)], : downstairs
Sep 22 23:14:27.741 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.741 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.741 INFO Created new region file "/tmp/downstairs-Fx2jwgfS/region.json"
Sep 22 23:14:27.741 DEBG IO Read  1004 has deps []
Sep 22 23:14:27.741 INFO UUID: c2d08264-7ded-412a-9acb-ca0bc576793e
Sep 22 23:14:27.741 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.741 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.741 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.741 INFO Using address: 127.0.0.1:34180, task: main
Sep 22 23:14:27.742 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.742 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:58985, task: repair
Sep 22 23:14:27.742 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:58985, task: repair
Sep 22 23:14:27.742 INFO listening, local_addr: 127.0.0.1:58985, task: repair
Sep 22 23:14:27.742 DEBG Read      :1004 deps:[] res:true
Sep 22 23:14:27.742 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:58985, task: repair
Sep 22 23:14:27.742 INFO Using repair address: 127.0.0.1:58985, task: main
Sep 22 23:14:27.742 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.742 DEBG up_ds_listen was notified
Sep 22 23:14:27.742 DEBG up_ds_listen process 1004
Sep 22 23:14:27.742 DEBG [A] ack job 1004:5, : downstairs
Sep 22 23:14:27.742 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.742 INFO Upstairs starts
Sep 22 23:14:27.742 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:14:27.742 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:14:27.743 INFO Crucible stats registered with UUID: d173eb3a-6ff3-4b10-89cb-971e46f2c897
Sep 22 23:14:27.743 INFO Crucible d173eb3a-6ff3-4b10-89cb-971e46f2c897 has session id: f6aaf660-c0e6-4906-96a3-4646c5c3c38c
Sep 22 23:14:27.743 DEBG Read      :1004 deps:[] res:true
Sep 22 23:14:27.743 DEBG IO Read  1005 has deps [JobId(1004)]
Sep 22 23:14:27.743 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.743 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.743 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.743 INFO [0] connecting to 127.0.0.1:35706, looper: 0
Sep 22 23:14:27.743 DEBG IO Write 1002 has deps [JobId(1001), JobId(1000)]
Sep 22 23:14:27.743 INFO [1] connecting to 127.0.0.1:51761, looper: 1
Sep 22 23:14:27.743 INFO [2] connecting to 127.0.0.1:34180, looper: 2
Sep 22 23:14:27.743 INFO up_listen starts, task: up_listen
Sep 22 23:14:27.743 INFO Wait for all three downstairs to come online
Sep 22 23:14:27.743 INFO Flush timeout: 0.5
Sep 22 23:14:27.743 INFO accepted connection from 127.0.0.1:34884, task: main
Sep 22 23:14:27.743 INFO accepted connection from 127.0.0.1:47731, task: main
Sep 22 23:14:27.743 DEBG Read      :1004 deps:[] res:true
Sep 22 23:14:27.743 INFO [0] d173eb3a-6ff3-4b10-89cb-971e46f2c897 looper connected, looper: 0
Sep 22 23:14:27.743 INFO [0] Proc runs for 127.0.0.1:35706 in state New
Sep 22 23:14:27.743 INFO accepted connection from 127.0.0.1:62343, task: main
Sep 22 23:14:27.744 INFO [2] d173eb3a-6ff3-4b10-89cb-971e46f2c897 looper connected, looper: 2
Sep 22 23:14:27.744 INFO [2] Proc runs for 127.0.0.1:34180 in state New
Sep 22 23:14:27.744 INFO [1] d173eb3a-6ff3-4b10-89cb-971e46f2c897 looper connected, looper: 1
Sep 22 23:14:27.744 DEBG Read      :1005 deps:[JobId(1004)] res:true
Sep 22 23:14:27.744 INFO [1] Proc runs for 127.0.0.1:51761 in state New
Sep 22 23:14:27.744 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.744 INFO Opened existing region file "/tmp/downstairs-HE4BLZZn/region.json"
Sep 22 23:14:27.744 INFO Database read version 1
Sep 22 23:14:27.744 INFO Database write version 1
Sep 22 23:14:27.744 INFO Connection request from d173eb3a-6ff3-4b10-89cb-971e46f2c897 with version 4, task: proc
Sep 22 23:14:27.744 INFO upstairs UpstairsConnection { upstairs_id: d173eb3a-6ff3-4b10-89cb-971e46f2c897, session_id: 7d35a049-e6b2-42d0-8692-1238474c8ef7, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.744 INFO Connection request from d173eb3a-6ff3-4b10-89cb-971e46f2c897 with version 4, task: proc
Sep 22 23:14:27.744 INFO upstairs UpstairsConnection { upstairs_id: d173eb3a-6ff3-4b10-89cb-971e46f2c897, session_id: 7d35a049-e6b2-42d0-8692-1238474c8ef7, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.744 INFO Connection request from d173eb3a-6ff3-4b10-89cb-971e46f2c897 with version 4, task: proc
Sep 22 23:14:27.744 INFO upstairs UpstairsConnection { upstairs_id: d173eb3a-6ff3-4b10-89cb-971e46f2c897, session_id: 7d35a049-e6b2-42d0-8692-1238474c8ef7, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.744 DEBG Read      :1005 deps:[JobId(1004)] res:true
The guest has requested activation
Sep 22 23:14:27.744 INFO [0] d173eb3a-6ff3-4b10-89cb-971e46f2c897 (7d35a049-e6b2-42d0-8692-1238474c8ef7) New New New ds_transition to WaitActive
Sep 22 23:14:27.744 INFO [0] Transition from New to WaitActive
Sep 22 23:14:27.744 INFO [2] d173eb3a-6ff3-4b10-89cb-971e46f2c897 (7d35a049-e6b2-42d0-8692-1238474c8ef7) WaitActive New New ds_transition to WaitActive
Sep 22 23:14:27.744 INFO [2] Transition from New to WaitActive
Sep 22 23:14:27.744 INFO [1] d173eb3a-6ff3-4b10-89cb-971e46f2c897 (7d35a049-e6b2-42d0-8692-1238474c8ef7) WaitActive New WaitActive ds_transition to WaitActive
Sep 22 23:14:27.744 INFO [1] Transition from New to WaitActive
Sep 22 23:14:27.744 INFO d173eb3a-6ff3-4b10-89cb-971e46f2c897 active request set
Sep 22 23:14:27.745 INFO [0] received activate with gen 1
Sep 22 23:14:27.745 INFO [0] client got ds_active_rx, promote! session 7d35a049-e6b2-42d0-8692-1238474c8ef7
Sep 22 23:14:27.745 DEBG Read      :1005 deps:[JobId(1004)] res:true
Sep 22 23:14:27.745 INFO [1] received activate with gen 1
Sep 22 23:14:27.745 INFO [1] client got ds_active_rx, promote! session 7d35a049-e6b2-42d0-8692-1238474c8ef7
Sep 22 23:14:27.745 INFO [2] received activate with gen 1
Sep 22 23:14:27.745 INFO [2] client got ds_active_rx, promote! session 7d35a049-e6b2-42d0-8692-1238474c8ef7
Sep 22 23:14:27.745 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.745 INFO UpstairsConnection { upstairs_id: d173eb3a-6ff3-4b10-89cb-971e46f2c897, session_id: 7d35a049-e6b2-42d0-8692-1238474c8ef7, gen: 1 } is now active (read-write)
Sep 22 23:14:27.745 INFO Opened existing region file "/tmp/downstairs-Fx2jwgfS/region.json"
Sep 22 23:14:27.745 INFO Database read version 1
Sep 22 23:14:27.745 INFO Database write version 1
Sep 22 23:14:27.745 INFO UpstairsConnection { upstairs_id: d173eb3a-6ff3-4b10-89cb-971e46f2c897, session_id: 7d35a049-e6b2-42d0-8692-1238474c8ef7, gen: 1 } is now active (read-write)
Sep 22 23:14:27.745 INFO UpstairsConnection { upstairs_id: d173eb3a-6ff3-4b10-89cb-971e46f2c897, session_id: 7d35a049-e6b2-42d0-8692-1238474c8ef7, gen: 1 } is now active (read-write)
Sep 22 23:14:27.745 INFO [0] downstairs client at 127.0.0.1:35706 has UUID a0901c3f-7fbf-40b8-85c2-142a27a641c8
Sep 22 23:14:27.745 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: a0901c3f-7fbf-40b8-85c2-142a27a641c8, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.745 INFO d173eb3a-6ff3-4b10-89cb-971e46f2c897 WaitActive WaitActive WaitActive
Sep 22 23:14:27.745 INFO [2] downstairs client at 127.0.0.1:34180 has UUID c2d08264-7ded-412a-9acb-ca0bc576793e
Sep 22 23:14:27.745 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: c2d08264-7ded-412a-9acb-ca0bc576793e, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.745 INFO d173eb3a-6ff3-4b10-89cb-971e46f2c897 WaitActive WaitActive WaitActive
Sep 22 23:14:27.745 INFO [1] downstairs client at 127.0.0.1:51761 has UUID e45251b5-0344-4afb-bb50-e9e4af673e5f
Sep 22 23:14:27.746 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: e45251b5-0344-4afb-bb50-e9e4af673e5f, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.746 INFO d173eb3a-6ff3-4b10-89cb-971e46f2c897 WaitActive WaitActive WaitActive
Sep 22 23:14:27.746 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.746 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.746 INFO UUID: cc461d70-432a-44c4-b9ed-07b65b90fa30
Sep 22 23:14:27.746 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.746 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.746 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.746 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.746 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.746 INFO Using address: 127.0.0.1:62268, task: main
Sep 22 23:14:27.746 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.746 DEBG up_ds_listen was notified
Sep 22 23:14:27.746 DEBG up_ds_listen process 1002
Sep 22 23:14:27.746 DEBG [A] ack job 1002:3, : downstairs
Sep 22 23:14:27.746 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.746 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.746 INFO Scrub at offset 10/10 sp:10
Sep 22 23:14:27.746 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.746 INFO [0] d173eb3a-6ff3-4b10-89cb-971e46f2c897 (7d35a049-e6b2-42d0-8692-1238474c8ef7) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.747 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.747 WARN [0] new RM replaced this: None
Sep 22 23:14:27.747 INFO [0] Starts reconcile loop
Sep 22 23:14:27.747 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:65334, task: repair
Sep 22 23:14:27.747 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:65334, task: repair
Sep 22 23:14:27.747 INFO [2] d173eb3a-6ff3-4b10-89cb-971e46f2c897 (7d35a049-e6b2-42d0-8692-1238474c8ef7) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.747 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.747 WARN [2] new RM replaced this: None
Sep 22 23:14:27.747 INFO listening, local_addr: 127.0.0.1:65334, task: repair
Sep 22 23:14:27.747 INFO [2] Starts reconcile loop
Sep 22 23:14:27.747 DEBG [0] Read AckReady 1004, : downstairs
Sep 22 23:14:27.747 INFO [1] d173eb3a-6ff3-4b10-89cb-971e46f2c897 (7d35a049-e6b2-42d0-8692-1238474c8ef7) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum
Sep 22 23:14:27.747 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.747 WARN [1] new RM replaced this: None
Sep 22 23:14:27.747 INFO [1] Starts reconcile loop
Sep 22 23:14:27.747 INFO UUID: d97d6679-3993-407e-b222-4165f2682efb
Sep 22 23:14:27.747 INFO [0] 127.0.0.1:35706 task reports connection:true
Sep 22 23:14:27.747 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.747 INFO d173eb3a-6ff3-4b10-89cb-971e46f2c897 WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:14:27.747 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:65334, task: repair
Sep 22 23:14:27.747 INFO [0]R  flush_numbers: [0, 0]
Sep 22 23:14:27.747 INFO [0]R  generation: [0, 0]
Sep 22 23:14:27.747 INFO [0]R  dirty: [false, false]
Sep 22 23:14:27.747 INFO Using repair address: 127.0.0.1:65334, task: main
Sep 22 23:14:27.747 INFO [1]R  flush_numbers: [0, 0]
Sep 22 23:14:27.747 INFO [1]R  generation: [0, 0]
Sep 22 23:14:27.747 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.747 INFO [1]R  dirty: [false, false]
Sep 22 23:14:27.747 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.747 INFO [2]R  flush_numbers: [0, 0]
Sep 22 23:14:27.747 INFO [2]R  generation: [0, 0]
Sep 22 23:14:27.747 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.747 INFO [2]R  dirty: [false, false]
Sep 22 23:14:27.747 INFO Max found gen is 1
Sep 22 23:14:27.747 INFO Using address: 127.0.0.1:40377, task: main
Sep 22 23:14:27.747 INFO Generation requested: 1 >= found:1
Sep 22 23:14:27.747 INFO Next flush: 1
Sep 22 23:14:27.747 INFO All extents match
Sep 22 23:14:27.747 INFO No downstairs repair required
Sep 22 23:14:27.747 INFO No initial repair work was required
Sep 22 23:14:27.747 INFO Set Downstairs and Upstairs active
Sep 22 23:14:27.747 INFO d173eb3a-6ff3-4b10-89cb-971e46f2c897 is now active with session: 7d35a049-e6b2-42d0-8692-1238474c8ef7
Sep 22 23:14:27.747 INFO d173eb3a-6ff3-4b10-89cb-971e46f2c897 Set Active after no repair
Sep 22 23:14:27.747 INFO Notify all downstairs, region set compare is done.
Sep 22 23:14:27.747 INFO Set check for repair
Sep 22 23:14:27.747 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.747 INFO [2] 127.0.0.1:34180 task reports connection:true
Sep 22 23:14:27.747 INFO d173eb3a-6ff3-4b10-89cb-971e46f2c897 Active Active Active
Sep 22 23:14:27.747 INFO Set check for repair
Sep 22 23:14:27.747 INFO [1] 127.0.0.1:51761 task reports connection:true
Sep 22 23:14:27.747 INFO d173eb3a-6ff3-4b10-89cb-971e46f2c897 Active Active Active
Sep 22 23:14:27.747 INFO Set check for repair
Sep 22 23:14:27.747 INFO Created new region file "/tmp/downstairs-AadW8v1X/region.json"
Sep 22 23:14:27.747 INFO [0] received reconcile message
Sep 22 23:14:27.747 INFO [0] All repairs completed, exit
Sep 22 23:14:27.747 INFO [0] Starts cmd_loop
Sep 22 23:14:27.747 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.747 INFO [1] received reconcile message
Sep 22 23:14:27.747 INFO [1] All repairs completed, exit
Sep 22 23:14:27.747 INFO [1] Starts cmd_loop
Sep 22 23:14:27.747 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:33016, task: repair
Sep 22 23:14:27.747 INFO [2] received reconcile message
Sep 22 23:14:27.747 INFO [2] All repairs completed, exit
Sep 22 23:14:27.747 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:33016, task: repair
Sep 22 23:14:27.747 INFO [2] Starts cmd_loop
The guest has finished waiting for activation
Sep 22 23:14:27.747 INFO listening, local_addr: 127.0.0.1:33016, task: repair
Sep 22 23:14:27.748 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:33016, task: repair
Sep 22 23:14:27.748 INFO Using repair address: 127.0.0.1:33016, task: main
Sep 22 23:14:27.748 INFO Scrub 510c33b3-4659-4dfe-9ea9-4fd9b3d18480 done in 0 seconds. Retries:0 scrub_size:5120 size:10 pause_milli:0
Sep 22 23:14:27.748 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.748 DEBG IO Flush 1003 has deps [JobId(1002), JobId(1001), JobId(1000)]
Sep 22 23:14:27.748 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.748 INFO Created new region file "/tmp/downstairs-tpdOXLU0/region.json"
Sep 22 23:14:27.749 DEBG [1] Read already AckReady 1004, : downstairs
Sep 22 23:14:27.749 DEBG [0] Read AckReady 1005, : downstairs
Sep 22 23:14:27.750 DEBG IO Write 1000 has deps []
Sep 22 23:14:27.750 DEBG up_ds_listen was notified
Sep 22 23:14:27.750 DEBG up_ds_listen process 1000
Sep 22 23:14:27.750 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:14:27.750 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.750 DEBG Flush     :1003 extent_limit None deps:[JobId(1002), JobId(1001), JobId(1000)] res:true f:1 g:1
Sep 22 23:14:27.750 DEBG [2] Read already AckReady 1004, : downstairs
Sep 22 23:14:27.750 DEBG Flush     :1003 extent_limit None deps:[JobId(1002), JobId(1001), JobId(1000)] res:true f:1 g:1
Sep 22 23:14:27.750 DEBG up_ds_listen was notified
Sep 22 23:14:27.751 DEBG up_ds_listen process 1004
Sep 22 23:14:27.751 DEBG [A] ack job 1004:5, : downstairs
Sep 22 23:14:27.751 DEBG Flush     :1003 extent_limit None deps:[JobId(1002), JobId(1001), JobId(1000)] res:true f:1 g:1
Sep 22 23:14:27.751 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.751 INFO Opened existing region file "/tmp/downstairs-AadW8v1X/region.json"
Sep 22 23:14:27.751 INFO Database read version 1
Sep 22 23:14:27.751 INFO Database write version 1
Sep 22 23:14:27.751 DEBG up_ds_listen was notified
Sep 22 23:14:27.751 DEBG up_ds_listen process 1003
Sep 22 23:14:27.751 DEBG [A] ack job 1003:4, : downstairs
Sep 22 23:14:27.751 DEBG [rc] retire 1003 clears [JobId(1000), JobId(1001), JobId(1002), JobId(1003)], : downstairs
Sep 22 23:14:27.751 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.751 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.751 DEBG IO Read  1004 has deps []
Sep 22 23:14:27.751 INFO Request to deactivate this guest
Sep 22 23:14:27.752 INFO d173eb3a-6ff3-4b10-89cb-971e46f2c897 set deactivating.
Sep 22 23:14:27.752 DEBG IO Flush 1001 has deps [JobId(1000)]
Sep 22 23:14:27.752 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.752 INFO Opened existing region file "/tmp/downstairs-tpdOXLU0/region.json"
Sep 22 23:14:27.752 INFO Database read version 1
Sep 22 23:14:27.752 INFO Database write version 1
Sep 22 23:14:27.752 DEBG [1] Read already AckReady 1005, : downstairs
Sep 22 23:14:27.752 DEBG Read      :1004 deps:[] res:true
Sep 22 23:14:27.753 DEBG Write     :1000 deps:[] res:true
Sep 22 23:14:27.753 DEBG Read      :1004 deps:[] res:true
Sep 22 23:14:27.753 DEBG Read      :1004 deps:[] res:true
Sep 22 23:14:27.754 INFO UUID: ac1cf4dc-f285-414c-b94a-cbeedf2f019d
Sep 22 23:14:27.754 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.754 DEBG Write     :1000 deps:[] res:true
Sep 22 23:14:27.754 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.754 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.754 INFO Using address: 127.0.0.1:60728, task: main
Sep 22 23:14:27.754 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.754 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:36934, task: repair
Sep 22 23:14:27.754 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:36934, task: repair
Sep 22 23:14:27.754 INFO listening, local_addr: 127.0.0.1:36934, task: repair
Sep 22 23:14:27.755 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:36934, task: repair
Sep 22 23:14:27.755 INFO Using repair address: 127.0.0.1:36934, task: main
Sep 22 23:14:27.755 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.755 DEBG [2] Read already AckReady 1005, : downstairs
Sep 22 23:14:27.755 DEBG up_ds_listen was notified
Sep 22 23:14:27.755 DEBG up_ds_listen process 1005
Sep 22 23:14:27.755 INFO UUID: 13286ed5-820a-470e-a2ba-89ce1d0ebb25
Sep 22 23:14:27.755 DEBG [A] ack job 1005:6, : downstairs
Sep 22 23:14:27.755 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.755 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.755 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.755 INFO Using address: 127.0.0.1:50371, task: main
Sep 22 23:14:27.755 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.755 DEBG Write     :1000 deps:[] res:true
Sep 22 23:14:27.756 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.756 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:53863, task: repair
Sep 22 23:14:27.756 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:53863, task: repair
Sep 22 23:14:27.756 INFO listening, local_addr: 127.0.0.1:53863, task: repair
Sep 22 23:14:27.756 INFO [0] deactivate job 1001 not InProgress flush, NO
Sep 22 23:14:27.756 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:53863, task: repair
Sep 22 23:14:27.756 INFO Using repair address: 127.0.0.1:53863, task: main
Sep 22 23:14:27.756 INFO [2] deactivate job 1001 not InProgress flush, NO
Sep 22 23:14:27.756 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.756 INFO [1] deactivate job 1001 not InProgress flush, NO
Sep 22 23:14:27.756 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.756 INFO Upstairs starts
Sep 22 23:14:27.756 INFO Created new region file "/tmp/downstairs-TELOR5uT/region.json"
Sep 22 23:14:27.756 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:14:27.756 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:14:27.756 INFO Crucible stats registered with UUID: de3b8039-2721-4588-b15f-2b25684f8d95
Sep 22 23:14:27.756 INFO Crucible de3b8039-2721-4588-b15f-2b25684f8d95 has session id: 1f4003f9-85a2-4adf-822f-455c625523f3
test test::integration_test_scrub_short_sparse ... ok
Sep 22 23:14:27.757 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.757 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.757 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.757 INFO [0] connecting to 127.0.0.1:39658, looper: 0
Sep 22 23:14:27.757 INFO [1] connecting to 127.0.0.1:62268, looper: 1
Sep 22 23:14:27.757 DEBG [0] Read AckReady 1004, : downstairs
Sep 22 23:14:27.757 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.757 INFO [2] connecting to 127.0.0.1:60728, looper: 2
Sep 22 23:14:27.757 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:1 g:1
Sep 22 23:14:27.757 INFO up_listen starts, task: up_listen
Sep 22 23:14:27.757 INFO Wait for all three downstairs to come online
Sep 22 23:14:27.757 INFO Created new region file "/tmp/downstairs-uCbezb5Z/region.json"
Sep 22 23:14:27.757 INFO Flush timeout: 0.5
Sep 22 23:14:27.757 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:1 g:1
Sep 22 23:14:27.757 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:1 g:1
Sep 22 23:14:27.757 INFO [0] check deactivate YES
Sep 22 23:14:27.757 INFO [0] d173eb3a-6ff3-4b10-89cb-971e46f2c897 (7d35a049-e6b2-42d0-8692-1238474c8ef7) Active Active Active ds_transition to Deactivated
Sep 22 23:14:27.757 INFO [0] Transition from Active to Deactivated
Sep 22 23:14:27.757 INFO accepted connection from 127.0.0.1:62055, task: main
Sep 22 23:14:27.758 INFO [2] check deactivate YES
Sep 22 23:14:27.758 INFO [2] d173eb3a-6ff3-4b10-89cb-971e46f2c897 (7d35a049-e6b2-42d0-8692-1238474c8ef7) Deactivated Active Active ds_transition to Deactivated
Sep 22 23:14:27.758 INFO [2] Transition from Active to Deactivated
Sep 22 23:14:27.758 INFO [1] de3b8039-2721-4588-b15f-2b25684f8d95 looper connected, looper: 1
Sep 22 23:14:27.758 DEBG [1] deactivate flush 1001 done, : downstairs
Sep 22 23:14:27.758 INFO [1] Proc runs for 127.0.0.1:62268 in state New
Sep 22 23:14:27.758 INFO [1] check deactivate YES
Sep 22 23:14:27.758 INFO [1] d173eb3a-6ff3-4b10-89cb-971e46f2c897 (7d35a049-e6b2-42d0-8692-1238474c8ef7) Deactivated Active Deactivated ds_transition to Deactivated
Sep 22 23:14:27.758 INFO [1] Transition from Active to Deactivated
Sep 22 23:14:27.758 INFO [0] de3b8039-2721-4588-b15f-2b25684f8d95 looper connected, looper: 0
Sep 22 23:14:27.758 INFO [0] Proc runs for 127.0.0.1:39658 in state New
Sep 22 23:14:27.758 ERRO 127.0.0.1:35706: proc: [0] client work task ended, Ok(Err([0] exits after deactivation)), so we end too, looper: 0
Sep 22 23:14:27.758 INFO [0] d173eb3a-6ff3-4b10-89cb-971e46f2c897 Gone missing, transition from Deactivated to New
Sep 22 23:14:27.758 INFO deactivate transition checking...
Sep 22 23:14:27.758 INFO deactivate_transition New Maybe 
Sep 22 23:14:27.758 INFO deactivate_transition Deactivated NO
Sep 22 23:14:27.758 INFO deactivate_transition Deactivated NO
Sep 22 23:14:27.758 INFO accepted connection from 127.0.0.1:63374, task: main
Sep 22 23:14:27.758 INFO [0] d173eb3a-6ff3-4b10-89cb-971e46f2c897 connection to 127.0.0.1:35706 closed, looper: 0
Sep 22 23:14:27.758 INFO [2] de3b8039-2721-4588-b15f-2b25684f8d95 looper connected, looper: 2
Sep 22 23:14:27.758 ERRO 127.0.0.1:34180: proc: [2] client work task ended, Ok(Err([2] exits after deactivation)), so we end too, looper: 2
Sep 22 23:14:27.758 INFO [2] d173eb3a-6ff3-4b10-89cb-971e46f2c897 Gone missing, transition from Deactivated to New
Sep 22 23:14:27.758 INFO [2] Proc runs for 127.0.0.1:60728 in state New
Sep 22 23:14:27.758 INFO deactivate transition checking...
Sep 22 23:14:27.758 INFO deactivate_transition New Maybe 
Sep 22 23:14:27.758 INFO deactivate_transition Deactivated NO
Sep 22 23:14:27.758 INFO deactivate_transition New Maybe 
Sep 22 23:14:27.758 INFO [2] d173eb3a-6ff3-4b10-89cb-971e46f2c897 connection to 127.0.0.1:34180 closed, looper: 2
Sep 22 23:14:27.758 DEBG up_ds_listen was notified
Sep 22 23:14:27.758 DEBG up_ds_listen process 1001
Sep 22 23:14:27.758 DEBG [A] ack job 1001:2, : downstairs
Sep 22 23:14:27.758 DEBG [rc] retire 1001 clears [JobId(1000), JobId(1001)], : downstairs
Sep 22 23:14:27.758 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.758 INFO accepted connection from 127.0.0.1:64760, task: main
Sep 22 23:14:27.758 ERRO 127.0.0.1:51761: proc: [1] client work task ended, Ok(Err([1] exits after deactivation)), so we end too, looper: 1
Sep 22 23:14:27.758 INFO [1] d173eb3a-6ff3-4b10-89cb-971e46f2c897 Gone missing, transition from Deactivated to New
Sep 22 23:14:27.758 INFO deactivate transition checking...
Sep 22 23:14:27.758 INFO deactivate_transition New Maybe 
Sep 22 23:14:27.758 INFO deactivate_transition New Maybe 
Sep 22 23:14:27.758 INFO deactivate_transition New Maybe 
Sep 22 23:14:27.758 INFO All DS in the proper state! -> INIT
Sep 22 23:14:27.758 INFO [1] d173eb3a-6ff3-4b10-89cb-971e46f2c897 connection to 127.0.0.1:51761 closed, looper: 1
Sep 22 23:14:27.758 INFO [0] 127.0.0.1:35706 task reports connection:false
Sep 22 23:14:27.758 INFO d173eb3a-6ff3-4b10-89cb-971e46f2c897 New New New
Sep 22 23:14:27.758 INFO [0] 127.0.0.1:35706 task reports offline
Sep 22 23:14:27.758 INFO [2] 127.0.0.1:34180 task reports connection:false
Sep 22 23:14:27.758 INFO d173eb3a-6ff3-4b10-89cb-971e46f2c897 New New New
Sep 22 23:14:27.758 INFO [2] 127.0.0.1:34180 task reports offline
Sep 22 23:14:27.758 INFO [1] 127.0.0.1:51761 task reports connection:false
Sep 22 23:14:27.758 INFO d173eb3a-6ff3-4b10-89cb-971e46f2c897 New New New
Sep 22 23:14:27.758 INFO [1] 127.0.0.1:51761 task reports offline
Sep 22 23:14:27.759 INFO Connection request from de3b8039-2721-4588-b15f-2b25684f8d95 with version 4, task: proc
Sep 22 23:14:27.759 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.759 INFO upstairs UpstairsConnection { upstairs_id: de3b8039-2721-4588-b15f-2b25684f8d95, session_id: d559352a-579e-4e6f-9baa-d23c3f55b78d, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.759 INFO Opened existing region file "/tmp/downstairs-NzNQXxni/region.json"
Sep 22 23:14:27.759 INFO Database read version 1
Sep 22 23:14:27.759 INFO Database write version 1
Sep 22 23:14:27.759 INFO Connection request from de3b8039-2721-4588-b15f-2b25684f8d95 with version 4, task: proc
Sep 22 23:14:27.759 INFO upstairs UpstairsConnection { upstairs_id: de3b8039-2721-4588-b15f-2b25684f8d95, session_id: d559352a-579e-4e6f-9baa-d23c3f55b78d, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.759 INFO Connection request from de3b8039-2721-4588-b15f-2b25684f8d95 with version 4, task: proc
Sep 22 23:14:27.759 INFO upstairs UpstairsConnection { upstairs_id: de3b8039-2721-4588-b15f-2b25684f8d95, session_id: d559352a-579e-4e6f-9baa-d23c3f55b78d, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.759 INFO [1] de3b8039-2721-4588-b15f-2b25684f8d95 (d559352a-579e-4e6f-9baa-d23c3f55b78d) New New New ds_transition to WaitActive
Sep 22 23:14:27.759 INFO [1] Transition from New to WaitActive
Sep 22 23:14:27.759 INFO [0] de3b8039-2721-4588-b15f-2b25684f8d95 (d559352a-579e-4e6f-9baa-d23c3f55b78d) New WaitActive New ds_transition to WaitActive
Sep 22 23:14:27.759 INFO [0] Transition from New to WaitActive
Sep 22 23:14:27.759 INFO [2] de3b8039-2721-4588-b15f-2b25684f8d95 (d559352a-579e-4e6f-9baa-d23c3f55b78d) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:14:27.759 INFO [2] Transition from New to WaitActive
The guest has requested activation
Sep 22 23:14:27.759 INFO de3b8039-2721-4588-b15f-2b25684f8d95 active request set
Sep 22 23:14:27.759 DEBG [1] Read already AckReady 1004, : downstairs
Sep 22 23:14:27.759 INFO [0] received activate with gen 1
Sep 22 23:14:27.760 INFO [0] client got ds_active_rx, promote! session d559352a-579e-4e6f-9baa-d23c3f55b78d
Sep 22 23:14:27.760 INFO [1] received activate with gen 1
Sep 22 23:14:27.760 INFO [1] client got ds_active_rx, promote! session d559352a-579e-4e6f-9baa-d23c3f55b78d
Sep 22 23:14:27.760 INFO [2] received activate with gen 1
Sep 22 23:14:27.760 INFO [2] client got ds_active_rx, promote! session d559352a-579e-4e6f-9baa-d23c3f55b78d
Sep 22 23:14:27.760 INFO UpstairsConnection { upstairs_id: de3b8039-2721-4588-b15f-2b25684f8d95, session_id: d559352a-579e-4e6f-9baa-d23c3f55b78d, gen: 1 } is now active (read-write)
test test::integration_test_scrub_short ... okSep 22 23:14:27.760 INFO UpstairsConnection { upstairs_id: de3b8039-2721-4588-b15f-2b25684f8d95, session_id: d559352a-579e-4e6f-9baa-d23c3f55b78d, gen: 1 } is now active (read-write)

Sep 22 23:14:27.760 INFO UpstairsConnection { upstairs_id: de3b8039-2721-4588-b15f-2b25684f8d95, session_id: d559352a-579e-4e6f-9baa-d23c3f55b78d, gen: 1 } is now active (read-write)
Sep 22 23:14:27.760 INFO UUID: a0901c3f-7fbf-40b8-85c2-142a27a641c8
Sep 22 23:14:27.760 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.760 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.760 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.760 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.760 INFO Using address: 127.0.0.1:34728, task: main
Sep 22 23:14:27.760 INFO Created new region file "/tmp/downstairs-14il0Ky1/region.json"
Sep 22 23:14:27.760 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.761 INFO [1] downstairs client at 127.0.0.1:62268 has UUID cc461d70-432a-44c4-b9ed-07b65b90fa30
Sep 22 23:14:27.761 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.761 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: cc461d70-432a-44c4-b9ed-07b65b90fa30, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.761 INFO Opened existing region file "/tmp/downstairs-TELOR5uT/region.json"
Sep 22 23:14:27.761 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:61304, task: repair
Sep 22 23:14:27.761 INFO Database read version 1
Sep 22 23:14:27.761 INFO Database write version 1
Sep 22 23:14:27.761 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:61304, task: repair
Sep 22 23:14:27.761 INFO de3b8039-2721-4588-b15f-2b25684f8d95 WaitActive WaitActive WaitActive
Sep 22 23:14:27.761 INFO listening, local_addr: 127.0.0.1:61304, task: repair
Sep 22 23:14:27.761 INFO [0] downstairs client at 127.0.0.1:39658 has UUID dcbee2a7-5a05-4339-b188-1d1bfb232386
Sep 22 23:14:27.761 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: dcbee2a7-5a05-4339-b188-1d1bfb232386, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.761 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:61304, task: repair
Sep 22 23:14:27.761 INFO de3b8039-2721-4588-b15f-2b25684f8d95 WaitActive WaitActive WaitActive
Sep 22 23:14:27.761 INFO Using repair address: 127.0.0.1:61304, task: main
Sep 22 23:14:27.761 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.761 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.761 INFO [2] downstairs client at 127.0.0.1:60728 has UUID ac1cf4dc-f285-414c-b94a-cbeedf2f019d
Sep 22 23:14:27.761 INFO Opened existing region file "/tmp/downstairs-JJG0Y7Ly/region.json"
Sep 22 23:14:27.761 INFO Database read version 1
Sep 22 23:14:27.761 INFO Database write version 1
Sep 22 23:14:27.761 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: ac1cf4dc-f285-414c-b94a-cbeedf2f019d, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.761 INFO de3b8039-2721-4588-b15f-2b25684f8d95 WaitActive WaitActive WaitActive
Sep 22 23:14:27.761 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.761 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.761 INFO Opened existing region file "/tmp/downstairs-uCbezb5Z/region.json"
Sep 22 23:14:27.761 INFO Database read version 1
Sep 22 23:14:27.761 INFO Database write version 1
Sep 22 23:14:27.761 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.762 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.762 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.762 INFO UUID: 5bccaeee-f6e7-4a30-aac1-e7711f75e3d5
Sep 22 23:14:27.762 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.762 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.762 DEBG [2] Read already AckReady 1004, : downstairs
Sep 22 23:14:27.762 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.762 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.762 INFO Using address: 127.0.0.1:53080, task: main
Sep 22 23:14:27.762 DEBG up_ds_listen was notified
Sep 22 23:14:27.762 DEBG up_ds_listen process 1004
Sep 22 23:14:27.762 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.762 DEBG [A] ack job 1004:5, : downstairs
Sep 22 23:14:27.762 INFO UUID: e45251b5-0344-4afb-bb50-e9e4af673e5f
Sep 22 23:14:27.762 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.762 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.762 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.762 INFO Using address: 127.0.0.1:61240, task: main
Sep 22 23:14:27.762 INFO [1] de3b8039-2721-4588-b15f-2b25684f8d95 (d559352a-579e-4e6f-9baa-d23c3f55b78d) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.762 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.762 WARN [1] new RM replaced this: None
Sep 22 23:14:27.762 INFO [1] Starts reconcile loop
Sep 22 23:14:27.762 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.763 INFO [0] de3b8039-2721-4588-b15f-2b25684f8d95 (d559352a-579e-4e6f-9baa-d23c3f55b78d) WaitActive WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.763 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:39280, task: repair
Sep 22 23:14:27.763 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.763 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:39280, task: repair
Sep 22 23:14:27.763 WARN [0] new RM replaced this: None
Sep 22 23:14:27.763 INFO [0] Starts reconcile loop
Sep 22 23:14:27.763 INFO listening, local_addr: 127.0.0.1:39280, task: repair
Sep 22 23:14:27.763 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.763 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.763 INFO [2] de3b8039-2721-4588-b15f-2b25684f8d95 (d559352a-579e-4e6f-9baa-d23c3f55b78d) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.763 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.763 WARN [2] new RM replaced this: None
Sep 22 23:14:27.763 INFO [2] Starts reconcile loop
Sep 22 23:14:27.763 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:50554, task: repair
Sep 22 23:14:27.763 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:50554, task: repair
Sep 22 23:14:27.763 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:39280, task: repair
Sep 22 23:14:27.763 INFO listening, local_addr: 127.0.0.1:50554, task: repair
Sep 22 23:14:27.763 INFO [1] 127.0.0.1:62268 task reports connection:true
Sep 22 23:14:27.763 INFO Using repair address: 127.0.0.1:39280, task: main
Sep 22 23:14:27.763 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.763 INFO de3b8039-2721-4588-b15f-2b25684f8d95 WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:14:27.763 INFO [0]R  flush_numbers: [0, 0]
Sep 22 23:14:27.763 INFO [0]R  generation: [0, 0]
Sep 22 23:14:27.763 INFO [0]R  dirty: [false, false]
Sep 22 23:14:27.763 INFO [1]R  flush_numbers: [0, 0]
Sep 22 23:14:27.763 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:50554, task: repair
Sep 22 23:14:27.763 INFO [1]R  generation: [0, 0]
Sep 22 23:14:27.763 INFO [1]R  dirty: [false, false]
Sep 22 23:14:27.763 INFO Using repair address: 127.0.0.1:50554, task: main
Sep 22 23:14:27.763 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.763 INFO [2]R  flush_numbers: [0, 0]
Sep 22 23:14:27.763 INFO [2]R  generation: [0, 0]
Sep 22 23:14:27.763 INFO [2]R  dirty: [false, false]
Sep 22 23:14:27.763 INFO Max found gen is 1
Sep 22 23:14:27.763 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.763 INFO Generation requested: 1 >= found:1
Sep 22 23:14:27.763 INFO Opened existing region file "/tmp/downstairs-RAGRvo5y/region.json"
Sep 22 23:14:27.763 INFO Database read version 1
Sep 22 23:14:27.763 INFO Next flush: 1
Sep 22 23:14:27.763 INFO Database write version 1
Sep 22 23:14:27.763 INFO All extents match
Sep 22 23:14:27.763 INFO No downstairs repair required
Sep 22 23:14:27.763 INFO No initial repair work was required
Sep 22 23:14:27.763 INFO Set Downstairs and Upstairs active
Sep 22 23:14:27.763 INFO de3b8039-2721-4588-b15f-2b25684f8d95 is now active with session: d559352a-579e-4e6f-9baa-d23c3f55b78d
Sep 22 23:14:27.763 INFO de3b8039-2721-4588-b15f-2b25684f8d95 Set Active after no repair
Sep 22 23:14:27.763 INFO Notify all downstairs, region set compare is done.
Sep 22 23:14:27.763 INFO Set check for repair
Sep 22 23:14:27.763 INFO [0] 127.0.0.1:39658 task reports connection:true
Sep 22 23:14:27.763 INFO de3b8039-2721-4588-b15f-2b25684f8d95 Active Active Active
Sep 22 23:14:27.763 INFO Set check for repair
Sep 22 23:14:27.763 INFO UUID: 2cb21d94-77f3-4d85-9833-6fd57802dc13
Sep 22 23:14:27.763 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.763 INFO [2] 127.0.0.1:60728 task reports connection:true
Sep 22 23:14:27.763 INFO de3b8039-2721-4588-b15f-2b25684f8d95 Active Active Active
Sep 22 23:14:27.763 INFO Set check for repair
Sep 22 23:14:27.763 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.764 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.764 INFO Using address: 127.0.0.1:62529, task: main
Sep 22 23:14:27.764 INFO [0] received reconcile message
Sep 22 23:14:27.764 INFO [0] All repairs completed, exit
Sep 22 23:14:27.764 INFO [0] Starts cmd_loop
Sep 22 23:14:27.764 INFO [1] received reconcile message
Sep 22 23:14:27.764 INFO [1] All repairs completed, exit
Sep 22 23:14:27.764 INFO [1] Starts cmd_loop
Sep 22 23:14:27.764 INFO [2] received reconcile message
Sep 22 23:14:27.764 INFO [2] All repairs completed, exit
Sep 22 23:14:27.764 INFO [2] Starts cmd_loop
The guest has finished waiting for activation
Sep 22 23:14:27.764 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.764 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:38091, task: repair
Sep 22 23:14:27.764 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:38091, task: repair
Sep 22 23:14:27.764 INFO listening, local_addr: 127.0.0.1:38091, task: repair
Sep 22 23:14:27.764 DEBG IO Read  1000 has deps []
Sep 22 23:14:27.764 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:38091, task: repair
Sep 22 23:14:27.764 INFO Using repair address: 127.0.0.1:38091, task: main
Sep 22 23:14:27.764 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.764 INFO Upstairs starts
Sep 22 23:14:27.765 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:14:27.765 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:14:27.765 INFO Crucible stats registered with UUID: 8c772b52-d880-4be4-a2d6-4ced50104923
Sep 22 23:14:27.765 INFO Crucible 8c772b52-d880-4be4-a2d6-4ced50104923 has session id: 74b4f51b-c95c-416c-b091-8e4cc8b12bee
Sep 22 23:14:27.765 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.765 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.765 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.765 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.765 INFO [0] connecting to 127.0.0.1:40377, looper: 0
Sep 22 23:14:27.765 INFO Opened existing region file "/tmp/downstairs-14il0Ky1/region.json"
Sep 22 23:14:27.765 INFO Database read version 1
Sep 22 23:14:27.765 INFO Database write version 1
Sep 22 23:14:27.765 INFO [1] connecting to 127.0.0.1:50371, looper: 1
Sep 22 23:14:27.765 INFO [2] connecting to 127.0.0.1:53080, looper: 2
Sep 22 23:14:27.765 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.765 INFO Created new region file "/tmp/downstairs-iS4nNbSN/region.json"
Sep 22 23:14:27.765 INFO up_listen starts, task: up_listen
Sep 22 23:14:27.765 INFO Wait for all three downstairs to come online
Sep 22 23:14:27.766 INFO Flush timeout: 0.5
Sep 22 23:14:27.766 INFO [0] 8c772b52-d880-4be4-a2d6-4ced50104923 looper connected, looper: 0
Sep 22 23:14:27.766 INFO [0] Proc runs for 127.0.0.1:40377 in state New
Sep 22 23:14:27.766 INFO accepted connection from 127.0.0.1:42064, task: main
Sep 22 23:14:27.766 INFO [2] 8c772b52-d880-4be4-a2d6-4ced50104923 looper connected, looper: 2
Sep 22 23:14:27.766 DEBG Read      :1000 deps:[] res:true
Sep 22 23:14:27.766 INFO [2] Proc runs for 127.0.0.1:53080 in state New
Sep 22 23:14:27.766 INFO [1] 8c772b52-d880-4be4-a2d6-4ced50104923 looper connected, looper: 1
Sep 22 23:14:27.766 INFO [1] Proc runs for 127.0.0.1:50371 in state New
Sep 22 23:14:27.766 INFO accepted connection from 127.0.0.1:53474, task: main
Sep 22 23:14:27.766 INFO accepted connection from 127.0.0.1:32942, task: main
Sep 22 23:14:27.766 DEBG Read      :1000 deps:[] res:true
Sep 22 23:14:27.767 INFO Connection request from 8c772b52-d880-4be4-a2d6-4ced50104923 with version 4, task: proc
Sep 22 23:14:27.767 INFO upstairs UpstairsConnection { upstairs_id: 8c772b52-d880-4be4-a2d6-4ced50104923, session_id: f18e75dd-0520-46ef-a5dd-f5dbb5b1b3d6, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.767 INFO Connection request from 8c772b52-d880-4be4-a2d6-4ced50104923 with version 4, task: proc
Sep 22 23:14:27.767 INFO upstairs UpstairsConnection { upstairs_id: 8c772b52-d880-4be4-a2d6-4ced50104923, session_id: f18e75dd-0520-46ef-a5dd-f5dbb5b1b3d6, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.767 DEBG Read      :1000 deps:[] res:true
Sep 22 23:14:27.767 INFO Connection request from 8c772b52-d880-4be4-a2d6-4ced50104923 with version 4, task: proc
Sep 22 23:14:27.767 INFO upstairs UpstairsConnection { upstairs_id: 8c772b52-d880-4be4-a2d6-4ced50104923, session_id: f18e75dd-0520-46ef-a5dd-f5dbb5b1b3d6, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.767 INFO UUID: c2d08264-7ded-412a-9acb-ca0bc576793e
Sep 22 23:14:27.767 INFO Blocks per extent:5 Total Extents: 2
The guest has requested activation
Sep 22 23:14:27.767 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.767 INFO UUID: cc26b772-bef2-4af0-a9fc-0733fa8e0d6d
Sep 22 23:14:27.767 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.767 INFO Using address: 127.0.0.1:49345, task: main
Sep 22 23:14:27.767 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.767 INFO [0] 8c772b52-d880-4be4-a2d6-4ced50104923 (f18e75dd-0520-46ef-a5dd-f5dbb5b1b3d6) New New New ds_transition to WaitActive
Sep 22 23:14:27.767 INFO [0] Transition from New to WaitActive
Sep 22 23:14:27.767 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.767 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.767 INFO Using address: 127.0.0.1:61471, task: main
test test::integration_test_scrub_useless ... ok
Sep 22 23:14:27.767 INFO [2] 8c772b52-d880-4be4-a2d6-4ced50104923 (f18e75dd-0520-46ef-a5dd-f5dbb5b1b3d6) WaitActive New New ds_transition to WaitActive
Sep 22 23:14:27.767 INFO [2] Transition from New to WaitActive
Sep 22 23:14:27.767 INFO [1] 8c772b52-d880-4be4-a2d6-4ced50104923 (f18e75dd-0520-46ef-a5dd-f5dbb5b1b3d6) WaitActive New WaitActive ds_transition to WaitActive
Sep 22 23:14:27.767 INFO [1] Transition from New to WaitActive
Sep 22 23:14:27.767 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.767 INFO 8c772b52-d880-4be4-a2d6-4ced50104923 active request set
Sep 22 23:14:27.767 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:57771, task: repair
Sep 22 23:14:27.767 INFO [0] received activate with gen 1
Sep 22 23:14:27.767 INFO [0] client got ds_active_rx, promote! session f18e75dd-0520-46ef-a5dd-f5dbb5b1b3d6
Sep 22 23:14:27.767 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:57771, task: repair
Sep 22 23:14:27.767 INFO listening, local_addr: 127.0.0.1:57771, task: repair
Sep 22 23:14:27.767 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.768 INFO [1] received activate with gen 1
Sep 22 23:14:27.768 INFO [1] client got ds_active_rx, promote! session f18e75dd-0520-46ef-a5dd-f5dbb5b1b3d6
Sep 22 23:14:27.768 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:41268, task: repair
Sep 22 23:14:27.768 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:57771, task: repair
Sep 22 23:14:27.768 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.768 INFO Using repair address: 127.0.0.1:57771, task: main
Sep 22 23:14:27.768 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:41268, task: repair
Sep 22 23:14:27.768 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.768 INFO [2] received activate with gen 1
Sep 22 23:14:27.768 INFO [2] client got ds_active_rx, promote! session f18e75dd-0520-46ef-a5dd-f5dbb5b1b3d6
Sep 22 23:14:27.768 INFO listening, local_addr: 127.0.0.1:41268, task: repair
Sep 22 23:14:27.768 INFO Created new region file "/tmp/downstairs-pJh0v4dT/region.json"
Sep 22 23:14:27.768 INFO Upstairs starts
Sep 22 23:14:27.768 INFO UpstairsConnection { upstairs_id: 8c772b52-d880-4be4-a2d6-4ced50104923, session_id: f18e75dd-0520-46ef-a5dd-f5dbb5b1b3d6, gen: 1 } is now active (read-write)
Sep 22 23:14:27.768 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:14:27.768 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:14:27.768 INFO Crucible stats registered with UUID: d173eb3a-6ff3-4b10-89cb-971e46f2c897
Sep 22 23:14:27.768 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:41268, task: repair
Sep 22 23:14:27.768 INFO Crucible d173eb3a-6ff3-4b10-89cb-971e46f2c897 has session id: 116b3e11-839c-4e16-8ff9-3034e728e74a
Sep 22 23:14:27.768 INFO Using repair address: 127.0.0.1:41268, task: main
Sep 22 23:14:27.768 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.768 INFO UpstairsConnection { upstairs_id: 8c772b52-d880-4be4-a2d6-4ced50104923, session_id: f18e75dd-0520-46ef-a5dd-f5dbb5b1b3d6, gen: 1 } is now active (read-write)
Sep 22 23:14:27.768 DEBG [1] Read AckReady 1000, : downstairs
Sep 22 23:14:27.768 WARN upstairs UpstairsConnection { upstairs_id: d173eb3a-6ff3-4b10-89cb-971e46f2c897, session_id: 7d35a049-e6b2-42d0-8692-1238474c8ef7, gen: 1 } disconnected, 0 jobs left, task: main
Sep 22 23:14:27.768 WARN upstairs UpstairsConnection { upstairs_id: d173eb3a-6ff3-4b10-89cb-971e46f2c897, session_id: 7d35a049-e6b2-42d0-8692-1238474c8ef7, gen: 1 } was previously active, clearing, task: main
Sep 22 23:14:27.768 INFO connection (127.0.0.1:34884): all done
Sep 22 23:14:27.768 INFO UpstairsConnection { upstairs_id: 8c772b52-d880-4be4-a2d6-4ced50104923, session_id: f18e75dd-0520-46ef-a5dd-f5dbb5b1b3d6, gen: 1 } is now active (read-write)
Sep 22 23:14:27.768 WARN upstairs UpstairsConnection { upstairs_id: d173eb3a-6ff3-4b10-89cb-971e46f2c897, session_id: 7d35a049-e6b2-42d0-8692-1238474c8ef7, gen: 1 } disconnected, 0 jobs left, task: main
Sep 22 23:14:27.768 WARN upstairs UpstairsConnection { upstairs_id: d173eb3a-6ff3-4b10-89cb-971e46f2c897, session_id: 7d35a049-e6b2-42d0-8692-1238474c8ef7, gen: 1 } was previously active, clearing, task: main
Sep 22 23:14:27.768 INFO connection (127.0.0.1:47731): all done
Sep 22 23:14:27.768 WARN upstairs UpstairsConnection { upstairs_id: d173eb3a-6ff3-4b10-89cb-971e46f2c897, session_id: 7d35a049-e6b2-42d0-8692-1238474c8ef7, gen: 1 } disconnected, 0 jobs left, task: main
Sep 22 23:14:27.768 WARN upstairs UpstairsConnection { upstairs_id: d173eb3a-6ff3-4b10-89cb-971e46f2c897, session_id: 7d35a049-e6b2-42d0-8692-1238474c8ef7, gen: 1 } was previously active, clearing, task: main
Sep 22 23:14:27.768 DEBG [0] Read already AckReady 1000, : downstairs
Sep 22 23:14:27.768 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.768 INFO connection (127.0.0.1:62343): all done
Sep 22 23:14:27.768 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.768 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.768 INFO Created new region file "/tmp/downstairs-hIre6xvC/region.json"
Sep 22 23:14:27.768 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.768 INFO [0] connecting to 127.0.0.1:34728, looper: 0
Sep 22 23:14:27.768 DEBG [2] Read already AckReady 1000, : downstairs
Sep 22 23:14:27.768 INFO [1] connecting to 127.0.0.1:61240, looper: 1
Sep 22 23:14:27.768 DEBG up_ds_listen was notified
Sep 22 23:14:27.768 DEBG up_ds_listen process 1000
Sep 22 23:14:27.769 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.769 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:14:27.769 INFO [2] connecting to 127.0.0.1:49345, looper: 2
Sep 22 23:14:27.769 INFO Opened existing region file "/tmp/downstairs-iS4nNbSN/region.json"
Sep 22 23:14:27.769 INFO [0] downstairs client at 127.0.0.1:40377 has UUID d97d6679-3993-407e-b222-4165f2682efb
Sep 22 23:14:27.769 INFO Database read version 1
Sep 22 23:14:27.769 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: d97d6679-3993-407e-b222-4165f2682efb, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.769 INFO Database write version 1
Sep 22 23:14:27.769 INFO up_listen starts, task: up_listen
Sep 22 23:14:27.769 INFO Wait for all three downstairs to come online
Sep 22 23:14:27.769 INFO Flush timeout: 0.5
Sep 22 23:14:27.769 INFO 8c772b52-d880-4be4-a2d6-4ced50104923 WaitActive WaitActive WaitActive
Sep 22 23:14:27.769 INFO [2] downstairs client at 127.0.0.1:53080 has UUID 5bccaeee-f6e7-4a30-aac1-e7711f75e3d5
Sep 22 23:14:27.769 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 5bccaeee-f6e7-4a30-aac1-e7711f75e3d5, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.769 INFO 8c772b52-d880-4be4-a2d6-4ced50104923 WaitActive WaitActive WaitActive
Sep 22 23:14:27.769 INFO [1] downstairs client at 127.0.0.1:50371 has UUID 13286ed5-820a-470e-a2ba-89ce1d0ebb25
Sep 22 23:14:27.769 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.769 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 13286ed5-820a-470e-a2ba-89ce1d0ebb25, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.769 INFO 8c772b52-d880-4be4-a2d6-4ced50104923 WaitActive WaitActive WaitActive
Sep 22 23:14:27.769 INFO accepted connection from 127.0.0.1:40760, task: main
Sep 22 23:14:27.769 INFO accepted connection from 127.0.0.1:53958, task: main
Sep 22 23:14:27.769 INFO [0] d173eb3a-6ff3-4b10-89cb-971e46f2c897 looper connected, looper: 0
Sep 22 23:14:27.769 INFO [0] Proc runs for 127.0.0.1:34728 in state New
Sep 22 23:14:27.769 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.769 INFO accepted connection from 127.0.0.1:53496, task: main
Sep 22 23:14:27.769 INFO [1] d173eb3a-6ff3-4b10-89cb-971e46f2c897 looper connected, looper: 1
Sep 22 23:14:27.769 INFO [1] Proc runs for 127.0.0.1:61240 in state New
Sep 22 23:14:27.769 INFO [2] d173eb3a-6ff3-4b10-89cb-971e46f2c897 looper connected, looper: 2
Sep 22 23:14:27.769 INFO [2] Proc runs for 127.0.0.1:49345 in state New
Sep 22 23:14:27.769 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.770 INFO Connection request from d173eb3a-6ff3-4b10-89cb-971e46f2c897 with version 4, task: proc
Sep 22 23:14:27.770 INFO upstairs UpstairsConnection { upstairs_id: d173eb3a-6ff3-4b10-89cb-971e46f2c897, session_id: 90c0414c-0a2e-4dcb-869c-f885bd47f77c, gen: 2 } connected, version 4, task: proc
Sep 22 23:14:27.770 INFO Connection request from d173eb3a-6ff3-4b10-89cb-971e46f2c897 with version 4, task: proc
Sep 22 23:14:27.770 INFO upstairs UpstairsConnection { upstairs_id: d173eb3a-6ff3-4b10-89cb-971e46f2c897, session_id: 90c0414c-0a2e-4dcb-869c-f885bd47f77c, gen: 2 } connected, version 4, task: proc
Sep 22 23:14:27.770 INFO Connection request from d173eb3a-6ff3-4b10-89cb-971e46f2c897 with version 4, task: proc
Sep 22 23:14:27.770 INFO upstairs UpstairsConnection { upstairs_id: d173eb3a-6ff3-4b10-89cb-971e46f2c897, session_id: 90c0414c-0a2e-4dcb-869c-f885bd47f77c, gen: 2 } connected, version 4, task: proc
Sep 22 23:14:27.770 INFO Current flush_numbers [0..12]: [0, 0]
The guest has requested activation
Sep 22 23:14:27.770 INFO [0] d173eb3a-6ff3-4b10-89cb-971e46f2c897 (90c0414c-0a2e-4dcb-869c-f885bd47f77c) New New New ds_transition to WaitActive
Sep 22 23:14:27.770 INFO [0] Transition from New to WaitActive
Sep 22 23:14:27.770 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.770 INFO [1] d173eb3a-6ff3-4b10-89cb-971e46f2c897 (90c0414c-0a2e-4dcb-869c-f885bd47f77c) WaitActive New New ds_transition to WaitActive
Sep 22 23:14:27.770 INFO [1] Transition from New to WaitActive
Sep 22 23:14:27.770 INFO [2] d173eb3a-6ff3-4b10-89cb-971e46f2c897 (90c0414c-0a2e-4dcb-869c-f885bd47f77c) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:14:27.770 INFO [2] Transition from New to WaitActive
Sep 22 23:14:27.770 INFO d173eb3a-6ff3-4b10-89cb-971e46f2c897 active request set
Sep 22 23:14:27.770 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.770 INFO [0] received activate with gen 2
Sep 22 23:14:27.770 INFO [0] client got ds_active_rx, promote! session 90c0414c-0a2e-4dcb-869c-f885bd47f77c
Sep 22 23:14:27.770 INFO [1] received activate with gen 2
Sep 22 23:14:27.770 INFO [1] client got ds_active_rx, promote! session 90c0414c-0a2e-4dcb-869c-f885bd47f77c
Sep 22 23:14:27.770 INFO [2] received activate with gen 2
Sep 22 23:14:27.770 INFO [2] client got ds_active_rx, promote! session 90c0414c-0a2e-4dcb-869c-f885bd47f77c
Sep 22 23:14:27.771 INFO UUID: cc5d982e-2876-45d7-90a8-4f67b873e08d
Sep 22 23:14:27.771 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.771 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.771 INFO [0] downstairs client at 127.0.0.1:34728 has UUID a0901c3f-7fbf-40b8-85c2-142a27a641c8
Sep 22 23:14:27.771 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: a0901c3f-7fbf-40b8-85c2-142a27a641c8, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.771 INFO d173eb3a-6ff3-4b10-89cb-971e46f2c897 WaitActive WaitActive WaitActive
Sep 22 23:14:27.771 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.771 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.771 INFO Using address: 127.0.0.1:50210, task: main
Sep 22 23:14:27.771 INFO [1] downstairs client at 127.0.0.1:61240 has UUID e45251b5-0344-4afb-bb50-e9e4af673e5f
Sep 22 23:14:27.771 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: e45251b5-0344-4afb-bb50-e9e4af673e5f, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.771 INFO [0] 8c772b52-d880-4be4-a2d6-4ced50104923 (f18e75dd-0520-46ef-a5dd-f5dbb5b1b3d6) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.771 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.771 INFO d173eb3a-6ff3-4b10-89cb-971e46f2c897 WaitActive WaitActive WaitActive
Sep 22 23:14:27.771 WARN [0] new RM replaced this: None
Sep 22 23:14:27.771 INFO [0] Starts reconcile loop
Sep 22 23:14:27.771 INFO [2] downstairs client at 127.0.0.1:49345 has UUID c2d08264-7ded-412a-9acb-ca0bc576793e
Sep 22 23:14:27.771 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: c2d08264-7ded-412a-9acb-ca0bc576793e, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.771 INFO d173eb3a-6ff3-4b10-89cb-971e46f2c897 WaitActive WaitActive WaitActive
Sep 22 23:14:27.771 INFO [2] 8c772b52-d880-4be4-a2d6-4ced50104923 (f18e75dd-0520-46ef-a5dd-f5dbb5b1b3d6) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.771 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.771 WARN [2] new RM replaced this: None
Sep 22 23:14:27.771 INFO [2] Starts reconcile loop
Sep 22 23:14:27.771 INFO [1] 8c772b52-d880-4be4-a2d6-4ced50104923 (f18e75dd-0520-46ef-a5dd-f5dbb5b1b3d6) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum
Sep 22 23:14:27.771 INFO Current flush_numbers [0..12]: [1, 1]
Sep 22 23:14:27.771 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.771 WARN [1] new RM replaced this: None
Sep 22 23:14:27.771 INFO [1] Starts reconcile loop
Sep 22 23:14:27.771 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.771 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.771 INFO [0] 127.0.0.1:40377 task reports connection:true
Sep 22 23:14:27.772 INFO 8c772b52-d880-4be4-a2d6-4ced50104923 WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:14:27.772 INFO [0]R  flush_numbers: [0, 0]
Sep 22 23:14:27.772 INFO [0]R  generation: [0, 0]
Sep 22 23:14:27.772 INFO [0]R  dirty: [false, false]
Sep 22 23:14:27.772 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:57719, task: repair
Sep 22 23:14:27.772 INFO [1]R  flush_numbers: [0, 0]
Sep 22 23:14:27.772 INFO [1]R  generation: [0, 0]
Sep 22 23:14:27.772 INFO [1]R  dirty: [false, false]
Sep 22 23:14:27.772 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:57719, task: repair
Sep 22 23:14:27.772 INFO [2]R  flush_numbers: [0, 0]
Sep 22 23:14:27.772 INFO [2]R  generation: [0, 0]
Sep 22 23:14:27.772 INFO [2]R  dirty: [false, false]
Sep 22 23:14:27.772 INFO listening, local_addr: 127.0.0.1:57719, task: repair
Sep 22 23:14:27.772 INFO Max found gen is 1
Sep 22 23:14:27.772 INFO Generation requested: 1 >= found:1
Sep 22 23:14:27.772 INFO Current flush_numbers [0..12]: [1, 1]
Sep 22 23:14:27.772 INFO Next flush: 1
Sep 22 23:14:27.772 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.772 INFO All extents match
Sep 22 23:14:27.772 INFO No downstairs repair required
Sep 22 23:14:27.772 INFO Opened existing region file "/tmp/downstairs-pJh0v4dT/region.json"
Sep 22 23:14:27.772 INFO Database read version 1
Sep 22 23:14:27.772 INFO No initial repair work was required
Sep 22 23:14:27.772 INFO Database write version 1
Sep 22 23:14:27.772 INFO Set Downstairs and Upstairs active
Sep 22 23:14:27.772 INFO 8c772b52-d880-4be4-a2d6-4ced50104923 is now active with session: f18e75dd-0520-46ef-a5dd-f5dbb5b1b3d6
Sep 22 23:14:27.772 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.772 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:57719, task: repair
Sep 22 23:14:27.772 INFO 8c772b52-d880-4be4-a2d6-4ced50104923 Set Active after no repair
Sep 22 23:14:27.772 INFO Notify all downstairs, region set compare is done.
Sep 22 23:14:27.772 INFO Using repair address: 127.0.0.1:57719, task: main
Sep 22 23:14:27.772 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.772 INFO Set check for repair
Sep 22 23:14:27.772 INFO [2] 127.0.0.1:53080 task reports connection:true
Sep 22 23:14:27.772 INFO Current flush_numbers [0..12]: [1, 1]
Sep 22 23:14:27.772 INFO 8c772b52-d880-4be4-a2d6-4ced50104923 Active Active Active
Sep 22 23:14:27.772 INFO Set check for repair
Sep 22 23:14:27.772 INFO [1] 127.0.0.1:50371 task reports connection:true
Sep 22 23:14:27.772 INFO 8c772b52-d880-4be4-a2d6-4ced50104923 Active Active Active
Sep 22 23:14:27.772 INFO Set check for repair
Sep 22 23:14:27.772 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.772 INFO [0] received reconcile message
Sep 22 23:14:27.772 INFO [0] All repairs completed, exit
Sep 22 23:14:27.772 INFO [0] Starts cmd_loop
Sep 22 23:14:27.772 INFO [0] d173eb3a-6ff3-4b10-89cb-971e46f2c897 (90c0414c-0a2e-4dcb-869c-f885bd47f77c) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.772 INFO [1] received reconcile message
Sep 22 23:14:27.772 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.772 INFO [1] All repairs completed, exit
Sep 22 23:14:27.772 WARN [0] new RM replaced this: None
Sep 22 23:14:27.772 INFO [1] Starts cmd_loop
Sep 22 23:14:27.772 INFO [0] Starts reconcile loop
Sep 22 23:14:27.772 INFO [2] received reconcile message
Sep 22 23:14:27.772 INFO [1] d173eb3a-6ff3-4b10-89cb-971e46f2c897 (90c0414c-0a2e-4dcb-869c-f885bd47f77c) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.772 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.772 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.772 WARN [1] new RM replaced this: None
Sep 22 23:14:27.772 INFO [2] All repairs completed, exit
Sep 22 23:14:27.772 INFO Opened existing region file "/tmp/downstairs-hIre6xvC/region.json"
Sep 22 23:14:27.772 INFO [1] Starts reconcile loop
Sep 22 23:14:27.772 INFO [2] Starts cmd_loop
Sep 22 23:14:27.772 INFO Database read version 1
Sep 22 23:14:27.773 INFO Database write version 1
Sep 22 23:14:27.773 INFO [2] d173eb3a-6ff3-4b10-89cb-971e46f2c897 (90c0414c-0a2e-4dcb-869c-f885bd47f77c) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.773 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.773 WARN [2] new RM replaced this: None
The guest has finished waiting for activation
Sep 22 23:14:27.773 INFO [2] Starts reconcile loop
Sep 22 23:14:27.773 INFO [0] 127.0.0.1:34728 task reports connection:true
Sep 22 23:14:27.773 INFO d173eb3a-6ff3-4b10-89cb-971e46f2c897 WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:14:27.773 INFO [0]R  flush_numbers: [1, 1]
Sep 22 23:14:27.773 INFO [0]R  generation: [1, 1]
Sep 22 23:14:27.773 INFO [0]R  dirty: [false, false]
Sep 22 23:14:27.773 INFO [1]R  flush_numbers: [1, 1]
Sep 22 23:14:27.773 INFO [1]R  generation: [1, 1]
Sep 22 23:14:27.773 INFO [1]R  dirty: [false, false]
Sep 22 23:14:27.773 INFO [2]R  flush_numbers: [1, 1]
Sep 22 23:14:27.773 INFO [2]R  generation: [1, 1]
Sep 22 23:14:27.773 INFO [2]R  dirty: [false, false]
Sep 22 23:14:27.773 INFO Max found gen is 2
Sep 22 23:14:27.773 INFO Generation requested: 2 >= found:2
Sep 22 23:14:27.773 INFO Next flush: 2
Sep 22 23:14:27.773 INFO All extents match
Sep 22 23:14:27.773 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.773 INFO No downstairs repair required
Sep 22 23:14:27.773 INFO No initial repair work was required
Sep 22 23:14:27.773 INFO Set Downstairs and Upstairs active
Sep 22 23:14:27.773 INFO d173eb3a-6ff3-4b10-89cb-971e46f2c897 is now active with session: 90c0414c-0a2e-4dcb-869c-f885bd47f77c
Sep 22 23:14:27.773 INFO d173eb3a-6ff3-4b10-89cb-971e46f2c897 Set Active after no repair
Sep 22 23:14:27.773 INFO Notify all downstairs, region set compare is done.
Sep 22 23:14:27.773 INFO Set check for repair
Sep 22 23:14:27.773 DEBG IO Read  1000 has deps []
Sep 22 23:14:27.773 INFO Created new region file "/tmp/downstairs-zWC5Bd4H/region.json"
Sep 22 23:14:27.773 INFO [1] 127.0.0.1:61240 task reports connection:true
Sep 22 23:14:27.773 INFO d173eb3a-6ff3-4b10-89cb-971e46f2c897 Active Active Active
Sep 22 23:14:27.773 INFO Set check for repair
Sep 22 23:14:27.773 INFO [2] 127.0.0.1:49345 task reports connection:true
Sep 22 23:14:27.773 DEBG IO Write 1001 has deps [JobId(1000)]
Sep 22 23:14:27.773 INFO d173eb3a-6ff3-4b10-89cb-971e46f2c897 Active Active Active
Sep 22 23:14:27.773 INFO Set check for repair
Sep 22 23:14:27.773 INFO [0] received reconcile message
Sep 22 23:14:27.773 INFO [0] All repairs completed, exit
Sep 22 23:14:27.773 INFO [0] Starts cmd_loop
Sep 22 23:14:27.773 DEBG up_ds_listen was notified
Sep 22 23:14:27.773 INFO [1] received reconcile message
Sep 22 23:14:27.773 DEBG up_ds_listen process 1001
Sep 22 23:14:27.773 INFO [1] All repairs completed, exit
Sep 22 23:14:27.773 DEBG [A] ack job 1001:2, : downstairs
Sep 22 23:14:27.773 INFO [1] Starts cmd_loop
Sep 22 23:14:27.773 INFO [2] received reconcile message
Sep 22 23:14:27.773 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.773 INFO [2] All repairs completed, exit
Sep 22 23:14:27.773 INFO [2] Starts cmd_loop
The guest has finished waiting for activation
Sep 22 23:14:27.774 DEBG IO Read  1000 has deps []
Sep 22 23:14:27.774 DEBG Read      :1000 deps:[] res:true
Sep 22 23:14:27.774 INFO UUID: 5ee54174-c6d6-4c0b-b60d-527f92f52f40
Sep 22 23:14:27.774 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.774 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.774 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.774 INFO Using address: 127.0.0.1:41940, task: main
Sep 22 23:14:27.774 DEBG Read      :1000 deps:[] res:true
Sep 22 23:14:27.774 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.774 DEBG Read      :1000 deps:[] res:true
Sep 22 23:14:27.775 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:47488, task: repair
Sep 22 23:14:27.775 DEBG Read      :1000 deps:[] res:true
Sep 22 23:14:27.775 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:47488, task: repair
Sep 22 23:14:27.775 INFO UUID: 17fc807b-48af-4955-a80e-8a9cfce39310
Sep 22 23:14:27.775 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.775 INFO listening, local_addr: 127.0.0.1:47488, task: repair
Sep 22 23:14:27.775 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.775 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.775 INFO Using address: 127.0.0.1:43395, task: main
Sep 22 23:14:27.775 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:47488, task: repair
Sep 22 23:14:27.775 INFO Using repair address: 127.0.0.1:47488, task: main
Sep 22 23:14:27.775 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.775 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.775 DEBG Read      :1000 deps:[] res:true
Sep 22 23:14:27.775 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.775 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:48582, task: repair
Sep 22 23:14:27.775 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:48582, task: repair
Sep 22 23:14:27.775 INFO listening, local_addr: 127.0.0.1:48582, task: repair
Sep 22 23:14:27.775 INFO Created new region file "/tmp/downstairs-jTnTqoHD/region.json"
Sep 22 23:14:27.776 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:48582, task: repair
Sep 22 23:14:27.776 INFO Using repair address: 127.0.0.1:48582, task: main
Sep 22 23:14:27.776 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.776 DEBG [0] Read AckReady 1000, : downstairs
Sep 22 23:14:27.776 DEBG Read      :1000 deps:[] res:true
Sep 22 23:14:27.776 DEBG [2] Read already AckReady 1000, : downstairs
Sep 22 23:14:27.776 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.776 INFO Created new region file "/tmp/downstairs-bEN17gOw/region.json"
Sep 22 23:14:27.776 DEBG [1] Read already AckReady 1000, : downstairs
Sep 22 23:14:27.776 DEBG up_ds_listen was notified
Sep 22 23:14:27.776 DEBG up_ds_listen process 1000
Sep 22 23:14:27.776 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:14:27.777 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.777 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.777 INFO Opened existing region file "/tmp/downstairs-zWC5Bd4H/region.json"
Sep 22 23:14:27.777 INFO Database read version 1
Sep 22 23:14:27.777 INFO Database write version 1
Sep 22 23:14:27.778 DEBG Write     :1001 deps:[JobId(1000)] res:true
Sep 22 23:14:27.779 DEBG Write     :1001 deps:[JobId(1000)] res:true
Sep 22 23:14:27.780 DEBG IO Write 1001 has deps [JobId(1000)]
Sep 22 23:14:27.780 DEBG up_ds_listen was notified
Sep 22 23:14:27.780 DEBG up_ds_listen process 1001
Sep 22 23:14:27.780 DEBG [A] ack job 1001:2, : downstairs
Sep 22 23:14:27.780 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.780 INFO UUID: e660550a-6e0f-4d89-b104-4c2e71068362
Sep 22 23:14:27.780 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.780 DEBG Write     :1001 deps:[JobId(1000)] res:true
Sep 22 23:14:27.780 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.780 DEBG [0] Read AckReady 1000, : downstairs
Sep 22 23:14:27.780 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.780 INFO Opened existing region file "/tmp/downstairs-jTnTqoHD/region.json"
Sep 22 23:14:27.780 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.780 INFO Database read version 1
Sep 22 23:14:27.780 INFO Database write version 1
Sep 22 23:14:27.780 INFO Using address: 127.0.0.1:58713, task: main
Sep 22 23:14:27.780 DEBG IO Read  1002 has deps [JobId(1001)]
Sep 22 23:14:27.780 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.781 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:64431, task: repair
Sep 22 23:14:27.781 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:64431, task: repair
Sep 22 23:14:27.781 INFO listening, local_addr: 127.0.0.1:64431, task: repair
Sep 22 23:14:27.781 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:64431, task: repair
Sep 22 23:14:27.781 INFO Using repair address: 127.0.0.1:64431, task: main
Sep 22 23:14:27.781 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.781 DEBG Read      :1002 deps:[JobId(1001)] res:true
Sep 22 23:14:27.781 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.781 INFO Opened existing region file "/tmp/downstairs-bEN17gOw/region.json"
Sep 22 23:14:27.781 INFO Database read version 1
Sep 22 23:14:27.781 INFO Database write version 1
Sep 22 23:14:27.782 DEBG Read      :1002 deps:[JobId(1001)] res:true
Sep 22 23:14:27.782 DEBG Read      :1002 deps:[JobId(1001)] res:true
Sep 22 23:14:27.782 INFO Upstairs starts
Sep 22 23:14:27.782 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:14:27.782 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:14:27.782 INFO Crucible stats registered with UUID: d583c1f2-1fde-4509-a36e-fd07084b7951
Sep 22 23:14:27.782 INFO Crucible d583c1f2-1fde-4509-a36e-fd07084b7951 has session id: 5384ce6a-6e03-461e-bf2d-cd3683006171
Sep 22 23:14:27.783 DEBG [1] Read already AckReady 1000, : downstairs
Sep 22 23:14:27.783 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.783 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.783 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.783 INFO [0] connecting to 127.0.0.1:62529, looper: 0
Sep 22 23:14:27.783 INFO [1] connecting to 127.0.0.1:50210, looper: 1
Sep 22 23:14:27.783 DEBG Write     :1001 deps:[JobId(1000)] res:true
Sep 22 23:14:27.783 INFO [2] connecting to 127.0.0.1:58713, looper: 2
Sep 22 23:14:27.783 INFO up_listen starts, task: up_listen
Sep 22 23:14:27.783 INFO Wait for all three downstairs to come online
Sep 22 23:14:27.783 INFO UUID: 83bc82a1-6a8c-4f0e-8d39-8e2df2a396fa
Sep 22 23:14:27.783 INFO Flush timeout: 0.5
Sep 22 23:14:27.783 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.783 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.783 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.783 INFO Using address: 127.0.0.1:61419, task: main
Sep 22 23:14:27.783 INFO accepted connection from 127.0.0.1:59757, task: main
Sep 22 23:14:27.784 INFO accepted connection from 127.0.0.1:54884, task: main
Sep 22 23:14:27.784 INFO accepted connection from 127.0.0.1:40330, task: main
Sep 22 23:14:27.784 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.784 INFO [0] d583c1f2-1fde-4509-a36e-fd07084b7951 looper connected, looper: 0
Sep 22 23:14:27.784 INFO [0] Proc runs for 127.0.0.1:62529 in state New
Sep 22 23:14:27.784 DEBG Write     :1001 deps:[JobId(1000)] res:true
Sep 22 23:14:27.784 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:43993, task: repair
Sep 22 23:14:27.784 INFO [1] d583c1f2-1fde-4509-a36e-fd07084b7951 looper connected, looper: 1
Sep 22 23:14:27.784 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:43993, task: repair
Sep 22 23:14:27.784 INFO [1] Proc runs for 127.0.0.1:50210 in state New
Sep 22 23:14:27.784 INFO listening, local_addr: 127.0.0.1:43993, task: repair
Sep 22 23:14:27.784 INFO [2] d583c1f2-1fde-4509-a36e-fd07084b7951 looper connected, looper: 2
Sep 22 23:14:27.784 INFO [2] Proc runs for 127.0.0.1:58713 in state New
Sep 22 23:14:27.784 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:43993, task: repair
Sep 22 23:14:27.784 INFO Using repair address: 127.0.0.1:43993, task: main
Sep 22 23:14:27.784 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.784 INFO Connection request from d583c1f2-1fde-4509-a36e-fd07084b7951 with version 4, task: proc
Sep 22 23:14:27.784 INFO upstairs UpstairsConnection { upstairs_id: d583c1f2-1fde-4509-a36e-fd07084b7951, session_id: ea4150c9-255d-4c41-aae3-77a60f4803c2, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.784 DEBG Write     :1001 deps:[JobId(1000)] res:true
Sep 22 23:14:27.785 INFO Connection request from d583c1f2-1fde-4509-a36e-fd07084b7951 with version 4, task: proc
Sep 22 23:14:27.785 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.785 INFO upstairs UpstairsConnection { upstairs_id: d583c1f2-1fde-4509-a36e-fd07084b7951, session_id: ea4150c9-255d-4c41-aae3-77a60f4803c2, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.785 INFO Connection request from d583c1f2-1fde-4509-a36e-fd07084b7951 with version 4, task: proc
Sep 22 23:14:27.785 INFO Created new region file "/tmp/downstairs-QARdZ8Qd/region.json"
Sep 22 23:14:27.785 INFO upstairs UpstairsConnection { upstairs_id: d583c1f2-1fde-4509-a36e-fd07084b7951, session_id: ea4150c9-255d-4c41-aae3-77a60f4803c2, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.785 INFO UUID: 69deab50-0fe3-44d0-8b06-66f3843da939
Sep 22 23:14:27.785 DEBG IO Read  1002 has deps [JobId(1001)]
Sep 22 23:14:27.785 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.785 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
The guest has requested activation
Sep 22 23:14:27.785 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.785 INFO Using address: 127.0.0.1:40972, task: main
Sep 22 23:14:27.785 INFO [0] d583c1f2-1fde-4509-a36e-fd07084b7951 (ea4150c9-255d-4c41-aae3-77a60f4803c2) New New New ds_transition to WaitActive
Sep 22 23:14:27.785 INFO [0] Transition from New to WaitActive
Sep 22 23:14:27.785 INFO [1] d583c1f2-1fde-4509-a36e-fd07084b7951 (ea4150c9-255d-4c41-aae3-77a60f4803c2) WaitActive New New ds_transition to WaitActive
Sep 22 23:14:27.785 INFO [1] Transition from New to WaitActive
Sep 22 23:14:27.785 INFO [2] d583c1f2-1fde-4509-a36e-fd07084b7951 (ea4150c9-255d-4c41-aae3-77a60f4803c2) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:14:27.785 INFO [2] Transition from New to WaitActive
Sep 22 23:14:27.785 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.785 INFO d583c1f2-1fde-4509-a36e-fd07084b7951 active request set
Sep 22 23:14:27.785 DEBG [2] Read already AckReady 1000, : downstairs
Sep 22 23:14:27.785 INFO [0] received activate with gen 1
Sep 22 23:14:27.785 DEBG up_ds_listen was notified
Sep 22 23:14:27.785 DEBG Read      :1002 deps:[JobId(1001)] res:true
Sep 22 23:14:27.785 INFO [0] client got ds_active_rx, promote! session ea4150c9-255d-4c41-aae3-77a60f4803c2
Sep 22 23:14:27.785 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:39838, task: repair
Sep 22 23:14:27.785 DEBG [1] Read AckReady 1002, : downstairs
Sep 22 23:14:27.785 DEBG up_ds_listen process 1000
Sep 22 23:14:27.785 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:39838, task: repair
Sep 22 23:14:27.785 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:14:27.785 INFO listening, local_addr: 127.0.0.1:39838, task: repair
Sep 22 23:14:27.785 INFO [1] received activate with gen 1
Sep 22 23:14:27.785 INFO [1] client got ds_active_rx, promote! session ea4150c9-255d-4c41-aae3-77a60f4803c2
Sep 22 23:14:27.786 INFO [2] received activate with gen 1
Sep 22 23:14:27.786 INFO [2] client got ds_active_rx, promote! session ea4150c9-255d-4c41-aae3-77a60f4803c2
Sep 22 23:14:27.786 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:39838, task: repair
Sep 22 23:14:27.786 INFO Using repair address: 127.0.0.1:39838, task: main
Sep 22 23:14:27.786 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.786 INFO UpstairsConnection { upstairs_id: d583c1f2-1fde-4509-a36e-fd07084b7951, session_id: ea4150c9-255d-4c41-aae3-77a60f4803c2, gen: 1 } is now active (read-write)
Sep 22 23:14:27.786 DEBG Read      :1002 deps:[JobId(1001)] res:true
Sep 22 23:14:27.786 INFO UpstairsConnection { upstairs_id: d583c1f2-1fde-4509-a36e-fd07084b7951, session_id: ea4150c9-255d-4c41-aae3-77a60f4803c2, gen: 1 } is now active (read-write)
Sep 22 23:14:27.786 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.786 INFO UpstairsConnection { upstairs_id: d583c1f2-1fde-4509-a36e-fd07084b7951, session_id: ea4150c9-255d-4c41-aae3-77a60f4803c2, gen: 1 } is now active (read-write)
Sep 22 23:14:27.786 DEBG Read      :1002 deps:[JobId(1001)] res:true
Sep 22 23:14:27.786 DEBG IO Flush 1001 has deps [JobId(1000)]
Sep 22 23:14:27.786 INFO [0] downstairs client at 127.0.0.1:62529 has UUID 2cb21d94-77f3-4d85-9833-6fd57802dc13
Sep 22 23:14:27.786 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 2cb21d94-77f3-4d85-9833-6fd57802dc13, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.787 INFO d583c1f2-1fde-4509-a36e-fd07084b7951 WaitActive WaitActive WaitActive
Sep 22 23:14:27.787 INFO [1] downstairs client at 127.0.0.1:50210 has UUID cc5d982e-2876-45d7-90a8-4f67b873e08d
Sep 22 23:14:27.787 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: cc5d982e-2876-45d7-90a8-4f67b873e08d, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.787 INFO d583c1f2-1fde-4509-a36e-fd07084b7951 WaitActive WaitActive WaitActive
Sep 22 23:14:27.787 INFO [2] downstairs client at 127.0.0.1:58713 has UUID e660550a-6e0f-4d89-b104-4c2e71068362
Sep 22 23:14:27.787 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: e660550a-6e0f-4d89-b104-4c2e71068362, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.787 INFO d583c1f2-1fde-4509-a36e-fd07084b7951 WaitActive WaitActive WaitActive
Sep 22 23:14:27.787 INFO Upstairs starts
Sep 22 23:14:27.787 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:14:27.787 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:14:27.787 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:2 g:2
Sep 22 23:14:27.787 INFO Crucible stats registered with UUID: d733e47e-2719-435d-bec3-afb64fc731d4
Sep 22 23:14:27.787 INFO Crucible d733e47e-2719-435d-bec3-afb64fc731d4 has session id: 34a9eb0f-3a64-4202-86be-263557e51230
Sep 22 23:14:27.787 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.787 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:2 g:2
Sep 22 23:14:27.787 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:2 g:2
Sep 22 23:14:27.787 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.787 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.787 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.787 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.787 INFO [0] connecting to 127.0.0.1:61471, looper: 0
Sep 22 23:14:27.787 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.787 DEBG up_ds_listen was notified
Sep 22 23:14:27.788 DEBG up_ds_listen process 1001
Sep 22 23:14:27.788 INFO [1] connecting to 127.0.0.1:43395, looper: 1
Sep 22 23:14:27.788 DEBG [A] ack job 1001:2, : downstairs
Sep 22 23:14:27.788 DEBG [rc] retire 1001 clears [JobId(1000), JobId(1001)], : downstairs
Sep 22 23:14:27.788 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.788 INFO [2] connecting to 127.0.0.1:40972, looper: 2
Sep 22 23:14:27.788 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.788 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.788 INFO up_listen starts, task: up_listen
Sep 22 23:14:27.788 INFO Wait for all three downstairs to come online
Sep 22 23:14:27.788 INFO Flush timeout: 0.5
Sep 22 23:14:27.788 INFO Created new region file "/tmp/downstairs-NuEPeMJL/region.json"
Sep 22 23:14:27.788 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.788 INFO accepted connection from 127.0.0.1:61129, task: main
Sep 22 23:14:27.788 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.788 INFO Opened existing region file "/tmp/downstairs-QARdZ8Qd/region.json"
Sep 22 23:14:27.788 INFO Database read version 1
Sep 22 23:14:27.788 INFO accepted connection from 127.0.0.1:48948, task: main
Sep 22 23:14:27.788 INFO Database write version 1
Sep 22 23:14:27.788 DEBG [0] Read already AckReady 1002, : downstairs
Sep 22 23:14:27.788 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.788 INFO accepted connection from 127.0.0.1:43249, task: main
Sep 22 23:14:27.788 INFO [0] d733e47e-2719-435d-bec3-afb64fc731d4 looper connected, looper: 0
Sep 22 23:14:27.788 INFO [0] Proc runs for 127.0.0.1:61471 in state New
Sep 22 23:14:27.788 INFO [0] d583c1f2-1fde-4509-a36e-fd07084b7951 (ea4150c9-255d-4c41-aae3-77a60f4803c2) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.788 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.788 WARN [0] new RM replaced this: None
Sep 22 23:14:27.788 INFO [1] d733e47e-2719-435d-bec3-afb64fc731d4 looper connected, looper: 1
Sep 22 23:14:27.788 INFO [1] Proc runs for 127.0.0.1:43395 in state New
Sep 22 23:14:27.788 INFO [0] Starts reconcile loop
Sep 22 23:14:27.789 INFO [2] d733e47e-2719-435d-bec3-afb64fc731d4 looper connected, looper: 2
Sep 22 23:14:27.789 INFO [1] d583c1f2-1fde-4509-a36e-fd07084b7951 (ea4150c9-255d-4c41-aae3-77a60f4803c2) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.789 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.789 INFO [2] Proc runs for 127.0.0.1:40972 in state New
Sep 22 23:14:27.789 WARN [1] new RM replaced this: None
Sep 22 23:14:27.789 INFO [1] Starts reconcile loop
Sep 22 23:14:27.789 INFO [2] d583c1f2-1fde-4509-a36e-fd07084b7951 (ea4150c9-255d-4c41-aae3-77a60f4803c2) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.789 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.789 WARN [2] new RM replaced this: None
Sep 22 23:14:27.789 INFO [2] Starts reconcile loop
Sep 22 23:14:27.789 INFO [0] 127.0.0.1:62529 task reports connection:true
Sep 22 23:14:27.789 INFO d583c1f2-1fde-4509-a36e-fd07084b7951 WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:14:27.789 INFO Connection request from d733e47e-2719-435d-bec3-afb64fc731d4 with version 4, task: proc
Sep 22 23:14:27.789 INFO [0]R  flush_numbers: [0, 0]
Sep 22 23:14:27.789 INFO [0]R  generation: [0, 0]
Sep 22 23:14:27.789 INFO upstairs UpstairsConnection { upstairs_id: d733e47e-2719-435d-bec3-afb64fc731d4, session_id: e0e1aae2-c32e-4145-89ce-1904fa05fe10, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.789 INFO [0]R  dirty: [false, false]
Sep 22 23:14:27.789 INFO [1]R  flush_numbers: [0, 0]
Sep 22 23:14:27.789 INFO [1]R  generation: [0, 0]
Sep 22 23:14:27.789 INFO [1]R  dirty: [false, false]
Sep 22 23:14:27.789 INFO [2]R  flush_numbers: [0, 0]
Sep 22 23:14:27.789 INFO [2]R  generation: [0, 0]
Sep 22 23:14:27.789 INFO [2]R  dirty: [false, false]
Sep 22 23:14:27.789 INFO Max found gen is 1
Sep 22 23:14:27.789 DEBG [0] Read AckReady 1002, : downstairs
Sep 22 23:14:27.789 INFO Generation requested: 1 >= found:1
Sep 22 23:14:27.789 INFO Next flush: 1
Sep 22 23:14:27.789 INFO Connection request from d733e47e-2719-435d-bec3-afb64fc731d4 with version 4, task: proc
Sep 22 23:14:27.789 INFO upstairs UpstairsConnection { upstairs_id: d733e47e-2719-435d-bec3-afb64fc731d4, session_id: e0e1aae2-c32e-4145-89ce-1904fa05fe10, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.789 INFO All extents match
Sep 22 23:14:27.789 INFO No downstairs repair required
Sep 22 23:14:27.789 INFO No initial repair work was required
Sep 22 23:14:27.789 INFO Set Downstairs and Upstairs active
Sep 22 23:14:27.789 INFO d583c1f2-1fde-4509-a36e-fd07084b7951 is now active with session: ea4150c9-255d-4c41-aae3-77a60f4803c2
Sep 22 23:14:27.789 INFO Connection request from d733e47e-2719-435d-bec3-afb64fc731d4 with version 4, task: proc
Sep 22 23:14:27.789 INFO d583c1f2-1fde-4509-a36e-fd07084b7951 Set Active after no repair
Sep 22 23:14:27.789 INFO upstairs UpstairsConnection { upstairs_id: d733e47e-2719-435d-bec3-afb64fc731d4, session_id: e0e1aae2-c32e-4145-89ce-1904fa05fe10, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.789 INFO Notify all downstairs, region set compare is done.
Sep 22 23:14:27.789 INFO Set check for repair
Sep 22 23:14:27.789 INFO [1] 127.0.0.1:50210 task reports connection:true
Sep 22 23:14:27.789 INFO d583c1f2-1fde-4509-a36e-fd07084b7951 Active Active Active
Sep 22 23:14:27.789 INFO Set check for repair
The guest has requested activation
Sep 22 23:14:27.789 INFO [2] 127.0.0.1:58713 task reports connection:true
Sep 22 23:14:27.789 INFO d583c1f2-1fde-4509-a36e-fd07084b7951 Active Active Active
Sep 22 23:14:27.789 INFO Set check for repair
Sep 22 23:14:27.789 INFO [0] d733e47e-2719-435d-bec3-afb64fc731d4 (e0e1aae2-c32e-4145-89ce-1904fa05fe10) New New New ds_transition to WaitActive
Sep 22 23:14:27.789 INFO [0] Transition from New to WaitActive
Sep 22 23:14:27.789 INFO [0] received reconcile message
Sep 22 23:14:27.789 INFO [0] All repairs completed, exit
Sep 22 23:14:27.789 INFO [1] d733e47e-2719-435d-bec3-afb64fc731d4 (e0e1aae2-c32e-4145-89ce-1904fa05fe10) WaitActive New New ds_transition to WaitActive
Sep 22 23:14:27.789 INFO [1] Transition from New to WaitActive
Sep 22 23:14:27.789 INFO [0] Starts cmd_loop
Sep 22 23:14:27.790 INFO [2] d733e47e-2719-435d-bec3-afb64fc731d4 (e0e1aae2-c32e-4145-89ce-1904fa05fe10) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:14:27.790 INFO [1] received reconcile message
Sep 22 23:14:27.790 INFO [2] Transition from New to WaitActive
Sep 22 23:14:27.790 INFO [1] All repairs completed, exit
Sep 22 23:14:27.790 INFO [1] Starts cmd_loop
Sep 22 23:14:27.790 INFO d733e47e-2719-435d-bec3-afb64fc731d4 active request set
Sep 22 23:14:27.790 INFO [2] received reconcile message
Sep 22 23:14:27.790 INFO [2] All repairs completed, exit
Sep 22 23:14:27.790 INFO [0] received activate with gen 1
Sep 22 23:14:27.790 INFO [2] Starts cmd_loop
Sep 22 23:14:27.790 INFO [0] client got ds_active_rx, promote! session e0e1aae2-c32e-4145-89ce-1904fa05fe10
Sep 22 23:14:27.790 INFO [1] received activate with gen 1
The guest has finished waiting for activation
Sep 22 23:14:27.790 INFO [1] client got ds_active_rx, promote! session e0e1aae2-c32e-4145-89ce-1904fa05fe10
Sep 22 23:14:27.790 INFO [2] received activate with gen 1
Sep 22 23:14:27.790 INFO [2] client got ds_active_rx, promote! session e0e1aae2-c32e-4145-89ce-1904fa05fe10
Sep 22 23:14:27.790 INFO UpstairsConnection { upstairs_id: d733e47e-2719-435d-bec3-afb64fc731d4, session_id: e0e1aae2-c32e-4145-89ce-1904fa05fe10, gen: 1 } is now active (read-write)
Sep 22 23:14:27.790 INFO UpstairsConnection { upstairs_id: d733e47e-2719-435d-bec3-afb64fc731d4, session_id: e0e1aae2-c32e-4145-89ce-1904fa05fe10, gen: 1 } is now active (read-write)
Sep 22 23:14:27.790 DEBG IO Read  1000 has deps []
Sep 22 23:14:27.790 INFO UpstairsConnection { upstairs_id: d733e47e-2719-435d-bec3-afb64fc731d4, session_id: e0e1aae2-c32e-4145-89ce-1904fa05fe10, gen: 1 } is now active (read-write)
Sep 22 23:14:27.791 INFO [0] downstairs client at 127.0.0.1:61471 has UUID cc26b772-bef2-4af0-a9fc-0733fa8e0d6d
Sep 22 23:14:27.791 DEBG [2] Read already AckReady 1002, : downstairs
Sep 22 23:14:27.791 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: cc26b772-bef2-4af0-a9fc-0733fa8e0d6d, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.791 INFO UUID: b64ddd6d-152e-4e06-b873-7258ee165522
Sep 22 23:14:27.791 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.791 INFO d733e47e-2719-435d-bec3-afb64fc731d4 WaitActive WaitActive WaitActive
Sep 22 23:14:27.791 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.791 INFO [1] downstairs client at 127.0.0.1:43395 has UUID 17fc807b-48af-4955-a80e-8a9cfce39310
Sep 22 23:14:27.791 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.791 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 17fc807b-48af-4955-a80e-8a9cfce39310, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.791 INFO Using address: 127.0.0.1:36571, task: main
Sep 22 23:14:27.791 DEBG [2] Read already AckReady 1002, : downstairs
Sep 22 23:14:27.791 INFO d733e47e-2719-435d-bec3-afb64fc731d4 WaitActive WaitActive WaitActive
Sep 22 23:14:27.791 DEBG up_ds_listen was notified
Sep 22 23:14:27.791 DEBG up_ds_listen process 1002
Sep 22 23:14:27.791 INFO [2] downstairs client at 127.0.0.1:40972 has UUID 69deab50-0fe3-44d0-8b06-66f3843da939
Sep 22 23:14:27.791 DEBG [A] ack job 1002:3, : downstairs
Sep 22 23:14:27.791 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 69deab50-0fe3-44d0-8b06-66f3843da939, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.791 INFO d733e47e-2719-435d-bec3-afb64fc731d4 WaitActive WaitActive WaitActive
Sep 22 23:14:27.791 DEBG Read      :1000 deps:[] res:true
Sep 22 23:14:27.791 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.791 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.791 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:36238, task: repair
Sep 22 23:14:27.791 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:36238, task: repair
Sep 22 23:14:27.792 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.792 INFO listening, local_addr: 127.0.0.1:36238, task: repair
Sep 22 23:14:27.792 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:36238, task: repair
Sep 22 23:14:27.792 INFO Using repair address: 127.0.0.1:36238, task: main
Sep 22 23:14:27.792 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.792 DEBG Read      :1000 deps:[] res:true
Sep 22 23:14:27.792 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.792 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.792 DEBG Read      :1000 deps:[] res:true
Sep 22 23:14:27.792 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.792 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.793 DEBG [1] Read already AckReady 1002, : downstairs
Sep 22 23:14:27.793 DEBG up_ds_listen was notified
Sep 22 23:14:27.793 DEBG up_ds_listen process 1002
Sep 22 23:14:27.793 DEBG [A] ack job 1002:3, : downstairs
Sep 22 23:14:27.793 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.793 INFO [0] d733e47e-2719-435d-bec3-afb64fc731d4 (e0e1aae2-c32e-4145-89ce-1904fa05fe10) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.793 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.793 WARN [0] new RM replaced this: None
Sep 22 23:14:27.793 INFO [0] Starts reconcile loop
Sep 22 23:14:27.793 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.793 INFO [1] d733e47e-2719-435d-bec3-afb64fc731d4 (e0e1aae2-c32e-4145-89ce-1904fa05fe10) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.793 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.793 WARN [1] new RM replaced this: None
Sep 22 23:14:27.793 INFO [1] Starts reconcile loop
Sep 22 23:14:27.793 INFO [2] d733e47e-2719-435d-bec3-afb64fc731d4 (e0e1aae2-c32e-4145-89ce-1904fa05fe10) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.793 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.793 WARN [2] new RM replaced this: None
Sep 22 23:14:27.793 INFO [2] Starts reconcile loop
Sep 22 23:14:27.793 INFO [0] 127.0.0.1:61471 task reports connection:true
Sep 22 23:14:27.793 INFO d733e47e-2719-435d-bec3-afb64fc731d4 WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:14:27.793 DEBG [0] Read AckReady 1000, : downstairs
Sep 22 23:14:27.793 INFO [0]R  flush_numbers: [0, 0]
Sep 22 23:14:27.793 INFO [0]R  generation: [0, 0]
Sep 22 23:14:27.793 INFO [0]R  dirty: [false, false]
Sep 22 23:14:27.793 INFO [1]R  flush_numbers: [0, 0]
Sep 22 23:14:27.793 INFO [1]R  generation: [0, 0]
Sep 22 23:14:27.793 INFO Upstairs starts
Sep 22 23:14:27.793 INFO [1]R  dirty: [false, false]
Sep 22 23:14:27.793 INFO [2]R  flush_numbers: [0, 0]
Sep 22 23:14:27.793 INFO [2]R  generation: [0, 0]
Sep 22 23:14:27.793 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:14:27.793 INFO [2]R  dirty: [false, false]
Sep 22 23:14:27.793 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.793 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:14:27.793 INFO Max found gen is 1
Sep 22 23:14:27.793 INFO Opened existing region file "/tmp/downstairs-NuEPeMJL/region.json"
Sep 22 23:14:27.793 INFO Database read version 1
Sep 22 23:14:27.793 INFO Crucible stats registered with UUID: 2937f30d-6553-49bb-81e4-1d5ec4457063
Sep 22 23:14:27.793 INFO Generation requested: 1 >= found:1
Sep 22 23:14:27.794 INFO Database write version 1
Sep 22 23:14:27.794 INFO Next flush: 1
Sep 22 23:14:27.794 INFO Crucible 2937f30d-6553-49bb-81e4-1d5ec4457063 has session id: 46a39c1c-6716-4158-bde6-2d848a4789fd
Sep 22 23:14:27.794 INFO All extents match
Sep 22 23:14:27.794 INFO No downstairs repair required
Sep 22 23:14:27.794 DEBG [1] Read already AckReady 1000, : downstairs
Sep 22 23:14:27.794 INFO No initial repair work was required
Sep 22 23:14:27.794 INFO Set Downstairs and Upstairs active
Sep 22 23:14:27.794 INFO d733e47e-2719-435d-bec3-afb64fc731d4 is now active with session: e0e1aae2-c32e-4145-89ce-1904fa05fe10
Sep 22 23:14:27.794 INFO d733e47e-2719-435d-bec3-afb64fc731d4 Set Active after no repair
Sep 22 23:14:27.794 INFO Notify all downstairs, region set compare is done.
Sep 22 23:14:27.794 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.794 INFO Set check for repair
Sep 22 23:14:27.794 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.794 INFO [1] 127.0.0.1:43395 task reports connection:true
Sep 22 23:14:27.794 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.794 DEBG [2] Read already AckReady 1000, : downstairs
Sep 22 23:14:27.794 INFO d733e47e-2719-435d-bec3-afb64fc731d4 Active Active Active
Sep 22 23:14:27.794 INFO Set check for repair
Sep 22 23:14:27.794 INFO [0] connecting to 127.0.0.1:41940, looper: 0
Sep 22 23:14:27.794 DEBG up_ds_listen was notified
Sep 22 23:14:27.794 INFO [2] 127.0.0.1:40972 task reports connection:true
Sep 22 23:14:27.794 DEBG up_ds_listen process 1000
Sep 22 23:14:27.794 INFO d733e47e-2719-435d-bec3-afb64fc731d4 Active Active Active
Sep 22 23:14:27.794 INFO Set check for repair
Sep 22 23:14:27.794 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:14:27.794 INFO [1] connecting to 127.0.0.1:61419, looper: 1
Sep 22 23:14:27.794 INFO [0] received reconcile message
Sep 22 23:14:27.794 INFO [0] All repairs completed, exit
Sep 22 23:14:27.794 INFO [0] Starts cmd_loop
Sep 22 23:14:27.794 INFO [1] received reconcile message
Sep 22 23:14:27.794 INFO [2] connecting to 127.0.0.1:36571, looper: 2
Sep 22 23:14:27.794 INFO [1] All repairs completed, exit
Sep 22 23:14:27.794 INFO [1] Starts cmd_loop
Sep 22 23:14:27.794 INFO up_listen starts, task: up_listen
Sep 22 23:14:27.794 INFO [2] received reconcile message
Sep 22 23:14:27.794 INFO Wait for all three downstairs to come online
Sep 22 23:14:27.794 INFO Flush timeout: 0.5
Sep 22 23:14:27.794 INFO [2] All repairs completed, exit
Sep 22 23:14:27.794 INFO [2] Starts cmd_loop
The guest has finished waiting for activation
Sep 22 23:14:27.794 INFO [0] 2937f30d-6553-49bb-81e4-1d5ec4457063 looper connected, looper: 0
Sep 22 23:14:27.794 INFO [0] Proc runs for 127.0.0.1:41940 in state New
Sep 22 23:14:27.794 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.794 INFO [1] 2937f30d-6553-49bb-81e4-1d5ec4457063 looper connected, looper: 1
Sep 22 23:14:27.794 INFO [1] Proc runs for 127.0.0.1:61419 in state New
Sep 22 23:14:27.795 INFO [2] 2937f30d-6553-49bb-81e4-1d5ec4457063 looper connected, looper: 2
Sep 22 23:14:27.795 INFO [2] Proc runs for 127.0.0.1:36571 in state New
Sep 22 23:14:27.795 DEBG IO Read  1000 has deps []
Sep 22 23:14:27.795 INFO accepted connection from 127.0.0.1:36187, task: main
Sep 22 23:14:27.795 INFO accepted connection from 127.0.0.1:43841, task: main
Sep 22 23:14:27.795 INFO accepted connection from 127.0.0.1:55496, task: main
Sep 22 23:14:27.795 INFO Connection request from 2937f30d-6553-49bb-81e4-1d5ec4457063 with version 4, task: proc
Sep 22 23:14:27.795 INFO upstairs UpstairsConnection { upstairs_id: 2937f30d-6553-49bb-81e4-1d5ec4457063, session_id: 3aafe1d5-c70b-4d8b-8ee4-fa029b9cf14d, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.795 INFO Connection request from 2937f30d-6553-49bb-81e4-1d5ec4457063 with version 4, task: proc
Sep 22 23:14:27.795 INFO upstairs UpstairsConnection { upstairs_id: 2937f30d-6553-49bb-81e4-1d5ec4457063, session_id: 3aafe1d5-c70b-4d8b-8ee4-fa029b9cf14d, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.795 INFO Connection request from 2937f30d-6553-49bb-81e4-1d5ec4457063 with version 4, task: proc
Sep 22 23:14:27.795 INFO upstairs UpstairsConnection { upstairs_id: 2937f30d-6553-49bb-81e4-1d5ec4457063, session_id: 3aafe1d5-c70b-4d8b-8ee4-fa029b9cf14d, gen: 1 } connected, version 4, task: proc
The guest has requested activation
Sep 22 23:14:27.796 INFO [0] 2937f30d-6553-49bb-81e4-1d5ec4457063 (3aafe1d5-c70b-4d8b-8ee4-fa029b9cf14d) New New New ds_transition to WaitActive
Sep 22 23:14:27.796 INFO [0] Transition from New to WaitActive
Sep 22 23:14:27.796 DEBG Read      :1000 deps:[] res:true
Sep 22 23:14:27.796 INFO [1] 2937f30d-6553-49bb-81e4-1d5ec4457063 (3aafe1d5-c70b-4d8b-8ee4-fa029b9cf14d) WaitActive New New ds_transition to WaitActive
Sep 22 23:14:27.796 INFO [1] Transition from New to WaitActive
Sep 22 23:14:27.796 INFO [2] 2937f30d-6553-49bb-81e4-1d5ec4457063 (3aafe1d5-c70b-4d8b-8ee4-fa029b9cf14d) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:14:27.796 INFO [2] Transition from New to WaitActive
Sep 22 23:14:27.796 INFO 2937f30d-6553-49bb-81e4-1d5ec4457063 active request set
Sep 22 23:14:27.796 INFO [0] received activate with gen 1
Sep 22 23:14:27.796 INFO [0] client got ds_active_rx, promote! session 3aafe1d5-c70b-4d8b-8ee4-fa029b9cf14d
Sep 22 23:14:27.796 INFO [1] received activate with gen 1
Sep 22 23:14:27.796 INFO [1] client got ds_active_rx, promote! session 3aafe1d5-c70b-4d8b-8ee4-fa029b9cf14d
Sep 22 23:14:27.796 INFO [2] received activate with gen 1
Sep 22 23:14:27.796 INFO [2] client got ds_active_rx, promote! session 3aafe1d5-c70b-4d8b-8ee4-fa029b9cf14d
Sep 22 23:14:27.796 INFO UpstairsConnection { upstairs_id: 2937f30d-6553-49bb-81e4-1d5ec4457063, session_id: 3aafe1d5-c70b-4d8b-8ee4-fa029b9cf14d, gen: 1 } is now active (read-write)
Sep 22 23:14:27.796 INFO UpstairsConnection { upstairs_id: 2937f30d-6553-49bb-81e4-1d5ec4457063, session_id: 3aafe1d5-c70b-4d8b-8ee4-fa029b9cf14d, gen: 1 } is now active (read-write)
Sep 22 23:14:27.796 DEBG Read      :1000 deps:[] res:true
Sep 22 23:14:27.796 INFO UpstairsConnection { upstairs_id: 2937f30d-6553-49bb-81e4-1d5ec4457063, session_id: 3aafe1d5-c70b-4d8b-8ee4-fa029b9cf14d, gen: 1 } is now active (read-write)
Sep 22 23:14:27.797 INFO [0] downstairs client at 127.0.0.1:41940 has UUID 5ee54174-c6d6-4c0b-b60d-527f92f52f40
Sep 22 23:14:27.797 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 5ee54174-c6d6-4c0b-b60d-527f92f52f40, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.797 INFO 2937f30d-6553-49bb-81e4-1d5ec4457063 WaitActive WaitActive WaitActive
Sep 22 23:14:27.797 INFO [1] downstairs client at 127.0.0.1:61419 has UUID 83bc82a1-6a8c-4f0e-8d39-8e2df2a396fa
Sep 22 23:14:27.797 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 83bc82a1-6a8c-4f0e-8d39-8e2df2a396fa, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.797 INFO 2937f30d-6553-49bb-81e4-1d5ec4457063 WaitActive WaitActive WaitActive
Sep 22 23:14:27.797 INFO [2] downstairs client at 127.0.0.1:36571 has UUID b64ddd6d-152e-4e06-b873-7258ee165522
Sep 22 23:14:27.797 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: b64ddd6d-152e-4e06-b873-7258ee165522, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.797 INFO 2937f30d-6553-49bb-81e4-1d5ec4457063 WaitActive WaitActive WaitActive
Sep 22 23:14:27.797 DEBG Read      :1000 deps:[] res:true
Sep 22 23:14:27.797 INFO Current flush_numbers [0..12]: [0, 0]
test test::integration_test_three_layers ... ok
Sep 22 23:14:27.797 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.798 INFO UUID: 63793045-02dd-4dbb-ad46-c6b5f41c9d53
Sep 22 23:14:27.798 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.798 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.798 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.798 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.798 INFO Using address: 127.0.0.1:53065, task: main
Sep 22 23:14:27.798 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.798 INFO Created new region file "/tmp/downstairs-5yJd5IRE/region.json"
test test::integration_test_two_layers ... ok
Sep 22 23:14:27.798 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.798 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.798 DEBG IO Write 1001 has deps [JobId(1000)]
Sep 22 23:14:27.798 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.798 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:58319, task: repair
Sep 22 23:14:27.798 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:58319, task: repair
Sep 22 23:14:27.798 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.798 INFO listening, local_addr: 127.0.0.1:58319, task: repair
Sep 22 23:14:27.798 INFO Created new region file "/tmp/downstairs-98yOFxje/region.json"
Sep 22 23:14:27.798 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:58319, task: repair
Sep 22 23:14:27.798 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.798 DEBG [0] Read AckReady 1000, : downstairs
Sep 22 23:14:27.798 INFO Using repair address: 127.0.0.1:58319, task: main
Sep 22 23:14:27.798 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.799 INFO [0] 2937f30d-6553-49bb-81e4-1d5ec4457063 (3aafe1d5-c70b-4d8b-8ee4-fa029b9cf14d) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.799 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.799 DEBG [1] Read already AckReady 1000, : downstairs
Sep 22 23:14:27.799 WARN [0] new RM replaced this: None
Sep 22 23:14:27.799 INFO [0] Starts reconcile loop
Sep 22 23:14:27.799 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.799 INFO [1] 2937f30d-6553-49bb-81e4-1d5ec4457063 (3aafe1d5-c70b-4d8b-8ee4-fa029b9cf14d) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.799 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.799 WARN [1] new RM replaced this: None
Sep 22 23:14:27.799 INFO [1] Starts reconcile loop
Sep 22 23:14:27.799 INFO Created new region file "/tmp/downstairs-A4dsswXF/region.json"
Sep 22 23:14:27.799 DEBG [2] Read already AckReady 1000, : downstairs
Sep 22 23:14:27.799 DEBG up_ds_listen was notified
Sep 22 23:14:27.799 INFO [2] 2937f30d-6553-49bb-81e4-1d5ec4457063 (3aafe1d5-c70b-4d8b-8ee4-fa029b9cf14d) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.799 DEBG up_ds_listen process 1000
Sep 22 23:14:27.799 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.799 WARN [2] new RM replaced this: None
Sep 22 23:14:27.799 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:14:27.799 INFO [2] Starts reconcile loop
Sep 22 23:14:27.799 INFO [0] 127.0.0.1:41940 task reports connection:true
Sep 22 23:14:27.799 INFO 2937f30d-6553-49bb-81e4-1d5ec4457063 WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:14:27.799 INFO [0]R  flush_numbers: [0, 0]
Sep 22 23:14:27.799 INFO [0]R  generation: [0, 0]
Sep 22 23:14:27.799 INFO [0]R  dirty: [false, false]
Sep 22 23:14:27.799 INFO [1]R  flush_numbers: [0, 0]
Sep 22 23:14:27.799 INFO [1]R  generation: [0, 0]
Sep 22 23:14:27.799 INFO [1]R  dirty: [false, false]
Sep 22 23:14:27.799 INFO [2]R  flush_numbers: [0, 0]
Sep 22 23:14:27.799 INFO [2]R  generation: [0, 0]
Sep 22 23:14:27.799 INFO [2]R  dirty: [false, false]
Sep 22 23:14:27.799 INFO Max found gen is 1
Sep 22 23:14:27.799 INFO Generation requested: 1 >= found:1
Sep 22 23:14:27.799 INFO Next flush: 1
Sep 22 23:14:27.799 INFO All extents match
Sep 22 23:14:27.799 INFO No downstairs repair required
Sep 22 23:14:27.799 INFO No initial repair work was required
Sep 22 23:14:27.799 INFO Set Downstairs and Upstairs active
Sep 22 23:14:27.799 INFO 2937f30d-6553-49bb-81e4-1d5ec4457063 is now active with session: 3aafe1d5-c70b-4d8b-8ee4-fa029b9cf14d
Sep 22 23:14:27.799 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.799 INFO 2937f30d-6553-49bb-81e4-1d5ec4457063 Set Active after no repair
Sep 22 23:14:27.799 INFO Notify all downstairs, region set compare is done.
Sep 22 23:14:27.799 INFO Set check for repair
Sep 22 23:14:27.800 INFO [1] 127.0.0.1:61419 task reports connection:true
Sep 22 23:14:27.800 INFO 2937f30d-6553-49bb-81e4-1d5ec4457063 Active Active Active
Sep 22 23:14:27.800 INFO Set check for repair
Sep 22 23:14:27.800 INFO [2] 127.0.0.1:36571 task reports connection:true
Sep 22 23:14:27.800 INFO 2937f30d-6553-49bb-81e4-1d5ec4457063 Active Active Active
Sep 22 23:14:27.800 INFO Set check for repair
Sep 22 23:14:27.800 INFO [0] received reconcile message
Sep 22 23:14:27.800 INFO [0] All repairs completed, exit
Sep 22 23:14:27.800 INFO [0] Starts cmd_loop
Sep 22 23:14:27.800 INFO [1] received reconcile message
Sep 22 23:14:27.800 INFO [1] All repairs completed, exit
Sep 22 23:14:27.800 INFO [1] Starts cmd_loop
Sep 22 23:14:27.800 INFO [2] received reconcile message
Sep 22 23:14:27.800 INFO [2] All repairs completed, exit
Sep 22 23:14:27.800 INFO [2] Starts cmd_loop
The guest has finished waiting for activation
Sep 22 23:14:27.800 DEBG IO Read  1000 has deps []
Sep 22 23:14:27.801 DEBG Read      :1000 deps:[] res:true
Sep 22 23:14:27.802 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.802 INFO Opened existing region file "/tmp/downstairs-98yOFxje/region.json"
Sep 22 23:14:27.802 INFO Database read version 1
Sep 22 23:14:27.802 INFO Database write version 1
Sep 22 23:14:27.802 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.802 INFO Opened existing region file "/tmp/downstairs-5yJd5IRE/region.json"
Sep 22 23:14:27.802 INFO Database read version 1
Sep 22 23:14:27.802 INFO Database write version 1
Sep 22 23:14:27.802 DEBG Read      :1000 deps:[] res:true
Sep 22 23:14:27.802 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.802 INFO Opened existing region file "/tmp/downstairs-A4dsswXF/region.json"
Sep 22 23:14:27.802 INFO Database read version 1
Sep 22 23:14:27.802 INFO Database write version 1
Sep 22 23:14:27.802 DEBG up_ds_listen was notified
Sep 22 23:14:27.802 DEBG up_ds_listen process 1001
Sep 22 23:14:27.802 DEBG [A] ack job 1001:2, : downstairs
Sep 22 23:14:27.802 DEBG Read      :1000 deps:[] res:true
Sep 22 23:14:27.802 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.802 DEBG IO Read  1002 has deps [JobId(1001)]
Sep 22 23:14:27.803 INFO UUID: 231089c9-45a4-4798-b0b3-3fef94d32b3b
Sep 22 23:14:27.803 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.803 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.803 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.803 INFO Using address: 127.0.0.1:39253, task: main
Sep 22 23:14:27.803 DEBG IO Write 1001 has deps [JobId(1000)]
Sep 22 23:14:27.803 DEBG up_ds_listen was notified
Sep 22 23:14:27.803 DEBG up_ds_listen process 1001
Sep 22 23:14:27.803 DEBG Read      :1002 deps:[JobId(1001)] res:true
Sep 22 23:14:27.803 DEBG [A] ack job 1001:2, : downstairs
Sep 22 23:14:27.803 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.803 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.803 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:35095, task: repair
Sep 22 23:14:27.803 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:35095, task: repair
Sep 22 23:14:27.803 DEBG [0] Read AckReady 1000, : downstairs
Sep 22 23:14:27.803 INFO listening, local_addr: 127.0.0.1:35095, task: repair
Sep 22 23:14:27.803 DEBG Read      :1002 deps:[JobId(1001)] res:true
Sep 22 23:14:27.804 INFO UUID: 73465f72-83ec-4ff4-8d45-e623bca59794
Sep 22 23:14:27.804 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:35095, task: repair
Sep 22 23:14:27.804 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.804 DEBG [1] Read already AckReady 1000, : downstairs
Sep 22 23:14:27.804 INFO Using repair address: 127.0.0.1:35095, task: main
Sep 22 23:14:27.804 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.804 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.804 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.804 DEBG Read      :1002 deps:[JobId(1001)] res:true
Sep 22 23:14:27.804 INFO Using address: 127.0.0.1:42670, task: main
Sep 22 23:14:27.804 DEBG [2] Read already AckReady 1000, : downstairs
Sep 22 23:14:27.804 DEBG up_ds_listen was notified
Sep 22 23:14:27.804 DEBG up_ds_listen process 1000
Sep 22 23:14:27.804 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:14:27.804 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.804 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.804 INFO Created new region file "/tmp/downstairs-Ap9nIgML/region.json"
Sep 22 23:14:27.804 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:54260, task: repair
Sep 22 23:14:27.804 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:54260, task: repair
Sep 22 23:14:27.804 INFO listening, local_addr: 127.0.0.1:54260, task: repair
Sep 22 23:14:27.804 INFO UUID: ee2cace4-5815-4aa2-9ac4-dbe4afbd13c0
Sep 22 23:14:27.804 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.804 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.804 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.804 INFO Using address: 127.0.0.1:39350, task: main
Sep 22 23:14:27.804 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:54260, task: repair
Sep 22 23:14:27.805 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.805 INFO Using repair address: 127.0.0.1:54260, task: main
Sep 22 23:14:27.805 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.805 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.805 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.805 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:52539, task: repair
Sep 22 23:14:27.805 INFO Created new region file "/tmp/downstairs-Kyfe8fba/region.json"
Sep 22 23:14:27.805 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:52539, task: repair
Sep 22 23:14:27.805 INFO listening, local_addr: 127.0.0.1:52539, task: repair
Sep 22 23:14:27.805 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:52539, task: repair
Sep 22 23:14:27.805 INFO Using repair address: 127.0.0.1:52539, task: main
Sep 22 23:14:27.805 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.805 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.806 INFO Created new region file "/tmp/downstairs-qgY2n0od/region.json"
Sep 22 23:14:27.806 DEBG [0] Read AckReady 1002, : downstairs
Sep 22 23:14:27.807 DEBG Write     :1001 deps:[JobId(1000)] res:true
Sep 22 23:14:27.808 DEBG Write     :1001 deps:[JobId(1000)] res:true
Sep 22 23:14:27.808 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.808 INFO Opened existing region file "/tmp/downstairs-Ap9nIgML/region.json"
Sep 22 23:14:27.808 INFO Database read version 1
Sep 22 23:14:27.808 INFO Database write version 1
Sep 22 23:14:27.808 DEBG [1] Read already AckReady 1002, : downstairs
Sep 22 23:14:27.808 DEBG IO Write 1001 has deps [JobId(1000)]
Sep 22 23:14:27.809 DEBG Write     :1001 deps:[JobId(1000)] res:true
Sep 22 23:14:27.809 DEBG IO Read  1002 has deps [JobId(1001)]
Sep 22 23:14:27.809 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.809 INFO Opened existing region file "/tmp/downstairs-Kyfe8fba/region.json"
Sep 22 23:14:27.809 INFO Database read version 1
Sep 22 23:14:27.809 INFO Database write version 1
Sep 22 23:14:27.810 DEBG Read      :1002 deps:[JobId(1001)] res:true
Sep 22 23:14:27.810 DEBG [2] Read already AckReady 1002, : downstairs
Sep 22 23:14:27.810 DEBG up_ds_listen was notified
Sep 22 23:14:27.810 DEBG up_ds_listen process 1002
Sep 22 23:14:27.810 DEBG [A] ack job 1002:3, : downstairs
Sep 22 23:14:27.810 DEBG Read      :1002 deps:[JobId(1001)] res:true
Sep 22 23:14:27.810 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.811 INFO UUID: 5d23d6e0-25c6-4cbe-ad9c-d04399dce2ff
Sep 22 23:14:27.811 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.811 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.811 INFO Opened existing region file "/tmp/downstairs-qgY2n0od/region.json"
Sep 22 23:14:27.811 INFO Database read version 1
Sep 22 23:14:27.811 INFO Database write version 1
Sep 22 23:14:27.811 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.811 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.811 INFO Using address: 127.0.0.1:46036, task: main
Sep 22 23:14:27.811 DEBG Read      :1002 deps:[JobId(1001)] res:true
Sep 22 23:14:27.811 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.811 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:58152, task: repair
Sep 22 23:14:27.811 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:58152, task: repair
Sep 22 23:14:27.811 INFO listening, local_addr: 127.0.0.1:58152, task: repair
Sep 22 23:14:27.811 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:58152, task: repair
Sep 22 23:14:27.812 INFO Using repair address: 127.0.0.1:58152, task: main
Sep 22 23:14:27.812 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.812 INFO UUID: 80f95ab7-da4b-4ddc-b498-8768e5802752
Sep 22 23:14:27.812 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.812 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.812 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.812 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.812 INFO Using address: 127.0.0.1:59248, task: main
Sep 22 23:14:27.812 INFO Created new region file "/tmp/downstairs-v7cW1efY/region.json"
Sep 22 23:14:27.813 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.813 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:52645, task: repair
Sep 22 23:14:27.813 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:52645, task: repair
Sep 22 23:14:27.813 INFO listening, local_addr: 127.0.0.1:52645, task: repair
Sep 22 23:14:27.813 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:52645, task: repair
Sep 22 23:14:27.813 INFO Using repair address: 127.0.0.1:52645, task: main
Sep 22 23:14:27.813 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.814 INFO UUID: 687c5d9e-4bfd-42d4-a91e-861a6091204d
Sep 22 23:14:27.814 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.814 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.814 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.814 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.814 INFO Using address: 127.0.0.1:40647, task: main
Sep 22 23:14:27.814 INFO Created new region file "/tmp/downstairs-M87v3oQj/region.json"
Sep 22 23:14:27.814 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.814 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:41223, task: repair
Sep 22 23:14:27.814 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:41223, task: repair
Sep 22 23:14:27.814 INFO listening, local_addr: 127.0.0.1:41223, task: repair
Sep 22 23:14:27.815 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:41223, task: repair
Sep 22 23:14:27.815 INFO Using repair address: 127.0.0.1:41223, task: main
Sep 22 23:14:27.815 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.815 DEBG [0] Read AckReady 1002, : downstairs
Sep 22 23:14:27.815 INFO Upstairs starts
Sep 22 23:14:27.815 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:14:27.815 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:14:27.815 INFO Crucible stats registered with UUID: ee584fbe-3f93-4d75-9965-f456698e3f69
Sep 22 23:14:27.815 INFO Crucible ee584fbe-3f93-4d75-9965-f456698e3f69 has session id: c547f3af-41d0-403e-8177-49c20795e0f9
test test::integration_test_two_layers_parent_smaller ... ok
Sep 22 23:14:27.815 DEBG up_ds_listen was notified
Sep 22 23:14:27.815 DEBG up_ds_listen process 1001
Sep 22 23:14:27.815 DEBG [A] ack job 1001:2, : downstairs
Sep 22 23:14:27.815 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.816 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.816 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.816 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.816 INFO [0] connecting to 127.0.0.1:53065, looper: 0
Sep 22 23:14:27.816 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.816 INFO [1] connecting to 127.0.0.1:39350, looper: 1
Sep 22 23:14:27.816 INFO Created new region file "/tmp/downstairs-AXy3vnYW/region.json"
Sep 22 23:14:27.816 INFO [2] connecting to 127.0.0.1:40647, looper: 2
Sep 22 23:14:27.816 INFO up_listen starts, task: up_listen
Sep 22 23:14:27.816 INFO Wait for all three downstairs to come online
Sep 22 23:14:27.816 INFO Flush timeout: 0.5
Sep 22 23:14:27.816 INFO accepted connection from 127.0.0.1:54717, task: main
Sep 22 23:14:27.816 INFO accepted connection from 127.0.0.1:58391, task: main
Sep 22 23:14:27.816 INFO [0] ee584fbe-3f93-4d75-9965-f456698e3f69 looper connected, looper: 0
Sep 22 23:14:27.816 INFO [0] Proc runs for 127.0.0.1:53065 in state New
Sep 22 23:14:27.816 INFO [1] ee584fbe-3f93-4d75-9965-f456698e3f69 looper connected, looper: 1
Sep 22 23:14:27.816 INFO [1] Proc runs for 127.0.0.1:39350 in state New
Sep 22 23:14:27.816 DEBG IO Read  1002 has deps [JobId(1001)]
Sep 22 23:14:27.816 INFO [2] ee584fbe-3f93-4d75-9965-f456698e3f69 looper connected, looper: 2
Sep 22 23:14:27.817 INFO [2] Proc runs for 127.0.0.1:40647 in state New
Sep 22 23:14:27.817 INFO accepted connection from 127.0.0.1:33403, task: main
Sep 22 23:14:27.817 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.817 INFO Opened existing region file "/tmp/downstairs-v7cW1efY/region.json"
Sep 22 23:14:27.817 INFO Database read version 1
Sep 22 23:14:27.817 INFO Upstairs starts
Sep 22 23:14:27.817 INFO Database write version 1
Sep 22 23:14:27.817 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:14:27.817 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:14:27.817 INFO Crucible stats registered with UUID: d173eb3a-6ff3-4b10-89cb-971e46f2c897
Sep 22 23:14:27.817 INFO Crucible d173eb3a-6ff3-4b10-89cb-971e46f2c897 has session id: 96ff5ca8-e85b-458b-8306-7ac3cdb25aba
Sep 22 23:14:27.817 INFO Connection request from ee584fbe-3f93-4d75-9965-f456698e3f69 with version 4, task: proc
Sep 22 23:14:27.817 INFO upstairs UpstairsConnection { upstairs_id: ee584fbe-3f93-4d75-9965-f456698e3f69, session_id: b4945474-3075-4e49-b652-62cc3ebba236, gen: 3 } connected, version 4, task: proc
Sep 22 23:14:27.817 INFO Connection request from ee584fbe-3f93-4d75-9965-f456698e3f69 with version 4, task: proc
Sep 22 23:14:27.817 INFO upstairs UpstairsConnection { upstairs_id: ee584fbe-3f93-4d75-9965-f456698e3f69, session_id: b4945474-3075-4e49-b652-62cc3ebba236, gen: 3 } connected, version 4, task: proc
Sep 22 23:14:27.817 DEBG [1] Read already AckReady 1002, : downstairs
Sep 22 23:14:27.817 INFO Connection request from ee584fbe-3f93-4d75-9965-f456698e3f69 with version 4, task: proc
Sep 22 23:14:27.817 INFO upstairs UpstairsConnection { upstairs_id: ee584fbe-3f93-4d75-9965-f456698e3f69, session_id: b4945474-3075-4e49-b652-62cc3ebba236, gen: 3 } connected, version 4, task: proc
Sep 22 23:14:27.817 DEBG Read      :1002 deps:[JobId(1001)] res:true
Sep 22 23:14:27.817 INFO [0] connecting to 127.0.0.1:34728, looper: 0
Sep 22 23:14:27.817 INFO [1] connecting to 127.0.0.1:61240, looper: 1
Sep 22 23:14:27.818 INFO [2] connecting to 127.0.0.1:49345, looper: 2
Sep 22 23:14:27.818 INFO up_listen starts, task: up_listen
Sep 22 23:14:27.818 INFO Wait for all three downstairs to come online
Sep 22 23:14:27.818 INFO Flush timeout: 0.5
Sep 22 23:14:27.818 INFO accepted connection from 127.0.0.1:34095, task: main
Sep 22 23:14:27.818 DEBG Read      :1002 deps:[JobId(1001)] res:true
Sep 22 23:14:27.818 INFO accepted connection from 127.0.0.1:37051, task: main
Sep 22 23:14:27.818 INFO accepted connection from 127.0.0.1:33972, task: main
Sep 22 23:14:27.818 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.818 INFO Opened existing region file "/tmp/downstairs-M87v3oQj/region.json"
Sep 22 23:14:27.818 INFO Database read version 1
Sep 22 23:14:27.818 INFO Database write version 1
Sep 22 23:14:27.818 INFO [0] ee584fbe-3f93-4d75-9965-f456698e3f69 (b4945474-3075-4e49-b652-62cc3ebba236) New New New ds_transition to WaitActive
Sep 22 23:14:27.818 INFO [0] Transition from New to WaitActive
Sep 22 23:14:27.818 INFO [1] ee584fbe-3f93-4d75-9965-f456698e3f69 (b4945474-3075-4e49-b652-62cc3ebba236) WaitActive New New ds_transition to WaitActive
Sep 22 23:14:27.818 INFO [1] Transition from New to WaitActive
Sep 22 23:14:27.818 INFO [2] ee584fbe-3f93-4d75-9965-f456698e3f69 (b4945474-3075-4e49-b652-62cc3ebba236) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:14:27.818 INFO [2] Transition from New to WaitActive
Sep 22 23:14:27.818 INFO [0] d173eb3a-6ff3-4b10-89cb-971e46f2c897 looper connected, looper: 0
Sep 22 23:14:27.818 INFO [0] Proc runs for 127.0.0.1:34728 in state New
Sep 22 23:14:27.818 INFO [1] d173eb3a-6ff3-4b10-89cb-971e46f2c897 looper connected, looper: 1
Sep 22 23:14:27.818 INFO [1] Proc runs for 127.0.0.1:61240 in state New
Sep 22 23:14:27.818 DEBG Read      :1002 deps:[JobId(1001)] res:true
Sep 22 23:14:27.818 INFO [2] d173eb3a-6ff3-4b10-89cb-971e46f2c897 looper connected, looper: 2
Sep 22 23:14:27.818 INFO [2] Proc runs for 127.0.0.1:49345 in state New
Sep 22 23:14:27.819 INFO Connection request from d173eb3a-6ff3-4b10-89cb-971e46f2c897 with version 4, task: proc
Sep 22 23:14:27.819 INFO upstairs UpstairsConnection { upstairs_id: d173eb3a-6ff3-4b10-89cb-971e46f2c897, session_id: bfec8e34-60e1-4c3f-9196-b8cad2cf6289, gen: 3 } connected, version 4, task: proc
Sep 22 23:14:27.819 INFO Connection request from d173eb3a-6ff3-4b10-89cb-971e46f2c897 with version 4, task: proc
Sep 22 23:14:27.819 INFO upstairs UpstairsConnection { upstairs_id: d173eb3a-6ff3-4b10-89cb-971e46f2c897, session_id: bfec8e34-60e1-4c3f-9196-b8cad2cf6289, gen: 3 } connected, version 4, task: proc
Sep 22 23:14:27.819 INFO Connection request from d173eb3a-6ff3-4b10-89cb-971e46f2c897 with version 4, task: proc
Sep 22 23:14:27.819 INFO upstairs UpstairsConnection { upstairs_id: d173eb3a-6ff3-4b10-89cb-971e46f2c897, session_id: bfec8e34-60e1-4c3f-9196-b8cad2cf6289, gen: 3 } connected, version 4, task: proc
Sep 22 23:14:27.819 INFO UUID: e165f739-3242-442c-822a-5ed698df44b4
Sep 22 23:14:27.819 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.819 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.819 INFO [0] d173eb3a-6ff3-4b10-89cb-971e46f2c897 (bfec8e34-60e1-4c3f-9196-b8cad2cf6289) New New New ds_transition to WaitActive
Sep 22 23:14:27.819 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.819 INFO [0] Transition from New to WaitActive
Sep 22 23:14:27.819 INFO Using address: 127.0.0.1:37718, task: main
Sep 22 23:14:27.819 INFO [1] d173eb3a-6ff3-4b10-89cb-971e46f2c897 (bfec8e34-60e1-4c3f-9196-b8cad2cf6289) WaitActive New New ds_transition to WaitActive
Sep 22 23:14:27.819 INFO [1] Transition from New to WaitActive
Sep 22 23:14:27.819 INFO [2] d173eb3a-6ff3-4b10-89cb-971e46f2c897 (bfec8e34-60e1-4c3f-9196-b8cad2cf6289) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:14:27.819 INFO [2] Transition from New to WaitActive
The guest has requested activation
Sep 22 23:14:27.820 INFO ee584fbe-3f93-4d75-9965-f456698e3f69 active request set
Sep 22 23:14:27.820 INFO [0] received activate with gen 3
Sep 22 23:14:27.820 DEBG [2] Read already AckReady 1002, : downstairs
Sep 22 23:14:27.820 INFO [0] client got ds_active_rx, promote! session b4945474-3075-4e49-b652-62cc3ebba236
Sep 22 23:14:27.820 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.820 INFO [1] received activate with gen 3
Sep 22 23:14:27.820 DEBG up_ds_listen was notified
Sep 22 23:14:27.820 INFO [1] client got ds_active_rx, promote! session b4945474-3075-4e49-b652-62cc3ebba236
Sep 22 23:14:27.820 DEBG up_ds_listen process 1002
Sep 22 23:14:27.820 INFO [2] received activate with gen 3
Sep 22 23:14:27.820 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:57972, task: repair
Sep 22 23:14:27.820 INFO [2] client got ds_active_rx, promote! session b4945474-3075-4e49-b652-62cc3ebba236
Sep 22 23:14:27.820 DEBG [A] ack job 1002:3, : downstairs
Sep 22 23:14:27.820 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:57972, task: repair
Sep 22 23:14:27.820 INFO listening, local_addr: 127.0.0.1:57972, task: repair
Sep 22 23:14:27.820 INFO UpstairsConnection { upstairs_id: ee584fbe-3f93-4d75-9965-f456698e3f69, session_id: b4945474-3075-4e49-b652-62cc3ebba236, gen: 3 } is now active (read-write)
Sep 22 23:14:27.820 INFO UpstairsConnection { upstairs_id: ee584fbe-3f93-4d75-9965-f456698e3f69, session_id: b4945474-3075-4e49-b652-62cc3ebba236, gen: 3 } is now active (read-write)
Sep 22 23:14:27.820 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:57972, task: repair
Sep 22 23:14:27.820 INFO UpstairsConnection { upstairs_id: ee584fbe-3f93-4d75-9965-f456698e3f69, session_id: b4945474-3075-4e49-b652-62cc3ebba236, gen: 3 } is now active (read-write)
Sep 22 23:14:27.820 INFO Using repair address: 127.0.0.1:57972, task: main
Sep 22 23:14:27.820 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.820 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.820 INFO [0] downstairs client at 127.0.0.1:53065 has UUID 63793045-02dd-4dbb-ad46-c6b5f41c9d53
Sep 22 23:14:27.820 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 63793045-02dd-4dbb-ad46-c6b5f41c9d53, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.820 INFO ee584fbe-3f93-4d75-9965-f456698e3f69 WaitActive WaitActive WaitActive
Sep 22 23:14:27.820 INFO [1] downstairs client at 127.0.0.1:39350 has UUID ee2cace4-5815-4aa2-9ac4-dbe4afbd13c0
Sep 22 23:14:27.820 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: ee2cace4-5815-4aa2-9ac4-dbe4afbd13c0, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.821 INFO ee584fbe-3f93-4d75-9965-f456698e3f69 WaitActive WaitActive WaitActive
Sep 22 23:14:27.821 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.821 INFO Opened existing region file "/tmp/downstairs-AXy3vnYW/region.json"
Sep 22 23:14:27.821 INFO [2] downstairs client at 127.0.0.1:40647 has UUID 687c5d9e-4bfd-42d4-a91e-861a6091204d
Sep 22 23:14:27.821 INFO Database read version 1
Sep 22 23:14:27.821 INFO Database write version 1
Sep 22 23:14:27.821 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 687c5d9e-4bfd-42d4-a91e-861a6091204d, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.821 INFO ee584fbe-3f93-4d75-9965-f456698e3f69 WaitActive WaitActive WaitActive
Sep 22 23:14:27.821 INFO UUID: 124ac7ec-a4db-423d-aef5-37bb4184fe05
Sep 22 23:14:27.821 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.821 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.821 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.821 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.821 INFO Using address: 127.0.0.1:39733, task: main
Sep 22 23:14:27.821 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.821 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.821 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.821 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.821 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:53212, task: repair
Sep 22 23:14:27.821 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:53212, task: repair
Sep 22 23:14:27.821 INFO listening, local_addr: 127.0.0.1:53212, task: repair
Sep 22 23:14:27.821 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.822 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:53212, task: repair
Sep 22 23:14:27.822 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.822 INFO Using repair address: 127.0.0.1:53212, task: main
Sep 22 23:14:27.822 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.822 INFO Upstairs starts
Sep 22 23:14:27.822 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:14:27.822 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:14:27.822 INFO Crucible stats registered with UUID: da9f63ff-0d34-4427-ba79-865cdebab321
Sep 22 23:14:27.822 INFO Crucible da9f63ff-0d34-4427-ba79-865cdebab321 has session id: b7dc2749-2f69-4d42-bf43-7535ee1f4ae4
Sep 22 23:14:27.822 INFO [0] ee584fbe-3f93-4d75-9965-f456698e3f69 (b4945474-3075-4e49-b652-62cc3ebba236) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.822 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.822 WARN [0] new RM replaced this: None
Sep 22 23:14:27.822 INFO [0] Starts reconcile loop
Sep 22 23:14:27.822 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.822 INFO [1] ee584fbe-3f93-4d75-9965-f456698e3f69 (b4945474-3075-4e49-b652-62cc3ebba236) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.822 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.822 WARN [1] new RM replaced this: None
Sep 22 23:14:27.822 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.822 INFO [1] Starts reconcile loop
Sep 22 23:14:27.822 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.822 INFO [2] ee584fbe-3f93-4d75-9965-f456698e3f69 (b4945474-3075-4e49-b652-62cc3ebba236) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.822 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.822 WARN [2] new RM replaced this: None
Sep 22 23:14:27.822 INFO [0] connecting to 127.0.0.1:39253, looper: 0
Sep 22 23:14:27.822 INFO [2] Starts reconcile loop
Sep 22 23:14:27.822 INFO [0] 127.0.0.1:53065 task reports connection:true
Sep 22 23:14:27.822 INFO ee584fbe-3f93-4d75-9965-f456698e3f69 WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:14:27.822 INFO [0]R  flush_numbers: [0, 0]
Sep 22 23:14:27.822 INFO [0]R  generation: [0, 0]
Sep 22 23:14:27.822 INFO [0]R  dirty: [false, false]
Sep 22 23:14:27.822 INFO [1] connecting to 127.0.0.1:46036, looper: 1
Sep 22 23:14:27.822 INFO [1]R  flush_numbers: [0, 0]
Sep 22 23:14:27.822 INFO [1]R  generation: [0, 0]
Sep 22 23:14:27.822 INFO [1]R  dirty: [false, false]
Sep 22 23:14:27.822 INFO [2]R  flush_numbers: [0, 0]
Sep 22 23:14:27.822 INFO [2]R  generation: [0, 0]
{"msg":"Upstairs startsSep 22 23:14:27.822 INFO [2]R  dirty: [false, false]
","v":0,"name":"crucible","level":30Sep 22 23:14:27.822 INFO Max found gen is 1
Sep 22 23:14:27.822 INFO Generation requested: 3 >= found:1
,"time":"2023-09-22T23:14:27.82289313Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
Sep 22 23:14:27.822 INFO Next flush: 1
{"msg":"Crucible Version: BuildInfo {\n    version: \"0.0.1\"Sep 22 23:14:27.822 INFO [2] connecting to 127.0.0.1:37718, looper: 2
,\n    git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n    git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n    git_branch: \"main\",\n    rustc_semver: \"Sep 22 23:14:27.822 INFO All extents match
1.70.0\",\n    rustc_channel: \"stable\",\n    rustc_host_triple: \"x86_64-unknown-illumos\",\n    rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n    cargo_triple: \"x86_64-unknown-illumos\",\n    debug: true,\n    opt_level: 0,\n}","v":Sep 22 23:14:27.822 INFO No downstairs repair required
0,"name":"crucible","level":30,"time":"Sep 22 23:14:27.822 INFO No initial repair work was required
2023-09-22T23:14:27.822983873Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{Sep 22 23:14:27.822 INFO Set Downstairs and Upstairs active
"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.823016525Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"Crucible stats registered with UUID: 77ae10ed-b83f-4607-b62b-b92e32be1f1cSep 22 23:14:27.823 INFO ee584fbe-3f93-4d75-9965-f456698e3f69 is now active with session: b4945474-3075-4e49-b652-62cc3ebba236
","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.823042292Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
Sep 22 23:14:27.823 INFO ee584fbe-3f93-4d75-9965-f456698e3f69 Set Active after no repair
{"msg":"Sep 22 23:14:27.823 INFO up_listen starts, task: up_listen
Sep 22 23:14:27.823 INFO Notify all downstairs, region set compare is done.
Crucible 77ae10ed-b83f-4607-b62b-b92e32be1f1c has session id: d9b12516-1c3f-4c95-b098-931fde4cb58b","v":0,"name":"crucible","level":30Sep 22 23:14:27.823 INFO Wait for all three downstairs to come online
,"time":"2023-09-22T23:14:27.823078924Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
Sep 22 23:14:27.823 INFO Flush timeout: 0.5
Sep 22 23:14:27.823 INFO Set check for repair
Sep 22 23:14:27.823 INFO [1] 127.0.0.1:39350 task reports connection:true
Sep 22 23:14:27.823 INFO ee584fbe-3f93-4d75-9965-f456698e3f69 Active Active Active
Sep 22 23:14:27.823 INFO Set check for repair
Sep 22 23:14:27.823 INFO [2] 127.0.0.1:40647 task reports connection:true
Sep 22 23:14:27.823 INFO ee584fbe-3f93-4d75-9965-f456698e3f69 Active Active Active
Sep 22 23:14:27.823 INFO Set check for repair
Sep 22 23:14:27.823 INFO [1] da9f63ff-0d34-4427-ba79-865cdebab321 looper connected, looper: 1
The guest has requested activation
Sep 22 23:14:27.823 INFO [1] Proc runs for 127.0.0.1:46036 in state New
Sep 22 23:14:27.823 INFO [0] received reconcile message
Sep 22 23:14:27.823 DEBG [0] Read AckReady 1002, : downstairs
Sep 22 23:14:27.823 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.823 INFO [0] All repairs completed, exit
Sep 22 23:14:27.823 INFO [0] Starts cmd_loop
Sep 22 23:14:27.823 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.823 INFO [0] da9f63ff-0d34-4427-ba79-865cdebab321 looper connected, looper: 0
Sep 22 23:14:27.823 INFO [0] Proc runs for 127.0.0.1:39253 in state New
Sep 22 23:14:27.823 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.823 INFO [2] da9f63ff-0d34-4427-ba79-865cdebab321 looper connected, looper: 2
{"Sep 22 23:14:27.823 INFO [2] Proc runs for 127.0.0.1:37718 in state New
msg":"[0] connecting to 127.0.0.1:42670","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.823647343Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"0"}
Sep 22 23:14:27.823 INFO [1] received reconcile message
Sep 22 23:14:27.823 INFO [1] All repairs completed, exit
Sep 22 23:14:27.823 INFO [1] Starts cmd_loop
{"msg":"[1] connecting to 127.0.0.1:59248","v":0,"name":"crucible","level":30Sep 22 23:14:27.823 INFO [2] received reconcile message
,"time":"2023-09-22T23:14:27.82379893Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"1"}
Sep 22 23:14:27.823 INFO [2] All repairs completed, exit
Sep 22 23:14:27.823 INFO [2] Starts cmd_loop
The guest has finished waiting for activation
{"msg":"[2] connecting to 127.0.0.1:39733","v":0,"name":"crucible","level":30Sep 22 23:14:27.823 INFO accepted connection from 127.0.0.1:60176, task: main
,"time":"2023-09-22T23:14:27.823951761Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"2"}
Sep 22 23:14:27.824 INFO accepted connection from 127.0.0.1:56689, task: main
Sep 22 23:14:27.824 INFO UUID: fe635929-4814-4e1e-bd6d-2218abf254fb
{"msg":"up_listen starts"Sep 22 23:14:27.824 INFO accepted connection from 127.0.0.1:64466, task: main
,"v":0,"name":"crucible","level":Sep 22 23:14:27.824 INFO Blocks per extent:5 Total Extents: 2
30,"time":"2023-09-22T23:14:27.824119905Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"task":"up_listen"}
{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.824170756Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"Sep 22 23:14:27.824 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.824215589Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
Sep 22 23:14:27.824 INFO Upstairs <-> Downstairs Message Version: 4, task: main
{"msg":"77ae10ed-b83f-4607-b62b-b92e32be1f1c active request set","v":0,"name":"crucible","level":30Sep 22 23:14:27.824 INFO Using address: 127.0.0.1:52814, task: main
,"time":"2023-09-22T23:14:27.824258263Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
Sep 22 23:14:27.824 INFO accepted connection from 127.0.0.1:46489, task: main
The guest has requested activation
Sep 22 23:14:27.824 INFO d173eb3a-6ff3-4b10-89cb-971e46f2c897 active request set
{"msg":"[0] 77ae10ed-b83f-4607-b62b-b92e32be1f1c looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.824543037Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"0"}
{"msg":"[0] Proc runs for 127.0.0.1:42670 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.824593927Z","Sep 22 23:14:27.824 INFO [0] received activate with gen 3
hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"Sep 22 23:14:27.824 INFO [0] client got ds_active_rx, promote! session bfec8e34-60e1-4c3f-9196-b8cad2cf6289
[1] 77ae10ed-b83f-4607-b62b-b92e32be1f1c looper connected","v":0,"name":"crucible","level":Sep 22 23:14:27.824 INFO Repair listens on 127.0.0.1:0, task: repair
30,"time":"2023-09-22T23:14:27.824652463Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"1"}
{"msg":"[1] Proc runs for 127.0.0.1:59248 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.824681174Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[2] 77ae10ed-b83f-4607-b62b-b92e32be1f1c looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.824701714Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"2"}
{"msg":"[2] Proc runs for 127.0.0.1:39733 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.824722603Z",Sep 22 23:14:27.824 INFO [1] received activate with gen 3
"hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
Sep 22 23:14:27.824 INFO [1] client got ds_active_rx, promote! session bfec8e34-60e1-4c3f-9196-b8cad2cf6289
Sep 22 23:14:27.824 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:45393, task: repair
Sep 22 23:14:27.824 INFO accepted connection from 127.0.0.1:63683, task: main
Sep 22 23:14:27.824 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:45393, task: repair
Sep 22 23:14:27.824 INFO [2] received activate with gen 3
Sep 22 23:14:27.824 INFO [2] client got ds_active_rx, promote! session bfec8e34-60e1-4c3f-9196-b8cad2cf6289
Sep 22 23:14:27.824 INFO listening, local_addr: 127.0.0.1:45393, task: repair
Sep 22 23:14:27.824 INFO accepted connection from 127.0.0.1:65112, task: main
Sep 22 23:14:27.825 INFO Signaling to UpstairsConnection { upstairs_id: d173eb3a-6ff3-4b10-89cb-971e46f2c897, session_id: 90c0414c-0a2e-4dcb-869c-f885bd47f77c, gen: 2 } thread that UpstairsConnection { upstairs_id: d173eb3a-6ff3-4b10-89cb-971e46f2c897, session_id: bfec8e34-60e1-4c3f-9196-b8cad2cf6289, gen: 3 } is being promoted (read-only)
Sep 22 23:14:27.825 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:45393, task: repair
Sep 22 23:14:27.825 INFO Using repair address: 127.0.0.1:45393, task: main
Sep 22 23:14:27.825 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.825 INFO Signaling to UpstairsConnection { upstairs_id: d173eb3a-6ff3-4b10-89cb-971e46f2c897, session_id: 90c0414c-0a2e-4dcb-869c-f885bd47f77c, gen: 2 } thread that UpstairsConnection { upstairs_id: d173eb3a-6ff3-4b10-89cb-971e46f2c897, session_id: bfec8e34-60e1-4c3f-9196-b8cad2cf6289, gen: 3 } is being promoted (read-only)
Sep 22 23:14:27.825 INFO Connection request from 77ae10ed-b83f-4607-b62b-b92e32be1f1c with version 4, task: proc
Sep 22 23:14:27.825 INFO upstairs UpstairsConnection { upstairs_id: 77ae10ed-b83f-4607-b62b-b92e32be1f1c, session_id: d1991615-52c2-49f2-91b4-d8c5e990be50, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.825 INFO Signaling to UpstairsConnection { upstairs_id: d173eb3a-6ff3-4b10-89cb-971e46f2c897, session_id: 90c0414c-0a2e-4dcb-869c-f885bd47f77c, gen: 2 } thread that UpstairsConnection { upstairs_id: d173eb3a-6ff3-4b10-89cb-971e46f2c897, session_id: bfec8e34-60e1-4c3f-9196-b8cad2cf6289, gen: 3 } is being promoted (read-only)
Sep 22 23:14:27.825 INFO Connection request from 77ae10ed-b83f-4607-b62b-b92e32be1f1c with version 4, task: proc
test test::integration_test_two_layers_parent_smaller_unwritten ... Sep 22 23:14:27.825 INFO current number of open files limit 65536 is already the maximum
ok
Sep 22 23:14:27.825 INFO upstairs UpstairsConnection { upstairs_id: 77ae10ed-b83f-4607-b62b-b92e32be1f1c, session_id: d1991615-52c2-49f2-91b4-d8c5e990be50, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.825 WARN Another upstairs UpstairsConnection { upstairs_id: d173eb3a-6ff3-4b10-89cb-971e46f2c897, session_id: bfec8e34-60e1-4c3f-9196-b8cad2cf6289, gen: 3 } promoted to active, shutting down connection for UpstairsConnection { upstairs_id: d173eb3a-6ff3-4b10-89cb-971e46f2c897, session_id: 90c0414c-0a2e-4dcb-869c-f885bd47f77c, gen: 2 }, task: main
Sep 22 23:14:27.825 INFO Connection request from 77ae10ed-b83f-4607-b62b-b92e32be1f1c with version 4, task: proc
Sep 22 23:14:27.825 INFO Created new region file "/tmp/downstairs-CtoLca49/region.json"
Sep 22 23:14:27.825 INFO connection (127.0.0.1:40760): all done
Sep 22 23:14:27.825 INFO upstairs UpstairsConnection { upstairs_id: 77ae10ed-b83f-4607-b62b-b92e32be1f1c, session_id: d1991615-52c2-49f2-91b4-d8c5e990be50, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.825 WARN Another upstairs UpstairsConnection { upstairs_id: d173eb3a-6ff3-4b10-89cb-971e46f2c897, session_id: bfec8e34-60e1-4c3f-9196-b8cad2cf6289, gen: 3 } promoted to active, shutting down connection for UpstairsConnection { upstairs_id: d173eb3a-6ff3-4b10-89cb-971e46f2c897, session_id: 90c0414c-0a2e-4dcb-869c-f885bd47f77c, gen: 2 }, task: main
Sep 22 23:14:27.825 INFO connection (127.0.0.1:53958): all done
Sep 22 23:14:27.825 WARN Another upstairs UpstairsConnection { upstairs_id: d173eb3a-6ff3-4b10-89cb-971e46f2c897, session_id: bfec8e34-60e1-4c3f-9196-b8cad2cf6289, gen: 3 } promoted to active, shutting down connection for UpstairsConnection { upstairs_id: d173eb3a-6ff3-4b10-89cb-971e46f2c897, session_id: 90c0414c-0a2e-4dcb-869c-f885bd47f77c, gen: 2 }, task: main
{"msg":"[0] 77ae10ed-b83f-4607-b62b-b92e32be1f1c (d1991615-52c2-49f2-91b4-d8c5e990be50) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.82570172Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{Sep 22 23:14:27.825 INFO connection (127.0.0.1:53496): all done
"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30Sep 22 23:14:27.825 INFO current number of open files limit 65536 is already the maximum
,"time":"2023-09-22T23:14:27.825757132Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[0] client is_active_req TRUE, promote! session d1991615-52c2-49f2-91b4-d8c5e990be50","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.825799487Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[1] 77ae10ed-b83f-4607-b62b-b92e32be1f1c (d1991615-52c2-49f2-91b4-d8c5e990be50) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.82584286Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
Sep 22 23:14:27.825 INFO Created new region file "/tmp/downstairs-MJYSYpkx/region.json"
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.82588716Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[1] client is_active_req TRUE, promote! session d1991615-52c2-49f2-91b4-d8c5e990be50","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.82592821Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[2] 77ae10ed-b83f-4607-b62b-b92e32be1f1c (d1991615-52c2-49f2-91b4-d8c5e990be50) WaitActive WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.825968738Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.826005153Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"Sep 22 23:14:27.826 ERRO [0] d173eb3a-6ff3-4b10-89cb-971e46f2c897 (90c0414c-0a2e-4dcb-869c-f885bd47f77c) cmd_loop saw YouAreNoLongerActive d173eb3a-6ff3-4b10-89cb-971e46f2c897 bfec8e34-60e1-4c3f-9196-b8cad2cf6289 3
[2] client is_active_req TRUE, promote! session d1991615-52c2-49f2-91b4-d8c5e990be50","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.826053561Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
Sep 22 23:14:27.826 INFO [0] d173eb3a-6ff3-4b10-89cb-971e46f2c897 (90c0414c-0a2e-4dcb-869c-f885bd47f77c) Active Active Active ds_transition to Disabled
Sep 22 23:14:27.826 INFO [0] Transition from Active to Disabled
Sep 22 23:14:27.826 DEBG [1] Read already AckReady 1002, : downstairs
Sep 22 23:14:27.826 INFO d173eb3a-6ff3-4b10-89cb-971e46f2c897 set inactive, session 90c0414c-0a2e-4dcb-869c-f885bd47f77c
Sep 22 23:14:27.826 ERRO 127.0.0.1:34728: proc: Generation number is too low: saw YouAreNoLongerActive with larger gen 3 than ours 2, looper: 0
Sep 22 23:14:27.826 INFO [0] d173eb3a-6ff3-4b10-89cb-971e46f2c897 Gone missing, transition from Disabled to Disconnected
Sep 22 23:14:27.826 INFO [0] d173eb3a-6ff3-4b10-89cb-971e46f2c897 connection to 127.0.0.1:34728 closed, looper: 0
Sep 22 23:14:27.826 ERRO [1] d173eb3a-6ff3-4b10-89cb-971e46f2c897 (90c0414c-0a2e-4dcb-869c-f885bd47f77c) cmd_loop saw YouAreNoLongerActive d173eb3a-6ff3-4b10-89cb-971e46f2c897 bfec8e34-60e1-4c3f-9196-b8cad2cf6289 3
Sep 22 23:14:27.826 INFO [1] d173eb3a-6ff3-4b10-89cb-971e46f2c897 (90c0414c-0a2e-4dcb-869c-f885bd47f77c) Disconnected Active Active ds_transition to Disabled
Sep 22 23:14:27.826 INFO [1] Transition from Active to Disabled
Sep 22 23:14:27.826 INFO d173eb3a-6ff3-4b10-89cb-971e46f2c897 set inactive, session 90c0414c-0a2e-4dcb-869c-f885bd47f77c
{"msg":"[0] downstairs client at 127.0.0.1:42670 has UUID 73465f72-83ec-4ff4-8d45-e623bca59794","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.826615495Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"Sep 22 23:14:27.826 ERRO 127.0.0.1:61240: proc: Generation number is too low: saw YouAreNoLongerActive with larger gen 3 than ours 2, looper: 1
[0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 73465f72-83ec-4ff4-8d45-e623bca59794, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30Sep 22 23:14:27.826 INFO [1] d173eb3a-6ff3-4b10-89cb-971e46f2c897 Gone missing, transition from Disabled to Disconnected
,"time":"2023-09-22T23:14:27.826663971Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
Sep 22 23:14:27.826 INFO [1] d173eb3a-6ff3-4b10-89cb-971e46f2c897 connection to 127.0.0.1:61240 closed, looper: 1
{"msg":"77ae10ed-b83f-4607-b62b-b92e32be1f1c WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.826706356Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[1] downstairs client at 127.0.0.1:59248 has UUID 80f95ab7-da4b-4ddc-b498-8768e5802752","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.826741841Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"Sep 22 23:14:27.826 ERRO [2] d173eb3a-6ff3-4b10-89cb-971e46f2c897 (90c0414c-0a2e-4dcb-869c-f885bd47f77c) cmd_loop saw YouAreNoLongerActive d173eb3a-6ff3-4b10-89cb-971e46f2c897 bfec8e34-60e1-4c3f-9196-b8cad2cf6289 3
[1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 80f95ab7-da4b-4ddc-b498-8768e5802752, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.826781832Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}Sep 22 23:14:27.826 INFO [2] d173eb3a-6ff3-4b10-89cb-971e46f2c897 (90c0414c-0a2e-4dcb-869c-f885bd47f77c) Disconnected Disconnected Active ds_transition to Disabled

{"msg":"Sep 22 23:14:27.826 INFO [2] Transition from Active to Disabled
77ae10ed-b83f-4607-b62b-b92e32be1f1c WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"Sep 22 23:14:27.826 INFO d173eb3a-6ff3-4b10-89cb-971e46f2c897 set inactive, session 90c0414c-0a2e-4dcb-869c-f885bd47f77c
2023-09-22T23:14:27.826830831Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[2] downstairs client at 127.0.0.1:39733 has UUID 124ac7ec-a4db-423d-aef5-37bb4184fe05","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.826877186Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 124ac7ec-a4db-423d-aef5-37bb4184fe05, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.826917455Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"77ae10ed-b83f-4607-b62b-b92e32be1f1c WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.826951795Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
Sep 22 23:14:27.826 ERRO 127.0.0.1:49345: proc: Generation number is too low: saw YouAreNoLongerActive with larger gen 3 than ours 2, looper: 2
Sep 22 23:14:27.827 INFO [2] d173eb3a-6ff3-4b10-89cb-971e46f2c897 Gone missing, transition from Disabled to Disconnected
Sep 22 23:14:27.827 INFO [2] d173eb3a-6ff3-4b10-89cb-971e46f2c897 connection to 127.0.0.1:49345 closed, looper: 2
Sep 22 23:14:27.827 WARN [0] pm_task rx.recv() is None
Sep 22 23:14:27.827 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.827 INFO [0] 127.0.0.1:34728 task reports connection:false
Sep 22 23:14:27.827 INFO d173eb3a-6ff3-4b10-89cb-971e46f2c897 Disconnected Disconnected Disconnected
Sep 22 23:14:27.827 INFO [0] 127.0.0.1:34728 task reports offline
Sep 22 23:14:27.827 INFO [1] 127.0.0.1:61240 task reports connection:false
Sep 22 23:14:27.827 INFO d173eb3a-6ff3-4b10-89cb-971e46f2c897 Disconnected Disconnected Disconnected
Sep 22 23:14:27.827 INFO [1] 127.0.0.1:61240 task reports offline
Sep 22 23:14:27.827 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.827 INFO [2] 127.0.0.1:49345 task reports connection:false
Sep 22 23:14:27.827 INFO d173eb3a-6ff3-4b10-89cb-971e46f2c897 Disconnected Disconnected Disconnected
Sep 22 23:14:27.827 INFO [2] 127.0.0.1:49345 task reports offline
Sep 22 23:14:27.827 WARN [1] pm_task rx.recv() is None
Sep 22 23:14:27.827 WARN [2] pm_task rx.recv() is None
Sep 22 23:14:27.827 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.827 INFO [0] downstairs client at 127.0.0.1:34728 has UUID a0901c3f-7fbf-40b8-85c2-142a27a641c8
Sep 22 23:14:27.827 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: a0901c3f-7fbf-40b8-85c2-142a27a641c8, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.828 INFO d173eb3a-6ff3-4b10-89cb-971e46f2c897 WaitActive WaitActive WaitActive
Sep 22 23:14:27.828 INFO [1] downstairs client at 127.0.0.1:61240 has UUID e45251b5-0344-4afb-bb50-e9e4af673e5f
Sep 22 23:14:27.828 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: e45251b5-0344-4afb-bb50-e9e4af673e5f, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.828 INFO d173eb3a-6ff3-4b10-89cb-971e46f2c897 WaitActive WaitActive WaitActive
Sep 22 23:14:27.828 INFO [2] downstairs client at 127.0.0.1:49345 has UUID c2d08264-7ded-412a-9acb-ca0bc576793e
Sep 22 23:14:27.828 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: c2d08264-7ded-412a-9acb-ca0bc576793e, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.828 INFO d173eb3a-6ff3-4b10-89cb-971e46f2c897 WaitActive WaitActive WaitActive
Sep 22 23:14:27.828 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.828 INFO Current flush_numbers [0..12]: [1, 1]
Sep 22 23:14:27.828 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.828 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.828 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.828 DEBG [2] Read already AckReady 1002, : downstairs
Sep 22 23:14:27.828 INFO Opened existing region file "/tmp/downstairs-CtoLca49/region.json"
Sep 22 23:14:27.828 INFO Database read version 1
Sep 22 23:14:27.828 INFO Database write version 1
Sep 22 23:14:27.828 DEBG up_ds_listen was notified
Sep 22 23:14:27.828 DEBG up_ds_listen process 1002
Sep 22 23:14:27.828 DEBG [A] ack job 1002:3, : downstairs
Sep 22 23:14:27.828 INFO Current flush_numbers [0..12]: [1, 1]
Sep 22 23:14:27.828 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.828 INFO Downstairs has completed Negotiation, task: proc
{"msg":"[0] 77ae10ed-b83f-4607-b62b-b92e32be1f1c (d1991615-52c2-49f2-91b4-d8c5e990be50) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.82911875Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{Sep 22 23:14:27.829 INFO Current flush_numbers [0..12]: [1, 1]
"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.829174622Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","Sep 22 23:14:27.829 DEBG up_ds_listen checked 1 jobs, back to waiting
pid":4301}
{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:27.829256161Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.829298958Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301Sep 22 23:14:27.829 INFO Downstairs has completed Negotiation, task: proc
}
{"msg":"[1] 77ae10ed-b83f-4607-b62b-b92e32be1f1c (d1991615-52c2-49f2-91b4-d8c5e990be50) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.82935074Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.829383976Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:27.829415325Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.829445925Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[2] 77ae10ed-b83f-4607-b62b-b92e32be1f1c (d1991615-52c2-49f2-91b4-d8c5e990be50) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.829481743Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
Sep 22 23:14:27.829 INFO [0] d173eb3a-6ff3-4b10-89cb-971e46f2c897 (bfec8e34-60e1-4c3f-9196-b8cad2cf6289) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30Sep 22 23:14:27.829 INFO [0] Transition from WaitActive to WaitQuorum
,"time":"2023-09-22T23:14:27.829518683Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"Sep 22 23:14:27.829 WARN [0] new RM replaced this: None
[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:27.829558923Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"Sep 22 23:14:27.829 INFO [0] Starts reconcile loop
[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.829596145Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[0] 127.0.0.1:42670 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.829628262Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"77ae10ed-b83f-4607-b62b-b92e32be1f1c WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30Sep 22 23:14:27.829 INFO [1] d173eb3a-6ff3-4b10-89cb-971e46f2c897 (bfec8e34-60e1-4c3f-9196-b8cad2cf6289) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
,"time":"2023-09-22T23:14:27.8296677Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[0]R  flush_numbers: [0, 0]Sep 22 23:14:27.829 INFO [1] Transition from WaitActive to WaitQuorum
","v":0,"name":"crucible","level":30,"time":"Sep 22 23:14:27.829 WARN [1] new RM replaced this: None
2023-09-22T23:14:27.829709698Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[0]R  generation: [0, 0]","v":0,"name":"crucible","level":30,"Sep 22 23:14:27.829 INFO [1] Starts reconcile loop
time":"2023-09-22T23:14:27.829743274Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[0]R  dirty: [false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.829778988Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[1]R  flush_numbers: [0, 0]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.829809397Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"The guest has finished waiting for activation
[1]R  generation: [0, 0]Sep 22 23:14:27.829 INFO [2] d173eb3a-6ff3-4b10-89cb-971e46f2c897 (bfec8e34-60e1-4c3f-9196-b8cad2cf6289) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
","v":0,"name":"crucible","level":30,"time":"Sep 22 23:14:27.829 INFO [2] Transition from WaitActive to WaitQuorum
2023-09-22T23:14:27.829850321Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[1]R  dirty: [false, false]","v":0,"name"Sep 22 23:14:27.829 WARN [2] new RM replaced this: None
:"crucible","level":30,"time":"2023-09-22T23:14:27.829890738Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"Sep 22 23:14:27.829 INFO [2] Starts reconcile loop
[2]R  flush_numbers: [0, 0]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.829926564Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[2]R  generation: [0, 0]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.82995768Z","hostname":"Sep 22 23:14:27.829 INFO current number of open files limit 65536 is already the maximum
ip-10-150-1-74.us-west-2.compute.internal"Sep 22 23:14:27.829 INFO [0] 127.0.0.1:34728 task reports connection:true
,"pid":4301}
{"msg":"[2]R  dirty: [false, false]","v":0,"name":"crucible","level":30Sep 22 23:14:27.829 INFO Opened existing region file "/tmp/downstairs-MJYSYpkx/region.json"
,Sep 22 23:14:27.829 INFO d173eb3a-6ff3-4b10-89cb-971e46f2c897 WaitQuorum WaitQuorum WaitQuorum
"Sep 22 23:14:27.830 INFO Database read version 1
time":"2023-09-22T23:14:27.830003224Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","Sep 22 23:14:27.830 INFO Database write version 1
pid":4301}
{"msg":"Max found gen is 1","v":0,"name":"crucible","level":30Sep 22 23:14:27.830 INFO [0]R  flush_numbers: [1, 1]
,"time":"2023-09-22T23:14:27.830048642Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
Sep 22 23:14:27.830 INFO [0]R  generation: [1, 1]
{"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible","level":30Sep 22 23:14:27.830 INFO [0]R  dirty: [false, false]
,"time":"2023-09-22T23:14:27.830082903Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msgSep 22 23:14:27.830 INFO [1]R  flush_numbers: [1, 1]
":"Next flush: 1","v":0,"name":"crucible","level":30Sep 22 23:14:27.830 INFO [1]R  generation: [1, 1]
,"time":"2023-09-22T23:14:27.830119033Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
Sep 22 23:14:27.830 INFO [1]R  dirty: [false, false]
{"msg":"All extents match","v":0,"name":"crucible","level":30Sep 22 23:14:27.830 INFO [2]R  flush_numbers: [1, 1]
,"time":"2023-09-22T23:14:27.830152662Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"Sep 22 23:14:27.830 INFO [2]R  generation: [1, 1]
No downstairs repair required","v":0,"name":"crucible","level":30Sep 22 23:14:27.830 INFO [2]R  dirty: [false, false]
,"time":"2023-09-22T23:14:27.830194341Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
Sep 22 23:14:27.830 INFO Max found gen is 2
{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"Sep 22 23:14:27.830 INFO Generation requested: 3 >= found:2
time":"2023-09-22T23:14:27.830231957Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"Sep 22 23:14:27.830 INFO Next flush: 2
Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.830267567Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301Sep 22 23:14:27.830 INFO All extents match
}
{"msg":"77ae10ed-b83f-4607-b62b-b92e32be1f1c is now active with session: d1991615-52c2-49f2-91b4-d8c5e990be50","v":0,"name":"Sep 22 23:14:27.830 INFO No downstairs repair required
crucible","level":30,"time":"2023-09-22T23:14:27.830311363Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","Sep 22 23:14:27.830 INFO No initial repair work was required
pid":4301}
{"msg":"77ae10ed-b83f-4607-b62b-b92e32be1f1c Set Active after no repair"Sep 22 23:14:27.830 INFO Set Downstairs and Upstairs active
,"v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.830353091Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"Notify all downstairs, region set compare is done.","v":Sep 22 23:14:27.830 INFO d173eb3a-6ff3-4b10-89cb-971e46f2c897 is now active with session: bfec8e34-60e1-4c3f-9196-b8cad2cf6289
0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.830388932Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal"Sep 22 23:14:27.830 INFO d173eb3a-6ff3-4b10-89cb-971e46f2c897 Set Active after no repair
,"pid":4301}
{"msg":"Set check for repair","v":0,"name":"crucible","level":30Sep 22 23:14:27.830 INFO Notify all downstairs, region set compare is done.
,"time":"2023-09-22T23:14:27.830423629Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[1] 127.0.0.1:59248 task reports connection:true","v":0,"name":"crucible","level":30Sep 22 23:14:27.830 INFO Set check for repair
,"time":"2023-09-22T23:14:27.830455403Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"77ae10ed-b83f-4607-b62b-b92e32be1f1c Active Active Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.830490609Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"Set check for repair","v":0,"name":"crucible","level":30Sep 22 23:14:27.830 INFO [1] 127.0.0.1:61240 task reports connection:true
,"time":"2023-09-22T23:14:27.830520587Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[2] 127.0.0.1:39733 task reports connection:true","v":0,"name":"crucible","level":30Sep 22 23:14:27.830 INFO d173eb3a-6ff3-4b10-89cb-971e46f2c897 Active Active Active
,"time":"2023-09-22T23:14:27.830559092Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{Sep 22 23:14:27.830 INFO Set check for repair
"msg":"77ae10ed-b83f-4607-b62b-b92e32be1f1c Active Active Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.830602663Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"Set check for repair","v":0,"name":"crucible","level":30Sep 22 23:14:27.830 INFO [2] 127.0.0.1:49345 task reports connection:true
,"time":"2023-09-22T23:14:27.830639423Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[0] received reconcile message","v":0,"nameSep 22 23:14:27.830 INFO d173eb3a-6ff3-4b10-89cb-971e46f2c897 Active Active Active
":"crucible","level":30,"time":"2023-09-22T23:14:27.830679811Z","hostname":"Sep 22 23:14:27.830 INFO Set check for repair
ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.830725517Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30Sep 22 23:14:27.830 INFO [0] received reconcile message
,"time":"2023-09-22T23:14:27.830760379Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[1] received reconcile message","v":0,"name":"crucible","Sep 22 23:14:27.830 INFO [0] All repairs completed, exit
level":30,"time":"2023-09-22T23:14:27.830806904Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301Sep 22 23:14:27.830 INFO [0] Starts cmd_loop
}
{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.830848478Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"Sep 22 23:14:27.830 INFO [1] received reconcile message
2023-09-22T23:14:27.830882621Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30Sep 22 23:14:27.830 INFO [1] All repairs completed, exit
,"time":"2023-09-22T23:14:27.830927951Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
Sep 22 23:14:27.830 INFO [1] Starts cmd_loop
{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:27.830968835Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30Sep 22 23:14:27.830 INFO [2] received reconcile message
,"time":"2023-09-22T23:14:27.83100805Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
Sep 22 23:14:27.831 INFO [2] All repairs completed, exit
Sep 22 23:14:27.831 INFO [2] Starts cmd_loop
The guest has finished waiting for activation
Sep 22 23:14:27.831 DEBG IO Read  1000 has deps []
Sep 22 23:14:27.832 INFO UUID: 8d8f9571-f1d4-4ccf-8009-092f7611cbf9
Sep 22 23:14:27.832 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.832 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.832 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.832 INFO Using address: 127.0.0.1:39989, task: main
Sep 22 23:14:27.832 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.832 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:49838, task: repair
Sep 22 23:14:27.832 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:49838, task: repair
Sep 22 23:14:27.832 INFO listening, local_addr: 127.0.0.1:49838, task: repair
Sep 22 23:14:27.832 INFO UUID: c0ea6320-dc19-4d5a-bcd9-72a0429ad856
Sep 22 23:14:27.832 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.833 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.833 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.833 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:49838, task: repair
Sep 22 23:14:27.833 INFO Using address: 127.0.0.1:43804, task: main
Sep 22 23:14:27.833 INFO Using repair address: 127.0.0.1:49838, task: main
Sep 22 23:14:27.833 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.833 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.833 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:36309, task: repair
Sep 22 23:14:27.833 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:36309, task: repair
Sep 22 23:14:27.833 INFO listening, local_addr: 127.0.0.1:36309, task: repair
Sep 22 23:14:27.833 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:36309, task: repair
Sep 22 23:14:27.833 INFO Using repair address: 127.0.0.1:36309, task: main
Sep 22 23:14:27.833 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.833 DEBG Read      :1000 deps:[] res:true
Sep 22 23:14:27.833 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.834 INFO Created new region file "/tmp/downstairs-mp86XavY/region.json"
Sep 22 23:14:27.834 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.834 INFO Created new region file "/tmp/downstairs-KmUjvjT5/region.json"
Sep 22 23:14:27.834 DEBG Read      :1000 deps:[] res:true
Sep 22 23:14:27.835 DEBG Read      :1000 deps:[] res:true
Sep 22 23:14:27.836 DEBG [0] Read AckReady 1000, : downstairs
Sep 22 23:14:27.836 DEBG [1] Read already AckReady 1000, : downstairs
Sep 22 23:14:27.836 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.836 INFO Opened existing region file "/tmp/downstairs-mp86XavY/region.json"
Sep 22 23:14:27.836 INFO Database read version 1
Sep 22 23:14:27.836 INFO Database write version 1
Sep 22 23:14:27.836 DEBG [2] Read already AckReady 1000, : downstairs
Sep 22 23:14:27.836 DEBG up_ds_listen was notified
Sep 22 23:14:27.837 DEBG up_ds_listen process 1000
Sep 22 23:14:27.837 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:14:27.837 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.837 INFO Opened existing region file "/tmp/downstairs-KmUjvjT5/region.json"
Sep 22 23:14:27.837 INFO Database read version 1
Sep 22 23:14:27.837 INFO Database write version 1
Sep 22 23:14:27.837 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.837 DEBG IO Read  1000 has deps []
Sep 22 23:14:27.838 INFO UUID: 83e10676-7666-4b88-b22b-93d3b70ce9e9
Sep 22 23:14:27.838 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.838 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.838 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.838 INFO Using address: 127.0.0.1:52387, task: main
Sep 22 23:14:27.838 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.838 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:40949, task: repair
Sep 22 23:14:27.838 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:40949, task: repair
Sep 22 23:14:27.838 INFO listening, local_addr: 127.0.0.1:40949, task: repair
Sep 22 23:14:27.838 INFO UUID: e00f36ef-b7ad-4a98-871a-5cea6209adc6
Sep 22 23:14:27.838 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.838 DEBG Read      :1000 deps:[] res:true
Sep 22 23:14:27.838 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:40949, task: repair
Sep 22 23:14:27.838 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.838 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.838 INFO Using repair address: 127.0.0.1:40949, task: main
Sep 22 23:14:27.838 INFO Using address: 127.0.0.1:60915, task: main
Sep 22 23:14:27.838 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.839 INFO Upstairs starts
Sep 22 23:14:27.839 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:14:27.839 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:14:27.839 INFO Crucible stats registered with UUID: 6ec3fe13-11cc-4835-9539-e92f50952191
Sep 22 23:14:27.839 INFO Crucible 6ec3fe13-11cc-4835-9539-e92f50952191 has session id: c5784896-58d8-45b1-afa5-9dc12f3cc620
Sep 22 23:14:27.839 DEBG Read      :1000 deps:[] res:true
Sep 22 23:14:27.839 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.839 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.839 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.839 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.839 INFO [0] connecting to 127.0.0.1:52814, looper: 0
Sep 22 23:14:27.839 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:60859, task: repair
Sep 22 23:14:27.839 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:60859, task: repair
Sep 22 23:14:27.839 INFO listening, local_addr: 127.0.0.1:60859, task: repair
Sep 22 23:14:27.839 INFO [1] connecting to 127.0.0.1:39989, looper: 1
Sep 22 23:14:27.839 DEBG Read      :1000 deps:[] res:true
Sep 22 23:14:27.839 INFO [2] connecting to 127.0.0.1:52387, looper: 2
Sep 22 23:14:27.839 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:60859, task: repair
Sep 22 23:14:27.839 INFO Using repair address: 127.0.0.1:60859, task: main
Sep 22 23:14:27.839 INFO up_listen starts, task: up_listen
Sep 22 23:14:27.839 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.839 INFO Wait for all three downstairs to come online
Sep 22 23:14:27.839 INFO Flush timeout: 0.5
Sep 22 23:14:27.839 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.839 INFO accepted connection from 127.0.0.1:50594, task: main
Sep 22 23:14:27.839 INFO Created new region file "/tmp/downstairs-Y7vkRtTl/region.json"
Sep 22 23:14:27.839 INFO accepted connection from 127.0.0.1:37331, task: main
Sep 22 23:14:27.839 INFO accepted connection from 127.0.0.1:36073, task: main
Sep 22 23:14:27.839 INFO [0] 6ec3fe13-11cc-4835-9539-e92f50952191 looper connected, looper: 0
Sep 22 23:14:27.840 INFO [0] Proc runs for 127.0.0.1:52814 in state New
Sep 22 23:14:27.840 INFO [1] 6ec3fe13-11cc-4835-9539-e92f50952191 looper connected, looper: 1
Sep 22 23:14:27.840 INFO [1] Proc runs for 127.0.0.1:39989 in state New
Sep 22 23:14:27.840 INFO [2] 6ec3fe13-11cc-4835-9539-e92f50952191 looper connected, looper: 2
test test::integration_test_two_layers_write_unwritten ... ok
Sep 22 23:14:27.840 INFO [2] Proc runs for 127.0.0.1:52387 in state New
test test::integration_test_upstairs_read_only_rejects_write ... ok
Sep 22 23:14:27.840 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.840 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.840 INFO Created new region file "/tmp/downstairs-vJkzDs9L/region.json"
Sep 22 23:14:27.840 INFO Created new region file "/tmp/downstairs-NPgDrsYr/region.json"
Sep 22 23:14:27.840 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.840 INFO Created new region file "/tmp/downstairs-rxcAS3Gc/region.json"
Sep 22 23:14:27.843 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.843 INFO Opened existing region file "/tmp/downstairs-Y7vkRtTl/region.json"
Sep 22 23:14:27.843 INFO Database read version 1
Sep 22 23:14:27.843 INFO Database write version 1
Sep 22 23:14:27.843 DEBG [0] Read AckReady 1000, : downstairs
Sep 22 23:14:27.844 INFO UUID: 7c134907-732f-4421-b0bd-357e07862d5c
Sep 22 23:14:27.844 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.844 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.844 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.844 INFO Using address: 127.0.0.1:42061, task: main
Sep 22 23:14:27.845 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.845 INFO Opened existing region file "/tmp/downstairs-NPgDrsYr/region.json"
Sep 22 23:14:27.845 INFO Database read version 1
Sep 22 23:14:27.845 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.845 INFO Database write version 1
Sep 22 23:14:27.845 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.845 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.845 INFO Opened existing region file "/tmp/downstairs-rxcAS3Gc/region.json"
Sep 22 23:14:27.845 INFO Opened existing region file "/tmp/downstairs-vJkzDs9L/region.json"
Sep 22 23:14:27.845 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:38359, task: repair
Sep 22 23:14:27.845 INFO Database read version 1
Sep 22 23:14:27.845 INFO Database read version 1
Sep 22 23:14:27.845 INFO Database write version 1
Sep 22 23:14:27.845 INFO Database write version 1
Sep 22 23:14:27.845 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:38359, task: repair
Sep 22 23:14:27.845 INFO listening, local_addr: 127.0.0.1:38359, task: repair
Sep 22 23:14:27.845 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:38359, task: repair
Sep 22 23:14:27.845 INFO Using repair address: 127.0.0.1:38359, task: main
Sep 22 23:14:27.845 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.846 DEBG [1] Read already AckReady 1000, : downstairs
Sep 22 23:14:27.846 INFO Upstairs starts
Sep 22 23:14:27.847 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:14:27.847 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:14:27.847 INFO Crucible stats registered with UUID: 63bdb731-3b56-40e2-821a-221152f1277d
Sep 22 23:14:27.847 INFO Crucible 63bdb731-3b56-40e2-821a-221152f1277d has session id: 230ff5c2-9acf-42b9-b693-e86cdd9897cd
Sep 22 23:14:27.847 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.847 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.847 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.847 INFO [0] connecting to 127.0.0.1:43804, looper: 0
Sep 22 23:14:27.847 INFO [1] connecting to 127.0.0.1:60915, looper: 1
Sep 22 23:14:27.847 INFO [2] connecting to 127.0.0.1:42061, looper: 2
Sep 22 23:14:27.847 INFO up_listen starts, task: up_listen
Sep 22 23:14:27.847 INFO Wait for all three downstairs to come online
Sep 22 23:14:27.847 INFO Flush timeout: 0.5
Sep 22 23:14:27.848 INFO accepted connection from 127.0.0.1:63241, task: main
Sep 22 23:14:27.848 INFO accepted connection from 127.0.0.1:55431, task: main
Sep 22 23:14:27.848 INFO accepted connection from 127.0.0.1:35399, task: main
Sep 22 23:14:27.848 INFO [0] 63bdb731-3b56-40e2-821a-221152f1277d looper connected, looper: 0
Sep 22 23:14:27.848 INFO [0] Proc runs for 127.0.0.1:43804 in state New
Sep 22 23:14:27.848 INFO UUID: 8e0311d9-6cee-4187-8ec5-c1067f73eb45
Sep 22 23:14:27.848 INFO [1] 63bdb731-3b56-40e2-821a-221152f1277d looper connected, looper: 1
Sep 22 23:14:27.848 INFO UUID: 81b401c7-cdba-4739-be96-9800dad0d240
Sep 22 23:14:27.848 INFO UUID: 30596aa6-8959-4f8c-98b8-ac91fdc30fc1
Sep 22 23:14:27.848 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.848 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.848 INFO [1] Proc runs for 127.0.0.1:60915 in state New
Sep 22 23:14:27.848 DEBG [2] Read already AckReady 1000, : downstairs
Sep 22 23:14:27.848 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.848 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.848 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.848 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.848 INFO Using address: 127.0.0.1:52905, task: main
Sep 22 23:14:27.848 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.848 INFO Using address: 127.0.0.1:47986, task: main
Sep 22 23:14:27.848 DEBG up_ds_listen was notified
Sep 22 23:14:27.848 INFO [2] 63bdb731-3b56-40e2-821a-221152f1277d looper connected, looper: 2
Sep 22 23:14:27.848 DEBG up_ds_listen process 1000
Sep 22 23:14:27.848 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.848 INFO [2] Proc runs for 127.0.0.1:42061 in state New
Sep 22 23:14:27.848 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:14:27.848 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.848 INFO Using address: 127.0.0.1:45648, task: main
Sep 22 23:14:27.849 INFO Connection request from 63bdb731-3b56-40e2-821a-221152f1277d with version 4, task: proc
Sep 22 23:14:27.849 INFO upstairs UpstairsConnection { upstairs_id: 63bdb731-3b56-40e2-821a-221152f1277d, session_id: 089624e5-8c38-462d-b605-f332f3805c9d, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.849 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.849 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.849 INFO Connection request from 63bdb731-3b56-40e2-821a-221152f1277d with version 4, task: proc
Sep 22 23:14:27.849 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:57508, task: repair
Sep 22 23:14:27.849 INFO upstairs UpstairsConnection { upstairs_id: 63bdb731-3b56-40e2-821a-221152f1277d, session_id: 089624e5-8c38-462d-b605-f332f3805c9d, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.849 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:57508, task: repair
Sep 22 23:14:27.849 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.849 INFO listening, local_addr: 127.0.0.1:57508, task: repair
Sep 22 23:14:27.849 INFO Connection request from 63bdb731-3b56-40e2-821a-221152f1277d with version 4, task: proc
Sep 22 23:14:27.849 INFO upstairs UpstairsConnection { upstairs_id: 63bdb731-3b56-40e2-821a-221152f1277d, session_id: 089624e5-8c38-462d-b605-f332f3805c9d, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.849 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:36948, task: repair
Sep 22 23:14:27.849 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:57508, task: repair
Sep 22 23:14:27.849 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:36948, task: repair
Sep 22 23:14:27.849 INFO Using repair address: 127.0.0.1:57508, task: main
Sep 22 23:14:27.849 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.849 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.849 INFO listening, local_addr: 127.0.0.1:36948, task: repair
The guest has requested activation
Sep 22 23:14:27.849 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:58541, task: repair
Sep 22 23:14:27.849 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.849 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:58541, task: repair
Sep 22 23:14:27.849 INFO listening, local_addr: 127.0.0.1:58541, task: repair
Sep 22 23:14:27.849 INFO [0] 63bdb731-3b56-40e2-821a-221152f1277d (089624e5-8c38-462d-b605-f332f3805c9d) New New New ds_transition to WaitActive
Sep 22 23:14:27.849 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:36948, task: repair
Sep 22 23:14:27.849 INFO [0] Transition from New to WaitActive
Sep 22 23:14:27.849 INFO Created new region file "/tmp/downstairs-b4bTiF9L/region.json"
Sep 22 23:14:27.849 INFO Using repair address: 127.0.0.1:36948, task: main
Sep 22 23:14:27.849 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.849 INFO [1] 63bdb731-3b56-40e2-821a-221152f1277d (089624e5-8c38-462d-b605-f332f3805c9d) WaitActive New New ds_transition to WaitActive
Sep 22 23:14:27.849 INFO [1] Transition from New to WaitActive
Sep 22 23:14:27.849 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:58541, task: repair
Sep 22 23:14:27.849 INFO Using repair address: 127.0.0.1:58541, task: main
Sep 22 23:14:27.849 INFO [2] 63bdb731-3b56-40e2-821a-221152f1277d (089624e5-8c38-462d-b605-f332f3805c9d) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:14:27.849 DEBG IO Flush 1001 has deps [JobId(1000)]
Sep 22 23:14:27.849 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.850 INFO [2] Transition from New to WaitActive
Sep 22 23:14:27.850 INFO 63bdb731-3b56-40e2-821a-221152f1277d active request set
Sep 22 23:14:27.850 INFO [0] received activate with gen 1
Sep 22 23:14:27.850 INFO [0] client got ds_active_rx, promote! session 089624e5-8c38-462d-b605-f332f3805c9d
Sep 22 23:14:27.850 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.850 INFO Connection request from 6ec3fe13-11cc-4835-9539-e92f50952191 with version 4, task: proc
Sep 22 23:14:27.850 INFO upstairs UpstairsConnection { upstairs_id: 6ec3fe13-11cc-4835-9539-e92f50952191, session_id: 2adb6f46-b730-4b58-84d3-87af2b7f60e0, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.850 INFO [1] received activate with gen 1
Sep 22 23:14:27.850 INFO [1] client got ds_active_rx, promote! session 089624e5-8c38-462d-b605-f332f3805c9d
Sep 22 23:14:27.850 INFO Created new region file "/tmp/downstairs-zarv4usP/region.json"
Sep 22 23:14:27.850 INFO Connection request from 6ec3fe13-11cc-4835-9539-e92f50952191 with version 4, task: proc
Sep 22 23:14:27.850 INFO upstairs UpstairsConnection { upstairs_id: 6ec3fe13-11cc-4835-9539-e92f50952191, session_id: 2adb6f46-b730-4b58-84d3-87af2b7f60e0, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.850 INFO [2] received activate with gen 1
Sep 22 23:14:27.850 INFO [2] client got ds_active_rx, promote! session 089624e5-8c38-462d-b605-f332f3805c9d
Sep 22 23:14:27.850 INFO Connection request from 6ec3fe13-11cc-4835-9539-e92f50952191 with version 4, task: proc
Sep 22 23:14:27.850 INFO upstairs UpstairsConnection { upstairs_id: 6ec3fe13-11cc-4835-9539-e92f50952191, session_id: 2adb6f46-b730-4b58-84d3-87af2b7f60e0, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.850 INFO UpstairsConnection { upstairs_id: 63bdb731-3b56-40e2-821a-221152f1277d, session_id: 089624e5-8c38-462d-b605-f332f3805c9d, gen: 1 } is now active (read-write)
Sep 22 23:14:27.850 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.850 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:1 g:3
Sep 22 23:14:27.850 WARN 6ec3fe13-11cc-4835-9539-e92f50952191 request to replace downstairs 127.0.0.1:52814 with 127.0.0.1:45648
Sep 22 23:14:27.850 INFO UpstairsConnection { upstairs_id: 63bdb731-3b56-40e2-821a-221152f1277d, session_id: 089624e5-8c38-462d-b605-f332f3805c9d, gen: 1 } is now active (read-write)
Sep 22 23:14:27.850 INFO 6ec3fe13-11cc-4835-9539-e92f50952191 found old target: 127.0.0.1:52814 at 0
Sep 22 23:14:27.850 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:1 g:3
Sep 22 23:14:27.850 INFO 6ec3fe13-11cc-4835-9539-e92f50952191 replacing old: 127.0.0.1:52814 at 0
Sep 22 23:14:27.850 INFO [0] client skip 0 in process jobs because fault, : downstairs
Sep 22 23:14:27.850 INFO [0] changed 0 jobs to fault skipped, : downstairs
Sep 22 23:14:27.850 INFO [0] 6ec3fe13-11cc-4835-9539-e92f50952191 (2adb6f46-b730-4b58-84d3-87af2b7f60e0) New New New ds_transition to Replacing
Sep 22 23:14:27.850 INFO UpstairsConnection { upstairs_id: 63bdb731-3b56-40e2-821a-221152f1277d, session_id: 089624e5-8c38-462d-b605-f332f3805c9d, gen: 1 } is now active (read-write)
Sep 22 23:14:27.850 INFO [0] Transition from New to Replacing
Sep 22 23:14:27.850 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:1 g:3
Sep 22 23:14:27.851 DEBG up_ds_listen was notified
Sep 22 23:14:27.851 DEBG up_ds_listen process 1001
Sep 22 23:14:27.851 DEBG [A] ack job 1001:2, : downstairs
Sep 22 23:14:27.851 DEBG [rc] retire 1001 clears [JobId(1000), JobId(1001)], : downstairs
Sep 22 23:14:27.851 INFO [0] downstairs client at 127.0.0.1:43804 has UUID c0ea6320-dc19-4d5a-bcd9-72a0429ad856
Sep 22 23:14:27.851 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.851 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: c0ea6320-dc19-4d5a-bcd9-72a0429ad856, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.851 INFO 63bdb731-3b56-40e2-821a-221152f1277d WaitActive WaitActive WaitActive
Sep 22 23:14:27.851 INFO [1] downstairs client at 127.0.0.1:60915 has UUID e00f36ef-b7ad-4a98-871a-5cea6209adc6
Sep 22 23:14:27.851 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: e00f36ef-b7ad-4a98-871a-5cea6209adc6, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.851 INFO 63bdb731-3b56-40e2-821a-221152f1277d WaitActive WaitActive WaitActive
Sep 22 23:14:27.851 INFO [2] downstairs client at 127.0.0.1:42061 has UUID 7c134907-732f-4421-b0bd-357e07862d5c
Sep 22 23:14:27.851 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 7c134907-732f-4421-b0bd-357e07862d5c, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.851 INFO 63bdb731-3b56-40e2-821a-221152f1277d WaitActive WaitActive WaitActive
Sep 22 23:14:27.851 DEBG IO Write 1002 has deps []
Sep 22 23:14:27.851 DEBG up_ds_listen was notified
Sep 22 23:14:27.851 DEBG up_ds_listen process 1002
Sep 22 23:14:27.852 DEBG [A] ack job 1002:3, : downstairs
Sep 22 23:14:27.852 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.852 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.853 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.853 DEBG Write     :1002 deps:[] res:true
Sep 22 23:14:27.853 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.853 INFO Opened existing region file "/tmp/downstairs-b4bTiF9L/region.json"
Sep 22 23:14:27.853 INFO Database read version 1
Sep 22 23:14:27.853 INFO Database write version 1
Sep 22 23:14:27.853 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.853 DEBG Write     :1002 deps:[] res:true
Sep 22 23:14:27.854 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.854 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.855 DEBG Write     :1002 deps:[] res:true
Sep 22 23:14:27.855 INFO Downstairs has completed Negotiation, task: proc
test test::integration_test_volume_inactive_replace_downstairs ... ok
Sep 22 23:14:27.855 INFO UUID: 05b93ff5-c936-4fa6-adb6-db93f2642835
Sep 22 23:14:27.855 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.855 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.855 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.855 INFO [0] 63bdb731-3b56-40e2-821a-221152f1277d (089624e5-8c38-462d-b605-f332f3805c9d) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.855 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.855 INFO Opened existing region file "/tmp/downstairs-zarv4usP/region.json"
Sep 22 23:14:27.855 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.855 INFO Database read version 1
Sep 22 23:14:27.855 WARN [0] new RM replaced this: None
Sep 22 23:14:27.855 INFO Using address: 127.0.0.1:48071, task: main
Sep 22 23:14:27.855 INFO Database write version 1
Sep 22 23:14:27.855 INFO [0] Starts reconcile loop
Sep 22 23:14:27.855 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.855 DEBG IO Read  1003 has deps [JobId(1002)]
Sep 22 23:14:27.855 INFO Created new region file "/tmp/downstairs-WRx1oCsz/region.json"
Sep 22 23:14:27.855 INFO [1] 63bdb731-3b56-40e2-821a-221152f1277d (089624e5-8c38-462d-b605-f332f3805c9d) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.855 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.855 WARN [1] new RM replaced this: None
Sep 22 23:14:27.855 INFO [1] Starts reconcile loop
Sep 22 23:14:27.855 INFO [2] 63bdb731-3b56-40e2-821a-221152f1277d (089624e5-8c38-462d-b605-f332f3805c9d) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.856 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.856 WARN [2] new RM replaced this: None
Sep 22 23:14:27.856 INFO [2] Starts reconcile loop
Sep 22 23:14:27.856 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.856 INFO [0] 127.0.0.1:43804 task reports connection:true
Sep 22 23:14:27.856 INFO 63bdb731-3b56-40e2-821a-221152f1277d WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:14:27.856 INFO [0]R  flush_numbers: [0, 0]
Sep 22 23:14:27.856 INFO [0]R  generation: [0, 0]
Sep 22 23:14:27.856 INFO [0]R  dirty: [false, false]
Sep 22 23:14:27.856 INFO [1]R  flush_numbers: [0, 0]
Sep 22 23:14:27.856 INFO [1]R  generation: [0, 0]
Sep 22 23:14:27.856 INFO [1]R  dirty: [false, false]
Sep 22 23:14:27.856 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:38532, task: repair
Sep 22 23:14:27.856 INFO [2]R  flush_numbers: [0, 0]
Sep 22 23:14:27.856 INFO [2]R  generation: [0, 0]
Sep 22 23:14:27.856 INFO [2]R  dirty: [false, false]
Sep 22 23:14:27.856 INFO Max found gen is 1
Sep 22 23:14:27.856 INFO Generation requested: 1 >= found:1
Sep 22 23:14:27.856 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:38532, task: repair
Sep 22 23:14:27.856 INFO Next flush: 1
Sep 22 23:14:27.856 INFO All extents match
Sep 22 23:14:27.856 INFO No downstairs repair required
Sep 22 23:14:27.856 INFO No initial repair work was required
Sep 22 23:14:27.856 INFO listening, local_addr: 127.0.0.1:38532, task: repair
Sep 22 23:14:27.856 INFO Set Downstairs and Upstairs active
Sep 22 23:14:27.856 INFO 63bdb731-3b56-40e2-821a-221152f1277d is now active with session: 089624e5-8c38-462d-b605-f332f3805c9d
Sep 22 23:14:27.856 INFO 63bdb731-3b56-40e2-821a-221152f1277d Set Active after no repair
Sep 22 23:14:27.856 INFO Notify all downstairs, region set compare is done.
Sep 22 23:14:27.856 INFO Set check for repair
Sep 22 23:14:27.856 INFO [1] 127.0.0.1:60915 task reports connection:true
Sep 22 23:14:27.856 INFO 63bdb731-3b56-40e2-821a-221152f1277d Active Active Active
Sep 22 23:14:27.856 INFO Set check for repair
Sep 22 23:14:27.856 INFO [2] 127.0.0.1:42061 task reports connection:true
Sep 22 23:14:27.856 INFO 63bdb731-3b56-40e2-821a-221152f1277d Active Active Active
Sep 22 23:14:27.856 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:38532, task: repair
Sep 22 23:14:27.856 INFO Set check for repair
Sep 22 23:14:27.856 INFO Using repair address: 127.0.0.1:38532, task: main
Sep 22 23:14:27.856 INFO [0] received reconcile message
Sep 22 23:14:27.856 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.856 INFO [0] All repairs completed, exit
Sep 22 23:14:27.856 INFO [0] Starts cmd_loop
Sep 22 23:14:27.856 INFO [1] received reconcile message
Sep 22 23:14:27.856 INFO [1] All repairs completed, exit
Sep 22 23:14:27.856 INFO [1] Starts cmd_loop
Sep 22 23:14:27.856 INFO UUID: ff3ebd3a-c67d-435f-8151-d080a804dbf5
Sep 22 23:14:27.856 INFO [2] received reconcile message
Sep 22 23:14:27.856 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.856 DEBG Read      :1003 deps:[JobId(1002)] res:true
Sep 22 23:14:27.856 INFO [2] All repairs completed, exit
Sep 22 23:14:27.856 INFO [2] Starts cmd_loop
Sep 22 23:14:27.856 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.856 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
The guest has finished waiting for activation
Sep 22 23:14:27.856 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.856 INFO Using address: 127.0.0.1:43716, task: main
Sep 22 23:14:27.856 INFO Created new region file "/tmp/downstairs-2VFKgxCo/region.json"
Sep 22 23:14:27.856 WARN 63bdb731-3b56-40e2-821a-221152f1277d request to replace downstairs 127.0.0.1:43804 with 127.0.0.1:60915
Sep 22 23:14:27.856 INFO 63bdb731-3b56-40e2-821a-221152f1277d found old target: 127.0.0.1:43804 at 0
Sep 22 23:14:27.856 INFO 63bdb731-3b56-40e2-821a-221152f1277d found new target: 127.0.0.1:60915 at 1
Sep 22 23:14:27.856 DEBG Read      :1003 deps:[JobId(1002)] res:true
Sep 22 23:14:27.857 WARN 63bdb731-3b56-40e2-821a-221152f1277d request to replace downstairs 127.0.0.1:60915 with 127.0.0.1:43804
Sep 22 23:14:27.857 INFO 63bdb731-3b56-40e2-821a-221152f1277d found new target: 127.0.0.1:43804 at 0
Sep 22 23:14:27.857 INFO 63bdb731-3b56-40e2-821a-221152f1277d found old target: 127.0.0.1:60915 at 1
Sep 22 23:14:27.857 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.857 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:51875, task: repair
Sep 22 23:14:27.857 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:51875, task: repair
Sep 22 23:14:27.857 INFO listening, local_addr: 127.0.0.1:51875, task: repair
Sep 22 23:14:27.857 DEBG Read      :1003 deps:[JobId(1002)] res:true
Sep 22 23:14:27.857 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:51875, task: repair
Sep 22 23:14:27.857 INFO Using repair address: 127.0.0.1:51875, task: main
Sep 22 23:14:27.857 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.858 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.858 INFO Created new region file "/tmp/downstairs-Af0iu2U2/region.json"
Sep 22 23:14:27.858 DEBG [0] Read AckReady 1003, : downstairs
Sep 22 23:14:27.859 DEBG [1] Read already AckReady 1003, : downstairs
test test::integration_test_volume_replace_active ... ok
Sep 22 23:14:27.859 DEBG [2] Read already AckReady 1003, : downstairs
Sep 22 23:14:27.859 DEBG up_ds_listen was notified
Sep 22 23:14:27.859 DEBG up_ds_listen process 1003
Sep 22 23:14:27.860 DEBG [A] ack job 1003:4, : downstairs
Sep 22 23:14:27.860 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.860 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.860 INFO Created new region file "/tmp/downstairs-jK06RaaJ/region.json"
Sep 22 23:14:27.860 DEBG IO Read  1001 has deps []
Sep 22 23:14:27.861 DEBG Read      :1001 deps:[] res:true
Sep 22 23:14:27.861 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.861 INFO Opened existing region file "/tmp/downstairs-2VFKgxCo/region.json"
Sep 22 23:14:27.861 INFO Database read version 1
Sep 22 23:14:27.861 INFO Database write version 1
Sep 22 23:14:27.861 DEBG Read      :1001 deps:[] res:true
Sep 22 23:14:27.861 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.861 INFO Opened existing region file "/tmp/downstairs-Af0iu2U2/region.json"
Sep 22 23:14:27.861 INFO Database read version 1
Sep 22 23:14:27.861 INFO Database write version 1
Sep 22 23:14:27.862 DEBG Read      :1001 deps:[] res:true
Sep 22 23:14:27.864 DEBG [0] Read AckReady 1001, : downstairs
Sep 22 23:14:27.865 INFO UUID: be9bde0c-bb2e-4b52-8e82-23af310505da
Sep 22 23:14:27.865 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.865 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.865 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.865 INFO Using address: 127.0.0.1:46338, task: main
Sep 22 23:14:27.865 INFO UUID: 36e095b9-e22f-4b83-be94-6325fd4ef84f
Sep 22 23:14:27.865 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.865 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.865 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.865 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.865 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:54607, task: repair
Sep 22 23:14:27.865 INFO Using address: 127.0.0.1:64857, task: main
Sep 22 23:14:27.865 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:54607, task: repair
Sep 22 23:14:27.865 INFO listening, local_addr: 127.0.0.1:54607, task: repair
Sep 22 23:14:27.865 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:54607, task: repair
Sep 22 23:14:27.866 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.866 INFO Using repair address: 127.0.0.1:54607, task: main
Sep 22 23:14:27.866 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.866 INFO Opened existing region file "/tmp/downstairs-jK06RaaJ/region.json"
Sep 22 23:14:27.866 INFO Database read version 1
Sep 22 23:14:27.866 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.866 INFO Database write version 1
Sep 22 23:14:27.866 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:57115, task: repair
Sep 22 23:14:27.866 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:57115, task: repair
Sep 22 23:14:27.866 INFO listening, local_addr: 127.0.0.1:57115, task: repair
Sep 22 23:14:27.866 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:57115, task: repair
Sep 22 23:14:27.866 INFO Using repair address: 127.0.0.1:57115, task: main
Sep 22 23:14:27.866 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.866 DEBG [1] Read already AckReady 1001, : downstairs
Sep 22 23:14:27.866 INFO Upstairs starts
Sep 22 23:14:27.866 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:14:27.866 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:14:27.866 INFO Crucible stats registered with UUID: e94af85e-3796-4fab-91a1-f12add9c3020
Sep 22 23:14:27.866 INFO Crucible e94af85e-3796-4fab-91a1-f12add9c3020 has session id: 61e725e9-9021-4fae-9e9d-a7be284a2c57
Sep 22 23:14:27.866 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.866 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.867 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.867 INFO Upstairs starts
Sep 22 23:14:27.867 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:14:27.867 INFO [0] connecting to 127.0.0.1:52905, looper: 0
Sep 22 23:14:27.867 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:14:27.867 INFO Crucible stats registered with UUID: 63fa212f-4948-4c7a-a14a-d7319688df16
Sep 22 23:14:27.867 INFO Crucible 63fa212f-4948-4c7a-a14a-d7319688df16 has session id: 9994941c-8051-4c2c-870a-bf5e9fb00b99
Sep 22 23:14:27.867 INFO [1] connecting to 127.0.0.1:48071, looper: 1
Sep 22 23:14:27.867 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.867 INFO [2] connecting to 127.0.0.1:46338, looper: 2
Sep 22 23:14:27.867 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.867 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.867 INFO up_listen starts, task: up_listen
Sep 22 23:14:27.867 INFO Wait for all three downstairs to come online
Sep 22 23:14:27.867 INFO Flush timeout: 0.5
Sep 22 23:14:27.867 INFO [0] connecting to 127.0.0.1:47986, looper: 0
Sep 22 23:14:27.867 INFO [1] connecting to 127.0.0.1:43716, looper: 1
Sep 22 23:14:27.867 INFO [2] connecting to 127.0.0.1:64857, looper: 2
Sep 22 23:14:27.867 INFO accepted connection from 127.0.0.1:56581, task: main
Sep 22 23:14:27.867 INFO accepted connection from 127.0.0.1:64564, task: main
Sep 22 23:14:27.867 INFO up_listen starts, task: up_listen
Sep 22 23:14:27.868 INFO Wait for all three downstairs to come online
Sep 22 23:14:27.868 INFO Flush timeout: 0.5
Sep 22 23:14:27.868 INFO accepted connection from 127.0.0.1:45396, task: main
Sep 22 23:14:27.868 INFO [0] e94af85e-3796-4fab-91a1-f12add9c3020 looper connected, looper: 0
Sep 22 23:14:27.868 INFO [0] Proc runs for 127.0.0.1:52905 in state New
Sep 22 23:14:27.868 INFO [1] e94af85e-3796-4fab-91a1-f12add9c3020 looper connected, looper: 1
Sep 22 23:14:27.868 INFO accepted connection from 127.0.0.1:59696, task: main
Sep 22 23:14:27.868 INFO [1] Proc runs for 127.0.0.1:48071 in state New
Sep 22 23:14:27.868 INFO accepted connection from 127.0.0.1:36772, task: main
Sep 22 23:14:27.868 INFO [2] e94af85e-3796-4fab-91a1-f12add9c3020 looper connected, looper: 2
Sep 22 23:14:27.868 INFO accepted connection from 127.0.0.1:50348, task: main
Sep 22 23:14:27.868 INFO [2] Proc runs for 127.0.0.1:46338 in state New
Sep 22 23:14:27.868 INFO UUID: b78cfd9a-1100-4037-aec9-2c2cabfc0ce6
Sep 22 23:14:27.868 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.868 INFO [0] 63fa212f-4948-4c7a-a14a-d7319688df16 looper connected, looper: 0
Sep 22 23:14:27.868 INFO [0] Proc runs for 127.0.0.1:47986 in state New
Sep 22 23:14:27.868 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.868 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.868 INFO Using address: 127.0.0.1:53467, task: main
Sep 22 23:14:27.868 INFO [1] 63fa212f-4948-4c7a-a14a-d7319688df16 looper connected, looper: 1
Sep 22 23:14:27.868 INFO [1] Proc runs for 127.0.0.1:43716 in state New
Sep 22 23:14:27.868 DEBG [2] Read already AckReady 1001, : downstairs
Sep 22 23:14:27.868 DEBG up_ds_listen was notified
Sep 22 23:14:27.868 INFO [2] 63fa212f-4948-4c7a-a14a-d7319688df16 looper connected, looper: 2
Sep 22 23:14:27.868 INFO Connection request from e94af85e-3796-4fab-91a1-f12add9c3020 with version 4, task: proc
Sep 22 23:14:27.868 DEBG up_ds_listen process 1001
Sep 22 23:14:27.868 DEBG [A] ack job 1001:2, : downstairs
Sep 22 23:14:27.868 INFO upstairs UpstairsConnection { upstairs_id: e94af85e-3796-4fab-91a1-f12add9c3020, session_id: a86ae319-11ad-4b2d-a80b-330efc92ebef, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.868 INFO [2] Proc runs for 127.0.0.1:64857 in state New
Sep 22 23:14:27.868 INFO Connection request from e94af85e-3796-4fab-91a1-f12add9c3020 with version 4, task: proc
Sep 22 23:14:27.868 INFO upstairs UpstairsConnection { upstairs_id: e94af85e-3796-4fab-91a1-f12add9c3020, session_id: a86ae319-11ad-4b2d-a80b-330efc92ebef, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.869 INFO Connection request from e94af85e-3796-4fab-91a1-f12add9c3020 with version 4, task: proc
Sep 22 23:14:27.869 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.869 INFO upstairs UpstairsConnection { upstairs_id: e94af85e-3796-4fab-91a1-f12add9c3020, session_id: a86ae319-11ad-4b2d-a80b-330efc92ebef, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.869 INFO Connection request from 63fa212f-4948-4c7a-a14a-d7319688df16 with version 4, task: proc
Sep 22 23:14:27.869 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.869 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:35868, task: repair
Sep 22 23:14:27.869 INFO upstairs UpstairsConnection { upstairs_id: 63fa212f-4948-4c7a-a14a-d7319688df16, session_id: 67fdb17a-86e0-451b-a532-c1ff4695781c, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.869 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:35868, task: repair
The guest has requested activation
Sep 22 23:14:27.869 INFO listening, local_addr: 127.0.0.1:35868, task: repair
Sep 22 23:14:27.869 INFO Connection request from 63fa212f-4948-4c7a-a14a-d7319688df16 with version 4, task: proc
Sep 22 23:14:27.869 INFO upstairs UpstairsConnection { upstairs_id: 63fa212f-4948-4c7a-a14a-d7319688df16, session_id: 67fdb17a-86e0-451b-a532-c1ff4695781c, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.869 INFO [0] e94af85e-3796-4fab-91a1-f12add9c3020 (a86ae319-11ad-4b2d-a80b-330efc92ebef) New New New ds_transition to WaitActive
Sep 22 23:14:27.869 INFO [0] Transition from New to WaitActive
Sep 22 23:14:27.869 INFO Connection request from 63fa212f-4948-4c7a-a14a-d7319688df16 with version 4, task: proc
Sep 22 23:14:27.869 INFO upstairs UpstairsConnection { upstairs_id: 63fa212f-4948-4c7a-a14a-d7319688df16, session_id: 67fdb17a-86e0-451b-a532-c1ff4695781c, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.869 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:35868, task: repair
Sep 22 23:14:27.869 INFO [1] e94af85e-3796-4fab-91a1-f12add9c3020 (a86ae319-11ad-4b2d-a80b-330efc92ebef) WaitActive New New ds_transition to WaitActive
Sep 22 23:14:27.869 INFO [1] Transition from New to WaitActive
Sep 22 23:14:27.869 INFO Using repair address: 127.0.0.1:35868, task: main
Sep 22 23:14:27.869 INFO No SSL acceptor configured, task: main
The guest has requested activation
Sep 22 23:14:27.869 INFO [2] e94af85e-3796-4fab-91a1-f12add9c3020 (a86ae319-11ad-4b2d-a80b-330efc92ebef) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:14:27.869 INFO [2] Transition from New to WaitActive
Sep 22 23:14:27.869 INFO [0] 63fa212f-4948-4c7a-a14a-d7319688df16 (67fdb17a-86e0-451b-a532-c1ff4695781c) New New New ds_transition to WaitActive
Sep 22 23:14:27.869 INFO e94af85e-3796-4fab-91a1-f12add9c3020 active request set
Sep 22 23:14:27.869 DEBG IO Flush 1004 has deps [JobId(1003), JobId(1002)]
Sep 22 23:14:27.869 INFO [0] Transition from New to WaitActive
Sep 22 23:14:27.869 INFO [0] received activate with gen 1
Sep 22 23:14:27.869 INFO [1] 63fa212f-4948-4c7a-a14a-d7319688df16 (67fdb17a-86e0-451b-a532-c1ff4695781c) WaitActive New New ds_transition to WaitActive
Sep 22 23:14:27.869 INFO [0] client got ds_active_rx, promote! session a86ae319-11ad-4b2d-a80b-330efc92ebef
Sep 22 23:14:27.869 INFO [1] Transition from New to WaitActive
Sep 22 23:14:27.869 INFO [1] received activate with gen 1
Sep 22 23:14:27.869 INFO [1] client got ds_active_rx, promote! session a86ae319-11ad-4b2d-a80b-330efc92ebef
Sep 22 23:14:27.869 INFO [2] 63fa212f-4948-4c7a-a14a-d7319688df16 (67fdb17a-86e0-451b-a532-c1ff4695781c) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:14:27.869 INFO [2] Transition from New to WaitActive
Sep 22 23:14:27.869 INFO [2] received activate with gen 1
Sep 22 23:14:27.869 INFO 63fa212f-4948-4c7a-a14a-d7319688df16 active request set
Sep 22 23:14:27.869 INFO [2] client got ds_active_rx, promote! session a86ae319-11ad-4b2d-a80b-330efc92ebef
Sep 22 23:14:27.870 INFO [0] received activate with gen 1
Sep 22 23:14:27.870 INFO [0] client got ds_active_rx, promote! session 67fdb17a-86e0-451b-a532-c1ff4695781c
Sep 22 23:14:27.870 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.870 INFO UpstairsConnection { upstairs_id: e94af85e-3796-4fab-91a1-f12add9c3020, session_id: a86ae319-11ad-4b2d-a80b-330efc92ebef, gen: 1 } is now active (read-write)
Sep 22 23:14:27.870 INFO [1] received activate with gen 1
Sep 22 23:14:27.870 INFO [1] client got ds_active_rx, promote! session 67fdb17a-86e0-451b-a532-c1ff4695781c
Sep 22 23:14:27.870 INFO Created new region file "/tmp/downstairs-5ryGVcVK/region.json"
Sep 22 23:14:27.870 INFO [2] received activate with gen 1
Sep 22 23:14:27.870 INFO [2] client got ds_active_rx, promote! session 67fdb17a-86e0-451b-a532-c1ff4695781c
Sep 22 23:14:27.870 INFO UpstairsConnection { upstairs_id: e94af85e-3796-4fab-91a1-f12add9c3020, session_id: a86ae319-11ad-4b2d-a80b-330efc92ebef, gen: 1 } is now active (read-write)
Sep 22 23:14:27.870 INFO UpstairsConnection { upstairs_id: 63fa212f-4948-4c7a-a14a-d7319688df16, session_id: 67fdb17a-86e0-451b-a532-c1ff4695781c, gen: 1 } is now active (read-write)
Sep 22 23:14:27.870 INFO UpstairsConnection { upstairs_id: e94af85e-3796-4fab-91a1-f12add9c3020, session_id: a86ae319-11ad-4b2d-a80b-330efc92ebef, gen: 1 } is now active (read-write)
Sep 22 23:14:27.870 INFO UpstairsConnection { upstairs_id: 63fa212f-4948-4c7a-a14a-d7319688df16, session_id: 67fdb17a-86e0-451b-a532-c1ff4695781c, gen: 1 } is now active (read-write)
Sep 22 23:14:27.870 INFO UpstairsConnection { upstairs_id: 63fa212f-4948-4c7a-a14a-d7319688df16, session_id: 67fdb17a-86e0-451b-a532-c1ff4695781c, gen: 1 } is now active (read-write)
Sep 22 23:14:27.870 DEBG Flush     :1004 extent_limit None deps:[JobId(1003), JobId(1002)] res:true f:2 g:3
Sep 22 23:14:27.870 DEBG Flush     :1004 extent_limit None deps:[JobId(1003), JobId(1002)] res:true f:2 g:3
Sep 22 23:14:27.870 DEBG Flush     :1004 extent_limit None deps:[JobId(1003), JobId(1002)] res:true f:2 g:3
Sep 22 23:14:27.870 INFO [0] downstairs client at 127.0.0.1:52905 has UUID 30596aa6-8959-4f8c-98b8-ac91fdc30fc1
Sep 22 23:14:27.870 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 30596aa6-8959-4f8c-98b8-ac91fdc30fc1, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.871 INFO e94af85e-3796-4fab-91a1-f12add9c3020 WaitActive WaitActive WaitActive
Sep 22 23:14:27.871 DEBG up_ds_listen was notified
Sep 22 23:14:27.871 INFO [1] downstairs client at 127.0.0.1:48071 has UUID 05b93ff5-c936-4fa6-adb6-db93f2642835
Sep 22 23:14:27.871 INFO [0] downstairs client at 127.0.0.1:47986 has UUID 8e0311d9-6cee-4187-8ec5-c1067f73eb45
Sep 22 23:14:27.871 DEBG up_ds_listen process 1004
Sep 22 23:14:27.871 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 05b93ff5-c936-4fa6-adb6-db93f2642835, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.871 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 8e0311d9-6cee-4187-8ec5-c1067f73eb45, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.871 DEBG [A] ack job 1004:5, : downstairs
Sep 22 23:14:27.871 INFO e94af85e-3796-4fab-91a1-f12add9c3020 WaitActive WaitActive WaitActive
Sep 22 23:14:27.871 INFO 63fa212f-4948-4c7a-a14a-d7319688df16 WaitActive WaitActive WaitActive
Sep 22 23:14:27.871 DEBG [rc] retire 1004 clears [JobId(1002), JobId(1003), JobId(1004)], : downstairs
Sep 22 23:14:27.871 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.871 INFO [1] downstairs client at 127.0.0.1:43716 has UUID ff3ebd3a-c67d-435f-8151-d080a804dbf5
Sep 22 23:14:27.871 INFO [2] downstairs client at 127.0.0.1:46338 has UUID be9bde0c-bb2e-4b52-8e82-23af310505da
Sep 22 23:14:27.871 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: ff3ebd3a-c67d-435f-8151-d080a804dbf5, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.871 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: be9bde0c-bb2e-4b52-8e82-23af310505da, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.871 INFO 63fa212f-4948-4c7a-a14a-d7319688df16 WaitActive WaitActive WaitActive
Sep 22 23:14:27.871 INFO e94af85e-3796-4fab-91a1-f12add9c3020 WaitActive WaitActive WaitActive
Sep 22 23:14:27.871 INFO [2] downstairs client at 127.0.0.1:64857 has UUID 36e095b9-e22f-4b83-be94-6325fd4ef84f
Sep 22 23:14:27.871 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 36e095b9-e22f-4b83-be94-6325fd4ef84f, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.871 INFO 63fa212f-4948-4c7a-a14a-d7319688df16 WaitActive WaitActive WaitActive
Sep 22 23:14:27.871 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.871 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.871 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.872 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.872 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.872 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.872 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.872 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.872 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.872 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.872 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.873 INFO [0] e94af85e-3796-4fab-91a1-f12add9c3020 (a86ae319-11ad-4b2d-a80b-330efc92ebef) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.873 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.873 WARN [0] new RM replaced this: None
Sep 22 23:14:27.873 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.873 INFO [0] Starts reconcile loop
Sep 22 23:14:27.873 INFO [1] e94af85e-3796-4fab-91a1-f12add9c3020 (a86ae319-11ad-4b2d-a80b-330efc92ebef) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.873 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.873 WARN [1] new RM replaced this: None
Sep 22 23:14:27.873 INFO [0] 63fa212f-4948-4c7a-a14a-d7319688df16 (67fdb17a-86e0-451b-a532-c1ff4695781c) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.873 INFO [1] Starts reconcile loop
Sep 22 23:14:27.873 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.873 WARN [0] new RM replaced this: None
Sep 22 23:14:27.873 INFO [0] Starts reconcile loop
Sep 22 23:14:27.873 INFO [2] e94af85e-3796-4fab-91a1-f12add9c3020 (a86ae319-11ad-4b2d-a80b-330efc92ebef) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.873 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.873 WARN [2] new RM replaced this: None
Sep 22 23:14:27.873 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.873 INFO [1] 63fa212f-4948-4c7a-a14a-d7319688df16 (67fdb17a-86e0-451b-a532-c1ff4695781c) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.873 INFO Opened existing region file "/tmp/downstairs-5ryGVcVK/region.json"
Sep 22 23:14:27.873 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.873 INFO [2] Starts reconcile loop
Sep 22 23:14:27.873 INFO Database read version 1
Sep 22 23:14:27.873 WARN [1] new RM replaced this: None
Sep 22 23:14:27.873 INFO Database write version 1
Sep 22 23:14:27.873 INFO [1] Starts reconcile loop
Sep 22 23:14:27.873 INFO [0] 127.0.0.1:52905 task reports connection:true
Sep 22 23:14:27.873 INFO e94af85e-3796-4fab-91a1-f12add9c3020 WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:14:27.873 INFO [2] 63fa212f-4948-4c7a-a14a-d7319688df16 (67fdb17a-86e0-451b-a532-c1ff4695781c) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.873 INFO [0]R  flush_numbers: [0, 0]
Sep 22 23:14:27.873 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.873 INFO [0]R  generation: [0, 0]
Sep 22 23:14:27.873 WARN [2] new RM replaced this: None
Sep 22 23:14:27.873 INFO [0]R  dirty: [false, false]
Sep 22 23:14:27.873 INFO [1]R  flush_numbers: [0, 0]
Sep 22 23:14:27.873 INFO [2] Starts reconcile loop
Sep 22 23:14:27.873 INFO [1]R  generation: [0, 0]
Sep 22 23:14:27.873 INFO [1]R  dirty: [false, false]
Sep 22 23:14:27.873 INFO [2]R  flush_numbers: [0, 0]
Sep 22 23:14:27.873 INFO [2]R  generation: [0, 0]
Sep 22 23:14:27.873 INFO [0] 127.0.0.1:47986 task reports connection:true
Sep 22 23:14:27.873 INFO [2]R  dirty: [false, false]
Sep 22 23:14:27.873 INFO Max found gen is 1
Sep 22 23:14:27.873 INFO 63fa212f-4948-4c7a-a14a-d7319688df16 WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:14:27.873 INFO Generation requested: 1 >= found:1
Sep 22 23:14:27.873 INFO Next flush: 1
Sep 22 23:14:27.873 INFO [0]R  flush_numbers: [0, 0]
Sep 22 23:14:27.873 INFO [0]R  generation: [0, 0]
Sep 22 23:14:27.873 INFO All extents match
Sep 22 23:14:27.873 INFO [0]R  dirty: [false, false]
Sep 22 23:14:27.873 INFO No downstairs repair required
Sep 22 23:14:27.873 INFO [1]R  flush_numbers: [0, 0]
Sep 22 23:14:27.873 INFO No initial repair work was required
Sep 22 23:14:27.873 INFO [1]R  generation: [0, 0]
Sep 22 23:14:27.873 INFO Set Downstairs and Upstairs active
Sep 22 23:14:27.873 INFO [1]R  dirty: [false, false]
Sep 22 23:14:27.873 INFO [2]R  flush_numbers: [0, 0]
Sep 22 23:14:27.873 INFO [2]R  generation: [0, 0]
Sep 22 23:14:27.873 INFO e94af85e-3796-4fab-91a1-f12add9c3020 is now active with session: a86ae319-11ad-4b2d-a80b-330efc92ebef
Sep 22 23:14:27.873 INFO [2]R  dirty: [false, false]
Sep 22 23:14:27.873 INFO Max found gen is 1
Sep 22 23:14:27.873 INFO e94af85e-3796-4fab-91a1-f12add9c3020 Set Active after no repair
Sep 22 23:14:27.873 INFO Generation requested: 1 >= found:1
Sep 22 23:14:27.873 INFO Notify all downstairs, region set compare is done.
Sep 22 23:14:27.873 INFO Next flush: 1
Sep 22 23:14:27.874 INFO Set check for repair
Sep 22 23:14:27.874 INFO All extents match
Sep 22 23:14:27.874 INFO No downstairs repair required
Sep 22 23:14:27.874 INFO No initial repair work was required
Sep 22 23:14:27.874 INFO [1] 127.0.0.1:48071 task reports connection:true
Sep 22 23:14:27.874 INFO Set Downstairs and Upstairs active
Sep 22 23:14:27.874 INFO e94af85e-3796-4fab-91a1-f12add9c3020 Active Active Active
Sep 22 23:14:27.874 INFO 63fa212f-4948-4c7a-a14a-d7319688df16 is now active with session: 67fdb17a-86e0-451b-a532-c1ff4695781c
Sep 22 23:14:27.874 INFO Set check for repair
Sep 22 23:14:27.874 INFO 63fa212f-4948-4c7a-a14a-d7319688df16 Set Active after no repair
Sep 22 23:14:27.874 INFO Notify all downstairs, region set compare is done.
Sep 22 23:14:27.874 INFO [2] 127.0.0.1:46338 task reports connection:true
Sep 22 23:14:27.874 INFO Set check for repair
Sep 22 23:14:27.874 INFO e94af85e-3796-4fab-91a1-f12add9c3020 Active Active Active
Sep 22 23:14:27.874 INFO Set check for repair
Sep 22 23:14:27.874 INFO [1] 127.0.0.1:43716 task reports connection:true
Sep 22 23:14:27.874 INFO 63fa212f-4948-4c7a-a14a-d7319688df16 Active Active Active
Sep 22 23:14:27.874 INFO Set check for repair
Sep 22 23:14:27.874 INFO [0] received reconcile message
Sep 22 23:14:27.874 INFO [0] All repairs completed, exit
Sep 22 23:14:27.874 INFO [2] 127.0.0.1:64857 task reports connection:true
Sep 22 23:14:27.874 INFO 63fa212f-4948-4c7a-a14a-d7319688df16 Active Active Active
Sep 22 23:14:27.874 INFO [0] Starts cmd_loop
Sep 22 23:14:27.874 INFO Set check for repair
Sep 22 23:14:27.874 INFO [1] received reconcile message
Sep 22 23:14:27.874 INFO [0] received reconcile message
Sep 22 23:14:27.874 INFO [1] All repairs completed, exit
Sep 22 23:14:27.874 INFO [0] All repairs completed, exit
Sep 22 23:14:27.874 INFO [1] Starts cmd_loop
Sep 22 23:14:27.874 INFO [0] Starts cmd_loop
Sep 22 23:14:27.874 INFO [2] received reconcile message
Sep 22 23:14:27.874 INFO [1] received reconcile message
Sep 22 23:14:27.874 INFO [2] All repairs completed, exit
Sep 22 23:14:27.874 INFO [1] All repairs completed, exit
Sep 22 23:14:27.874 INFO [2] Starts cmd_loop
Sep 22 23:14:27.874 INFO [1] Starts cmd_loop
The guest has finished waiting for activation
Sep 22 23:14:27.874 INFO [2] received reconcile message
Sep 22 23:14:27.874 INFO [2] All repairs completed, exit
Sep 22 23:14:27.874 INFO [2] Starts cmd_loop
The guest has finished waiting for activation
Sep 22 23:14:27.874 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.875 INFO Created new region file "/tmp/downstairs-uSBD3d00/region.json"
Sep 22 23:14:27.875 INFO UUID: 555bafbf-d9bb-408a-aa71-328bc8a8cc92
Sep 22 23:14:27.875 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.875 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.875 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.875 INFO Using address: 127.0.0.1:38389, task: main
Sep 22 23:14:27.875 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.876 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:46434, task: repair
Sep 22 23:14:27.876 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:46434, task: repair
Sep 22 23:14:27.876 INFO listening, local_addr: 127.0.0.1:46434, task: repair
Sep 22 23:14:27.876 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:46434, task: repair
Sep 22 23:14:27.876 INFO Using repair address: 127.0.0.1:46434, task: main
Sep 22 23:14:27.876 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.876 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.876 INFO Created new region file "/tmp/downstairs-51vT3sLU/region.json"
Sep 22 23:14:27.877 DEBG IO Write 1000 has deps []
Sep 22 23:14:27.877 DEBG up_ds_listen was notified
Sep 22 23:14:27.877 DEBG up_ds_listen process 1000
Sep 22 23:14:27.877 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:14:27.877 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.878 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.879 INFO Created new region file "/tmp/downstairs-08law0EE/region.json"
Sep 22 23:14:27.880 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.880 INFO Opened existing region file "/tmp/downstairs-uSBD3d00/region.json"
Sep 22 23:14:27.880 INFO Database read version 1
Sep 22 23:14:27.880 INFO Database write version 1
test test::integration_test_snapshot_backed_vol ... ok
Sep 22 23:14:27.881 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.881 INFO Opened existing region file "/tmp/downstairs-n8rm3T5A/region.json"
Sep 22 23:14:27.881 INFO Database read version 1
Sep 22 23:14:27.881 INFO Database write version 1
Sep 22 23:14:27.881 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.881 INFO Created new region file "/tmp/downstairs-C2d5BFdC/region.json"
Sep 22 23:14:27.881 INFO UUID: 5dd2e15b-101c-44f7-b514-c1b4ac930978
Sep 22 23:14:27.881 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.881 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.882 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.882 INFO Using address: 127.0.0.1:47413, task: main
Sep 22 23:14:27.882 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.882 INFO Opened existing region file "/tmp/downstairs-08law0EE/region.json"
Sep 22 23:14:27.882 INFO Database read version 1
Sep 22 23:14:27.882 INFO Database write version 1
Sep 22 23:14:27.882 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.882 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:43170, task: repair
Sep 22 23:14:27.882 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:43170, task: repair
Sep 22 23:14:27.882 INFO listening, local_addr: 127.0.0.1:43170, task: repair
Sep 22 23:14:27.882 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:43170, task: repair
Sep 22 23:14:27.882 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.882 INFO Opened existing region file "/tmp/downstairs-51vT3sLU/region.json"
Sep 22 23:14:27.882 INFO Using repair address: 127.0.0.1:43170, task: main
Sep 22 23:14:27.882 INFO Database read version 1
Sep 22 23:14:27.882 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.882 INFO Database write version 1
Sep 22 23:14:27.883 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.883 INFO Created new region file "/tmp/downstairs-3q9fZVF8/region.json"
Sep 22 23:14:27.884 INFO UUID: b7c0d6d3-333f-4423-864b-ebd0befef183
Sep 22 23:14:27.884 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.884 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.884 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.884 INFO Using address: 127.0.0.1:56689, task: main
Sep 22 23:14:27.884 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.884 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:47224, task: repair
Sep 22 23:14:27.884 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:47224, task: repair
Sep 22 23:14:27.884 INFO listening, local_addr: 127.0.0.1:47224, task: repair
Sep 22 23:14:27.884 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:47224, task: repair
Sep 22 23:14:27.884 INFO Using repair address: 127.0.0.1:47224, task: main
Sep 22 23:14:27.884 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.885 INFO UUID: 16b61f39-4c68-4f56-ae07-8508cf0adb07
Sep 22 23:14:27.885 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.885 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.885 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.885 INFO Using address: 127.0.0.1:58380, task: main
Sep 22 23:14:27.885 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.885 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:62376, task: repair
Sep 22 23:14:27.885 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:62376, task: repair
Sep 22 23:14:27.885 INFO listening, local_addr: 127.0.0.1:62376, task: repair
Sep 22 23:14:27.886 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.886 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:62376, task: repair
Sep 22 23:14:27.886 WARN e94af85e-3796-4fab-91a1-f12add9c3020 request to replace downstairs 127.0.0.1:52905 with 127.0.0.1:56689
Sep 22 23:14:27.886 INFO e94af85e-3796-4fab-91a1-f12add9c3020 found old target: 127.0.0.1:52905 at 0
Sep 22 23:14:27.886 INFO e94af85e-3796-4fab-91a1-f12add9c3020 replacing old: 127.0.0.1:52905 at 0
Sep 22 23:14:27.886 INFO Using repair address: 127.0.0.1:62376, task: main
Sep 22 23:14:27.886 INFO [0] client skip 1 in process jobs because fault, : downstairs
Sep 22 23:14:27.886 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.886 INFO [0] changed 1 jobs to fault skipped, : downstairs
Sep 22 23:14:27.886 INFO [0] e94af85e-3796-4fab-91a1-f12add9c3020 (a86ae319-11ad-4b2d-a80b-330efc92ebef) Active Active Active ds_transition to Replacing
Sep 22 23:14:27.886 INFO [0] Transition from Active to Replacing
Sep 22 23:14:27.886 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.886 INFO Opened existing region file "/tmp/downstairs-C2d5BFdC/region.json"
Sep 22 23:14:27.886 INFO Database read version 1
Sep 22 23:14:27.886 INFO Database write version 1
Sep 22 23:14:27.886 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.886 DEBG Write     :1000 deps:[] res:true
Sep 22 23:14:27.886 INFO Created new region file "/tmp/downstairs-p0sBB51P/region.json"
Sep 22 23:14:27.887 DEBG Write     :1000 deps:[] res:true
Sep 22 23:14:27.888 DEBG Write     :1000 deps:[] res:true
Sep 22 23:14:27.888 WARN [0] e94af85e-3796-4fab-91a1-f12add9c3020 WARNING finish job 1000 when downstairs state:Replacing
Sep 22 23:14:27.888 WARN [0] Dropping already skipped job 1000, : downstairs
Sep 22 23:14:27.888 WARN [0] will exit pm_task, this downstairs Replacing
Sep 22 23:14:27.888 DEBG up_ds_listen was notified
Sep 22 23:14:27.888 DEBG up_ds_listen checked 0 jobs, back to waiting
Sep 22 23:14:27.888 ERRO 127.0.0.1:52905: proc: [0] client work task ended, Ok(Err([0] This downstairs now in Replacing)), so we end too, looper: 0
Sep 22 23:14:27.888 INFO [0] e94af85e-3796-4fab-91a1-f12add9c3020 Gone missing, transition from Replacing to Replaced
Sep 22 23:14:27.888 INFO [0] e94af85e-3796-4fab-91a1-f12add9c3020 connection to 127.0.0.1:52905 closed, looper: 0
Sep 22 23:14:27.888 INFO [0] 127.0.0.1:52905 task reports connection:false
Sep 22 23:14:27.888 INFO e94af85e-3796-4fab-91a1-f12add9c3020 Replaced Active Active
Sep 22 23:14:27.888 INFO [0] 127.0.0.1:52905 task reports offline
Sep 22 23:14:27.888 WARN upstairs UpstairsConnection { upstairs_id: e94af85e-3796-4fab-91a1-f12add9c3020, session_id: a86ae319-11ad-4b2d-a80b-330efc92ebef, gen: 1 } disconnected, 0 jobs left, task: main
Sep 22 23:14:27.888 WARN upstairs UpstairsConnection { upstairs_id: e94af85e-3796-4fab-91a1-f12add9c3020, session_id: a86ae319-11ad-4b2d-a80b-330efc92ebef, gen: 1 } was previously active, clearing, task: main
Sep 22 23:14:27.888 INFO connection (127.0.0.1:64564): all done
Sep 22 23:14:27.889 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.889 INFO UUID: da4b10d8-d31d-4fd9-811a-957d3b103709
Sep 22 23:14:27.889 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.889 INFO Opened existing region file "/tmp/downstairs-3q9fZVF8/region.json"
Sep 22 23:14:27.889 INFO Database read version 1
Sep 22 23:14:27.889 INFO Database write version 1
Sep 22 23:14:27.889 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.889 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.889 INFO Using address: 127.0.0.1:62077, task: main
Sep 22 23:14:27.889 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.889 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:34610, task: repair
Sep 22 23:14:27.889 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:34610, task: repair
Sep 22 23:14:27.890 INFO listening, local_addr: 127.0.0.1:34610, task: repair
Sep 22 23:14:27.890 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:34610, task: repair
Sep 22 23:14:27.890 INFO Using repair address: 127.0.0.1:34610, task: main
Sep 22 23:14:27.890 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.890 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.890 INFO Created new region file "/tmp/downstairs-NcuCFDzp/region.json"
Sep 22 23:14:27.891 INFO UUID: 8db2395a-ec39-4e72-9a2f-a6d3e8a113ec
Sep 22 23:14:27.891 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.891 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.891 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.891 INFO Using address: 127.0.0.1:38707, task: main
Sep 22 23:14:27.892 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.892 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:65488, task: repair
Sep 22 23:14:27.892 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:65488, task: repair
Sep 22 23:14:27.892 INFO listening, local_addr: 127.0.0.1:65488, task: repair
Sep 22 23:14:27.892 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.892 INFO Opened existing region file "/tmp/downstairs-p0sBB51P/region.json"
Sep 22 23:14:27.892 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:65488, task: repair
Sep 22 23:14:27.892 INFO Database read version 1
Sep 22 23:14:27.892 INFO Database write version 1
Sep 22 23:14:27.892 INFO Using repair address: 127.0.0.1:65488, task: main
Sep 22 23:14:27.892 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.893 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.893 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.893 WARN 63fa212f-4948-4c7a-a14a-d7319688df16 request to replace downstairs 127.0.0.1:47413 with 127.0.0.1:38707
Sep 22 23:14:27.893 WARN 63fa212f-4948-4c7a-a14a-d7319688df16 downstairs 127.0.0.1:47413 not found
Sep 22 23:14:27.895 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.895 INFO Opened existing region file "/tmp/downstairs-NcuCFDzp/region.json"
Sep 22 23:14:27.895 INFO Database read version 1
Sep 22 23:14:27.895 INFO Database write version 1
Sep 22 23:14:27.895 INFO UUID: 0f3af6ad-c169-4a67-ae79-bc5859c15909
Sep 22 23:14:27.895 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.896 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.896 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.896 INFO Using address: 127.0.0.1:48636, task: main
Sep 22 23:14:27.896 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.896 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:59973, task: repair
Sep 22 23:14:27.896 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:59973, task: repair
Sep 22 23:14:27.896 INFO listening, local_addr: 127.0.0.1:59973, task: repair
Sep 22 23:14:27.896 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:59973, task: repair
Sep 22 23:14:27.896 INFO Using repair address: 127.0.0.1:59973, task: main
Sep 22 23:14:27.896 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.897 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.897 INFO Created new region file "/tmp/downstairs-mBHM8lj3/region.json"
Sep 22 23:14:27.898 INFO UUID: a11fec66-3ce4-4c3c-bad2-f8da3b3c9b3e
Sep 22 23:14:27.898 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.898 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.898 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.898 INFO Using address: 127.0.0.1:55380, task: main
Sep 22 23:14:27.898 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.898 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:45964, task: repair
Sep 22 23:14:27.898 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:45964, task: repair
Sep 22 23:14:27.898 INFO listening, local_addr: 127.0.0.1:45964, task: repair
Sep 22 23:14:27.899 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:45964, task: repair
Sep 22 23:14:27.899 INFO Using repair address: 127.0.0.1:45964, task: main
Sep 22 23:14:27.899 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.899 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.899 INFO Created new region file "/tmp/downstairs-6VydVRSH/region.json"
test test::integration_test_volume_replace_bad_downstairs ... ok
Sep 22 23:14:27.899 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.900 INFO Created new region file "/tmp/downstairs-gsF9WdKL/region.json"
Sep 22 23:14:27.901 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.901 INFO Opened existing region file "/tmp/downstairs-mBHM8lj3/region.json"
Sep 22 23:14:27.901 INFO Database read version 1
Sep 22 23:14:27.901 INFO Database write version 1
Sep 22 23:14:27.903 INFO UUID: aef90a89-454d-4861-b15d-20c99780f2e8
Sep 22 23:14:27.903 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.903 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.903 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.903 INFO Using address: 127.0.0.1:59653, task: main
Sep 22 23:14:27.903 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.903 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.903 INFO Opened existing region file "/tmp/downstairs-6VydVRSH/region.json"
Sep 22 23:14:27.903 INFO Database read version 1
Sep 22 23:14:27.903 INFO Database write version 1
Sep 22 23:14:27.903 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:51333, task: repair
Sep 22 23:14:27.903 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:51333, task: repair
Sep 22 23:14:27.903 INFO listening, local_addr: 127.0.0.1:51333, task: repair
Sep 22 23:14:27.903 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:51333, task: repair
Sep 22 23:14:27.903 INFO Using repair address: 127.0.0.1:51333, task: main
Sep 22 23:14:27.904 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.904 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.904 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.904 INFO Opened existing region file "/tmp/downstairs-gsF9WdKL/region.json"
Sep 22 23:14:27.904 INFO Database read version 1
Sep 22 23:14:27.904 INFO Database write version 1
Sep 22 23:14:27.904 INFO Created new region file "/tmp/downstairs-nHU5oa1F/region.json"
Sep 22 23:14:27.906 INFO UUID: 7d1add5d-5a13-414b-bc02-b8d357908557
Sep 22 23:14:27.906 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.907 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.907 INFO UUID: 9a08289f-c73f-4f3d-af8e-b9eb4fe46f56
Sep 22 23:14:27.907 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.907 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.907 INFO Using address: 127.0.0.1:41467, task: main
Sep 22 23:14:27.907 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.907 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.907 INFO Using address: 127.0.0.1:63499, task: main
Sep 22 23:14:27.907 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.907 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.907 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:52641, task: repair
Sep 22 23:14:27.907 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:52641, task: repair
Sep 22 23:14:27.907 INFO listening, local_addr: 127.0.0.1:52641, task: repair
Sep 22 23:14:27.907 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:56010, task: repair
Sep 22 23:14:27.907 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:56010, task: repair
Sep 22 23:14:27.907 INFO listening, local_addr: 127.0.0.1:56010, task: repair
Sep 22 23:14:27.907 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:52641, task: repair
Sep 22 23:14:27.907 INFO Using repair address: 127.0.0.1:52641, task: main
Sep 22 23:14:27.907 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.907 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:56010, task: repair
Sep 22 23:14:27.907 INFO Using repair address: 127.0.0.1:56010, task: main
Sep 22 23:14:27.907 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.908 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.908 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.908 INFO Created new region file "/tmp/downstairs-NOXcduXR/region.json"
Sep 22 23:14:27.908 INFO Created new region file "/tmp/downstairs-An1Xd1ec/region.json"
Sep 22 23:14:27.908 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.908 INFO Opened existing region file "/tmp/downstairs-nHU5oa1F/region.json"
Sep 22 23:14:27.908 INFO Database read version 1
Sep 22 23:14:27.908 INFO Database write version 1
Sep 22 23:14:27.911 INFO UUID: 6085db72-2a98-4898-ac66-9af2876a589f
Sep 22 23:14:27.911 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.911 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.911 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.911 INFO Using address: 127.0.0.1:56727, task: main
Sep 22 23:14:27.912 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.912 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:50342, task: repair
Sep 22 23:14:27.912 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:50342, task: repair
Sep 22 23:14:27.912 INFO listening, local_addr: 127.0.0.1:50342, task: repair
Sep 22 23:14:27.912 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:50342, task: repair
Sep 22 23:14:27.912 INFO Using repair address: 127.0.0.1:50342, task: main
Sep 22 23:14:27.912 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.912 INFO Upstairs starts
Sep 22 23:14:27.912 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:14:27.912 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:14:27.913 INFO Crucible stats registered with UUID: 93c2fcb7-2c23-478d-80cc-ee6a7af6fab3
Sep 22 23:14:27.913 INFO Crucible 93c2fcb7-2c23-478d-80cc-ee6a7af6fab3 has session id: 165cc207-0c17-4a5d-a35b-263abf9ec78c
Sep 22 23:14:27.913 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.913 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.913 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.913 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.913 INFO Opened existing region file "/tmp/downstairs-An1Xd1ec/region.json"
Sep 22 23:14:27.913 INFO Database read version 1
Sep 22 23:14:27.913 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.913 INFO Database write version 1
Sep 22 23:14:27.913 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.913 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.913 INFO [0] connecting to 127.0.0.1:53467, looper: 0
Sep 22 23:14:27.913 INFO [1] connecting to 127.0.0.1:38389, looper: 1
Sep 22 23:14:27.913 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.913 INFO Opened existing region file "/tmp/downstairs-NOXcduXR/region.json"
Sep 22 23:14:27.913 INFO Database read version 1
Sep 22 23:14:27.913 INFO [2] connecting to 127.0.0.1:58380, looper: 2
Sep 22 23:14:27.913 INFO Database write version 1
Sep 22 23:14:27.913 INFO up_listen starts, task: up_listen
Sep 22 23:14:27.913 INFO Wait for all three downstairs to come online
Sep 22 23:14:27.913 INFO Flush timeout: 0.5
Sep 22 23:14:27.913 INFO accepted connection from 127.0.0.1:43005, task: main
Sep 22 23:14:27.913 INFO accepted connection from 127.0.0.1:48596, task: main
Sep 22 23:14:27.913 INFO accepted connection from 127.0.0.1:56983, task: main
Sep 22 23:14:27.914 INFO [0] 93c2fcb7-2c23-478d-80cc-ee6a7af6fab3 looper connected, looper: 0
Sep 22 23:14:27.914 INFO [0] Proc runs for 127.0.0.1:53467 in state New
Sep 22 23:14:27.914 INFO [1] 93c2fcb7-2c23-478d-80cc-ee6a7af6fab3 looper connected, looper: 1
Sep 22 23:14:27.914 INFO [1] Proc runs for 127.0.0.1:38389 in state New
Sep 22 23:14:27.914 INFO [2] 93c2fcb7-2c23-478d-80cc-ee6a7af6fab3 looper connected, looper: 2
Sep 22 23:14:27.914 INFO [2] Proc runs for 127.0.0.1:58380 in state New
Sep 22 23:14:27.914 INFO Upstairs starts
Sep 22 23:14:27.914 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:14:27.914 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:14:27.914 INFO Crucible stats registered with UUID: 0b427989-df2d-43e0-a6c9-e66d936a0f2b
Sep 22 23:14:27.914 INFO Crucible 0b427989-df2d-43e0-a6c9-e66d936a0f2b has session id: aad445e1-34d8-45f9-8dac-afbdaed159f9
Sep 22 23:14:27.914 INFO Connection request from 93c2fcb7-2c23-478d-80cc-ee6a7af6fab3 with version 4, task: proc
Sep 22 23:14:27.914 INFO upstairs UpstairsConnection { upstairs_id: 93c2fcb7-2c23-478d-80cc-ee6a7af6fab3, session_id: 4916c75c-d050-4100-ba66-5ccbce289fef, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.914 INFO Connection request from 93c2fcb7-2c23-478d-80cc-ee6a7af6fab3 with version 4, task: proc
Sep 22 23:14:27.915 INFO upstairs UpstairsConnection { upstairs_id: 93c2fcb7-2c23-478d-80cc-ee6a7af6fab3, session_id: 4916c75c-d050-4100-ba66-5ccbce289fef, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.915 INFO Connection request from 93c2fcb7-2c23-478d-80cc-ee6a7af6fab3 with version 4, task: proc
Sep 22 23:14:27.915 INFO upstairs UpstairsConnection { upstairs_id: 93c2fcb7-2c23-478d-80cc-ee6a7af6fab3, session_id: 4916c75c-d050-4100-ba66-5ccbce289fef, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.915 INFO [0] connecting to 127.0.0.1:48636, looper: 0
Sep 22 23:14:27.915 INFO [1] connecting to 127.0.0.1:59653, looper: 1
Sep 22 23:14:27.915 INFO [2] connecting to 127.0.0.1:56727, looper: 2
Sep 22 23:14:27.915 INFO up_listen starts, task: up_listen
Sep 22 23:14:27.915 INFO Wait for all three downstairs to come online
Sep 22 23:14:27.915 INFO Flush timeout: 0.5
Sep 22 23:14:27.915 INFO accepted connection from 127.0.0.1:36075, task: main
Sep 22 23:14:27.915 INFO accepted connection from 127.0.0.1:40019, task: main
Sep 22 23:14:27.915 INFO accepted connection from 127.0.0.1:57920, task: main
Sep 22 23:14:27.916 INFO [0] 93c2fcb7-2c23-478d-80cc-ee6a7af6fab3 (4916c75c-d050-4100-ba66-5ccbce289fef) New New New ds_transition to WaitActive
Sep 22 23:14:27.916 INFO [0] Transition from New to WaitActive
Sep 22 23:14:27.916 INFO [1] 93c2fcb7-2c23-478d-80cc-ee6a7af6fab3 (4916c75c-d050-4100-ba66-5ccbce289fef) WaitActive New New ds_transition to WaitActive
Sep 22 23:14:27.916 INFO [1] Transition from New to WaitActive
Sep 22 23:14:27.916 INFO [2] 93c2fcb7-2c23-478d-80cc-ee6a7af6fab3 (4916c75c-d050-4100-ba66-5ccbce289fef) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:14:27.916 INFO [2] Transition from New to WaitActive
Sep 22 23:14:27.916 INFO [0] 0b427989-df2d-43e0-a6c9-e66d936a0f2b looper connected, looper: 0
Sep 22 23:14:27.916 INFO [0] Proc runs for 127.0.0.1:48636 in state New
Sep 22 23:14:27.916 INFO UUID: 27e21158-4dda-4f80-89f0-fd433654b40d
Sep 22 23:14:27.916 INFO [1] 0b427989-df2d-43e0-a6c9-e66d936a0f2b looper connected, looper: 1
Sep 22 23:14:27.916 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.916 INFO [1] Proc runs for 127.0.0.1:59653 in state New
Sep 22 23:14:27.916 INFO [2] 0b427989-df2d-43e0-a6c9-e66d936a0f2b looper connected, looper: 2
Sep 22 23:14:27.916 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.916 INFO [2] Proc runs for 127.0.0.1:56727 in state New
Sep 22 23:14:27.916 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.916 INFO Using address: 127.0.0.1:35764, task: main
Sep 22 23:14:27.916 INFO UUID: 2f70c471-1dde-474e-a4ed-18909de1eaf6
Sep 22 23:14:27.916 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.916 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.916 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.916 INFO Using address: 127.0.0.1:35263, task: main
Sep 22 23:14:27.916 INFO Connection request from 0b427989-df2d-43e0-a6c9-e66d936a0f2b with version 4, task: proc
Sep 22 23:14:27.916 INFO upstairs UpstairsConnection { upstairs_id: 0b427989-df2d-43e0-a6c9-e66d936a0f2b, session_id: b04125f1-41ca-469f-96f2-8c8ce78343c3, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.916 INFO Connection request from 0b427989-df2d-43e0-a6c9-e66d936a0f2b with version 4, task: proc
Sep 22 23:14:27.916 INFO upstairs UpstairsConnection { upstairs_id: 0b427989-df2d-43e0-a6c9-e66d936a0f2b, session_id: b04125f1-41ca-469f-96f2-8c8ce78343c3, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.916 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.916 INFO Connection request from 0b427989-df2d-43e0-a6c9-e66d936a0f2b with version 4, task: proc
Sep 22 23:14:27.916 INFO upstairs UpstairsConnection { upstairs_id: 0b427989-df2d-43e0-a6c9-e66d936a0f2b, session_id: b04125f1-41ca-469f-96f2-8c8ce78343c3, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.917 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:33592, task: repair
Sep 22 23:14:27.917 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:33592, task: repair
Sep 22 23:14:27.917 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.917 INFO listening, local_addr: 127.0.0.1:33592, task: repair
Sep 22 23:14:27.917 INFO [0] 0b427989-df2d-43e0-a6c9-e66d936a0f2b (b04125f1-41ca-469f-96f2-8c8ce78343c3) New New New ds_transition to WaitActive
Sep 22 23:14:27.917 INFO [0] Transition from New to WaitActive
Sep 22 23:14:27.917 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:47379, task: repair
Sep 22 23:14:27.917 INFO [1] 0b427989-df2d-43e0-a6c9-e66d936a0f2b (b04125f1-41ca-469f-96f2-8c8ce78343c3) WaitActive New New ds_transition to WaitActive
Sep 22 23:14:27.917 INFO [1] Transition from New to WaitActive
Sep 22 23:14:27.917 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:47379, task: repair
Sep 22 23:14:27.917 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:33592, task: repair
Sep 22 23:14:27.917 INFO [2] 0b427989-df2d-43e0-a6c9-e66d936a0f2b (b04125f1-41ca-469f-96f2-8c8ce78343c3) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:14:27.917 INFO [2] Transition from New to WaitActive
Sep 22 23:14:27.917 INFO Using repair address: 127.0.0.1:33592, task: main
Sep 22 23:14:27.917 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.917 INFO listening, local_addr: 127.0.0.1:47379, task: repair
The guest has requested activation
Sep 22 23:14:27.917 INFO 93c2fcb7-2c23-478d-80cc-ee6a7af6fab3 active request set
Sep 22 23:14:27.917 INFO [0] received activate with gen 1
Sep 22 23:14:27.917 INFO [0] client got ds_active_rx, promote! session 4916c75c-d050-4100-ba66-5ccbce289fef
Sep 22 23:14:27.917 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:47379, task: repair
Sep 22 23:14:27.917 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.917 INFO [1] received activate with gen 1
Sep 22 23:14:27.917 INFO [1] client got ds_active_rx, promote! session 4916c75c-d050-4100-ba66-5ccbce289fef
Sep 22 23:14:27.917 INFO [2] received activate with gen 1
Sep 22 23:14:27.917 INFO [2] client got ds_active_rx, promote! session 4916c75c-d050-4100-ba66-5ccbce289fef
Sep 22 23:14:27.917 INFO Using repair address: 127.0.0.1:47379, task: main
Sep 22 23:14:27.917 INFO Created new region file "/tmp/downstairs-xae6Zhcj/region.json"
Sep 22 23:14:27.917 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.917 INFO UpstairsConnection { upstairs_id: 93c2fcb7-2c23-478d-80cc-ee6a7af6fab3, session_id: 4916c75c-d050-4100-ba66-5ccbce289fef, gen: 1 } is now active (read-write)
Sep 22 23:14:27.917 INFO UpstairsConnection { upstairs_id: 93c2fcb7-2c23-478d-80cc-ee6a7af6fab3, session_id: 4916c75c-d050-4100-ba66-5ccbce289fef, gen: 1 } is now active (read-write)
Sep 22 23:14:27.917 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.917 INFO UpstairsConnection { upstairs_id: 93c2fcb7-2c23-478d-80cc-ee6a7af6fab3, session_id: 4916c75c-d050-4100-ba66-5ccbce289fef, gen: 1 } is now active (read-write)
Sep 22 23:14:27.918 INFO Created new region file "/tmp/downstairs-U79LijME/region.json"
Sep 22 23:14:27.918 INFO [0] downstairs client at 127.0.0.1:53467 has UUID b78cfd9a-1100-4037-aec9-2c2cabfc0ce6
Sep 22 23:14:27.918 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: b78cfd9a-1100-4037-aec9-2c2cabfc0ce6, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.918 INFO 93c2fcb7-2c23-478d-80cc-ee6a7af6fab3 WaitActive WaitActive WaitActive
Sep 22 23:14:27.918 INFO [1] downstairs client at 127.0.0.1:38389 has UUID 555bafbf-d9bb-408a-aa71-328bc8a8cc92
Sep 22 23:14:27.918 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 555bafbf-d9bb-408a-aa71-328bc8a8cc92, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.918 INFO 93c2fcb7-2c23-478d-80cc-ee6a7af6fab3 WaitActive WaitActive WaitActive
Sep 22 23:14:27.918 INFO [2] downstairs client at 127.0.0.1:58380 has UUID 16b61f39-4c68-4f56-ae07-8508cf0adb07
Sep 22 23:14:27.918 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 16b61f39-4c68-4f56-ae07-8508cf0adb07, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.918 INFO 93c2fcb7-2c23-478d-80cc-ee6a7af6fab3 WaitActive WaitActive WaitActive
Sep 22 23:14:27.918 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.918 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.919 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.919 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.919 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.919 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.919 INFO [0] 93c2fcb7-2c23-478d-80cc-ee6a7af6fab3 (4916c75c-d050-4100-ba66-5ccbce289fef) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.919 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.919 WARN [0] new RM replaced this: None
Sep 22 23:14:27.919 INFO [0] Starts reconcile loop
Sep 22 23:14:27.919 INFO [1] 93c2fcb7-2c23-478d-80cc-ee6a7af6fab3 (4916c75c-d050-4100-ba66-5ccbce289fef) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.919 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.919 WARN [1] new RM replaced this: None
Sep 22 23:14:27.919 INFO [1] Starts reconcile loop
Sep 22 23:14:27.919 INFO [2] 93c2fcb7-2c23-478d-80cc-ee6a7af6fab3 (4916c75c-d050-4100-ba66-5ccbce289fef) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.919 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.919 WARN [2] new RM replaced this: None
Sep 22 23:14:27.919 INFO [2] Starts reconcile loop
Sep 22 23:14:27.920 INFO [0] 127.0.0.1:53467 task reports connection:true
Sep 22 23:14:27.920 INFO 93c2fcb7-2c23-478d-80cc-ee6a7af6fab3 WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:14:27.920 INFO [0]R  flush_numbers: [0, 0]
Sep 22 23:14:27.920 INFO [0]R  generation: [0, 0]
Sep 22 23:14:27.920 INFO [0]R  dirty: [false, false]
Sep 22 23:14:27.920 INFO [1]R  flush_numbers: [0, 0]
Sep 22 23:14:27.920 INFO [1]R  generation: [0, 0]
Sep 22 23:14:27.920 INFO [1]R  dirty: [false, false]
Sep 22 23:14:27.920 INFO [2]R  flush_numbers: [0, 0]
Sep 22 23:14:27.920 INFO [2]R  generation: [0, 0]
Sep 22 23:14:27.920 INFO [2]R  dirty: [false, false]
Sep 22 23:14:27.920 INFO Max found gen is 1
Sep 22 23:14:27.920 INFO Generation requested: 1 >= found:1
Sep 22 23:14:27.920 INFO Next flush: 1
Sep 22 23:14:27.920 INFO All extents match
Sep 22 23:14:27.920 INFO No downstairs repair required
Sep 22 23:14:27.920 INFO No initial repair work was required
Sep 22 23:14:27.920 INFO Set Downstairs and Upstairs active
Sep 22 23:14:27.920 INFO 93c2fcb7-2c23-478d-80cc-ee6a7af6fab3 is now active with session: 4916c75c-d050-4100-ba66-5ccbce289fef
Sep 22 23:14:27.920 INFO 93c2fcb7-2c23-478d-80cc-ee6a7af6fab3 Set Active after no repair
Sep 22 23:14:27.920 INFO Notify all downstairs, region set compare is done.
Sep 22 23:14:27.920 INFO Set check for repair
Sep 22 23:14:27.920 INFO [1] 127.0.0.1:38389 task reports connection:true
Sep 22 23:14:27.920 INFO 93c2fcb7-2c23-478d-80cc-ee6a7af6fab3 Active Active Active
Sep 22 23:14:27.920 INFO Set check for repair
Sep 22 23:14:27.920 INFO [2] 127.0.0.1:58380 task reports connection:true
Sep 22 23:14:27.920 INFO 93c2fcb7-2c23-478d-80cc-ee6a7af6fab3 Active Active Active
Sep 22 23:14:27.920 INFO Set check for repair
Sep 22 23:14:27.920 INFO [0] received reconcile message
Sep 22 23:14:27.920 INFO [0] All repairs completed, exit
Sep 22 23:14:27.920 INFO [0] Starts cmd_loop
Sep 22 23:14:27.920 INFO [1] received reconcile message
Sep 22 23:14:27.920 INFO [1] All repairs completed, exit
Sep 22 23:14:27.920 INFO [1] Starts cmd_loop
Sep 22 23:14:27.920 INFO [2] received reconcile message
Sep 22 23:14:27.920 INFO [2] All repairs completed, exit
Sep 22 23:14:27.920 INFO [2] Starts cmd_loop
The guest has finished waiting for activation
The guest has requested activation
Sep 22 23:14:27.921 INFO 0b427989-df2d-43e0-a6c9-e66d936a0f2b active request set
Sep 22 23:14:27.921 INFO [0] received activate with gen 1
Sep 22 23:14:27.921 INFO [0] client got ds_active_rx, promote! session b04125f1-41ca-469f-96f2-8c8ce78343c3
Sep 22 23:14:27.921 INFO [1] received activate with gen 1
Sep 22 23:14:27.921 INFO [1] client got ds_active_rx, promote! session b04125f1-41ca-469f-96f2-8c8ce78343c3
Sep 22 23:14:27.921 INFO [2] received activate with gen 1
Sep 22 23:14:27.921 INFO [2] client got ds_active_rx, promote! session b04125f1-41ca-469f-96f2-8c8ce78343c3
Sep 22 23:14:27.921 INFO UpstairsConnection { upstairs_id: 0b427989-df2d-43e0-a6c9-e66d936a0f2b, session_id: b04125f1-41ca-469f-96f2-8c8ce78343c3, gen: 1 } is now active (read-write)
Sep 22 23:14:27.921 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.921 INFO Opened existing region file "/tmp/downstairs-xae6Zhcj/region.json"
Sep 22 23:14:27.921 INFO Database read version 1
Sep 22 23:14:27.921 INFO UpstairsConnection { upstairs_id: 0b427989-df2d-43e0-a6c9-e66d936a0f2b, session_id: b04125f1-41ca-469f-96f2-8c8ce78343c3, gen: 1 } is now active (read-write)
Sep 22 23:14:27.921 INFO Database write version 1
Sep 22 23:14:27.921 INFO UpstairsConnection { upstairs_id: 0b427989-df2d-43e0-a6c9-e66d936a0f2b, session_id: b04125f1-41ca-469f-96f2-8c8ce78343c3, gen: 1 } is now active (read-write)
Sep 22 23:14:27.921 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.921 INFO Opened existing region file "/tmp/downstairs-U79LijME/region.json"
Sep 22 23:14:27.921 INFO Database read version 1
Sep 22 23:14:27.921 INFO Database write version 1
Sep 22 23:14:27.921 INFO [0] downstairs client at 127.0.0.1:48636 has UUID 0f3af6ad-c169-4a67-ae79-bc5859c15909
Sep 22 23:14:27.921 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 0f3af6ad-c169-4a67-ae79-bc5859c15909, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.921 INFO 0b427989-df2d-43e0-a6c9-e66d936a0f2b WaitActive WaitActive WaitActive
Sep 22 23:14:27.922 INFO [1] downstairs client at 127.0.0.1:59653 has UUID aef90a89-454d-4861-b15d-20c99780f2e8
Sep 22 23:14:27.922 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: aef90a89-454d-4861-b15d-20c99780f2e8, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.922 INFO 0b427989-df2d-43e0-a6c9-e66d936a0f2b WaitActive WaitActive WaitActive
Sep 22 23:14:27.922 INFO [2] downstairs client at 127.0.0.1:56727 has UUID 6085db72-2a98-4898-ac66-9af2876a589f
Sep 22 23:14:27.922 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 6085db72-2a98-4898-ac66-9af2876a589f, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.922 INFO 0b427989-df2d-43e0-a6c9-e66d936a0f2b WaitActive WaitActive WaitActive
Sep 22 23:14:27.922 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.922 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.922 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.922 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.922 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.923 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.923 INFO [0] 0b427989-df2d-43e0-a6c9-e66d936a0f2b (b04125f1-41ca-469f-96f2-8c8ce78343c3) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.923 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.923 WARN [0] new RM replaced this: None
Sep 22 23:14:27.923 INFO [0] Starts reconcile loop
Sep 22 23:14:27.923 INFO [1] 0b427989-df2d-43e0-a6c9-e66d936a0f2b (b04125f1-41ca-469f-96f2-8c8ce78343c3) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.923 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.923 WARN [1] new RM replaced this: None
Sep 22 23:14:27.923 INFO [1] Starts reconcile loop
Sep 22 23:14:27.923 INFO [2] 0b427989-df2d-43e0-a6c9-e66d936a0f2b (b04125f1-41ca-469f-96f2-8c8ce78343c3) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.923 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.923 WARN [2] new RM replaced this: None
Sep 22 23:14:27.923 INFO [2] Starts reconcile loop
Sep 22 23:14:27.923 INFO [0] 127.0.0.1:48636 task reports connection:true
Sep 22 23:14:27.923 INFO 0b427989-df2d-43e0-a6c9-e66d936a0f2b WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:14:27.923 INFO [0]R  flush_numbers: [0, 0]
Sep 22 23:14:27.923 INFO [0]R  generation: [0, 0]
Sep 22 23:14:27.923 INFO [0]R  dirty: [false, false]
Sep 22 23:14:27.923 INFO [1]R  flush_numbers: [0, 0]
Sep 22 23:14:27.923 INFO [1]R  generation: [0, 0]
Sep 22 23:14:27.923 INFO [1]R  dirty: [false, false]
Sep 22 23:14:27.923 INFO [2]R  flush_numbers: [0, 0]
Sep 22 23:14:27.923 INFO [2]R  generation: [0, 0]
Sep 22 23:14:27.923 INFO [2]R  dirty: [false, false]
Sep 22 23:14:27.923 INFO Max found gen is 1
Sep 22 23:14:27.923 INFO Generation requested: 1 >= found:1
Sep 22 23:14:27.923 INFO Next flush: 1
Sep 22 23:14:27.923 INFO All extents match
Sep 22 23:14:27.923 INFO No downstairs repair required
Sep 22 23:14:27.923 INFO No initial repair work was required
Sep 22 23:14:27.923 INFO Set Downstairs and Upstairs active
Sep 22 23:14:27.923 INFO 0b427989-df2d-43e0-a6c9-e66d936a0f2b is now active with session: b04125f1-41ca-469f-96f2-8c8ce78343c3
Sep 22 23:14:27.923 INFO 0b427989-df2d-43e0-a6c9-e66d936a0f2b Set Active after no repair
Sep 22 23:14:27.923 INFO Notify all downstairs, region set compare is done.
Sep 22 23:14:27.923 INFO Set check for repair
Sep 22 23:14:27.923 INFO [1] 127.0.0.1:59653 task reports connection:true
Sep 22 23:14:27.923 INFO 0b427989-df2d-43e0-a6c9-e66d936a0f2b Active Active Active
Sep 22 23:14:27.923 INFO Set check for repair
Sep 22 23:14:27.923 INFO [2] 127.0.0.1:56727 task reports connection:true
Sep 22 23:14:27.923 INFO 0b427989-df2d-43e0-a6c9-e66d936a0f2b Active Active Active
Sep 22 23:14:27.923 INFO Set check for repair
Sep 22 23:14:27.923 INFO [0] received reconcile message
Sep 22 23:14:27.923 INFO [0] All repairs completed, exit
Sep 22 23:14:27.924 INFO [0] Starts cmd_loop
Sep 22 23:14:27.924 INFO [1] received reconcile message
Sep 22 23:14:27.924 INFO [1] All repairs completed, exit
Sep 22 23:14:27.924 INFO [1] Starts cmd_loop
Sep 22 23:14:27.924 INFO [2] received reconcile message
Sep 22 23:14:27.924 INFO [2] All repairs completed, exit
Sep 22 23:14:27.924 INFO [2] Starts cmd_loop
The guest has finished waiting for activation
Sep 22 23:14:27.924 INFO UUID: 52a150fb-f4af-4b8f-9c89-001d2d0a9421
Sep 22 23:14:27.924 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.924 INFO UUID: c1d22958-e424-406b-a939-8b7b5f9f6177
Sep 22 23:14:27.924 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.924 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.924 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.924 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.924 INFO Using address: 127.0.0.1:61372, task: main
Sep 22 23:14:27.924 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.924 INFO Using address: 127.0.0.1:43569, task: main
Sep 22 23:14:27.924 DEBG IO Write 1000 has deps []
Sep 22 23:14:27.925 DEBG up_ds_listen was notified
Sep 22 23:14:27.925 DEBG up_ds_listen process 1000
Sep 22 23:14:27.925 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.925 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:14:27.925 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.925 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:33546, task: repair
Sep 22 23:14:27.925 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.925 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:33546, task: repair
Sep 22 23:14:27.925 INFO listening, local_addr: 127.0.0.1:33546, task: repair
Sep 22 23:14:27.925 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:41883, task: repair
Sep 22 23:14:27.925 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:41883, task: repair
Sep 22 23:14:27.925 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:33546, task: repair
Sep 22 23:14:27.925 INFO listening, local_addr: 127.0.0.1:41883, task: repair
Sep 22 23:14:27.925 INFO Using repair address: 127.0.0.1:33546, task: main
Sep 22 23:14:27.925 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.925 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.925 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:41883, task: repair
Sep 22 23:14:27.925 INFO Created new region file "/tmp/downstairs-Cj3zW210/region.json"
Sep 22 23:14:27.925 INFO Using repair address: 127.0.0.1:41883, task: main
Sep 22 23:14:27.925 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.926 DEBG Write     :1000 deps:[] res:true
Sep 22 23:14:27.926 INFO Upstairs starts
Sep 22 23:14:27.926 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:14:27.926 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:14:27.926 INFO Crucible stats registered with UUID: c29f404c-9fcb-4abe-97d9-77d5448a344e
Sep 22 23:14:27.926 INFO Crucible c29f404c-9fcb-4abe-97d9-77d5448a344e has session id: 224aedb5-8127-4463-8d4f-ad8a5dc531ba
Sep 22 23:14:27.926 DEBG Write     :1000 deps:[] res:true
Sep 22 23:14:27.926 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.926 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.927 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.927 INFO [0] connecting to 127.0.0.1:63499, looper: 0
Sep 22 23:14:27.927 INFO [1] connecting to 127.0.0.1:35263, looper: 1
Sep 22 23:14:27.927 DEBG Write     :1000 deps:[] res:true
Sep 22 23:14:27.927 INFO [2] connecting to 127.0.0.1:61372, looper: 2
Sep 22 23:14:27.927 INFO up_listen starts, task: up_listen
Sep 22 23:14:27.927 INFO Wait for all three downstairs to come online
Sep 22 23:14:27.927 INFO Flush timeout: 0.5
Sep 22 23:14:27.927 INFO accepted connection from 127.0.0.1:36199, task: main
Sep 22 23:14:27.927 DEBG IO Write 1000 has deps []
Sep 22 23:14:27.927 INFO accepted connection from 127.0.0.1:46428, task: main
Sep 22 23:14:27.928 INFO accepted connection from 127.0.0.1:42343, task: main
Sep 22 23:14:27.928 DEBG up_ds_listen was notified
Sep 22 23:14:27.928 DEBG up_ds_listen process 1000
Sep 22 23:14:27.928 INFO [0] c29f404c-9fcb-4abe-97d9-77d5448a344e looper connected, looper: 0
Sep 22 23:14:27.928 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:14:27.928 INFO [0] Proc runs for 127.0.0.1:63499 in state New
Sep 22 23:14:27.928 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.928 INFO [1] c29f404c-9fcb-4abe-97d9-77d5448a344e looper connected, looper: 1
Sep 22 23:14:27.928 INFO [1] Proc runs for 127.0.0.1:35263 in state New
Sep 22 23:14:27.928 INFO [2] c29f404c-9fcb-4abe-97d9-77d5448a344e looper connected, looper: 2
Sep 22 23:14:27.928 INFO [2] Proc runs for 127.0.0.1:61372 in state New
Sep 22 23:14:27.928 INFO Connection request from c29f404c-9fcb-4abe-97d9-77d5448a344e with version 4, task: proc
Sep 22 23:14:27.928 INFO upstairs UpstairsConnection { upstairs_id: c29f404c-9fcb-4abe-97d9-77d5448a344e, session_id: d5786959-64ba-4a45-85e5-b539511f4f59, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.928 INFO Connection request from c29f404c-9fcb-4abe-97d9-77d5448a344e with version 4, task: proc
Sep 22 23:14:27.928 INFO upstairs UpstairsConnection { upstairs_id: c29f404c-9fcb-4abe-97d9-77d5448a344e, session_id: d5786959-64ba-4a45-85e5-b539511f4f59, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.928 INFO Connection request from c29f404c-9fcb-4abe-97d9-77d5448a344e with version 4, task: proc
Sep 22 23:14:27.928 INFO upstairs UpstairsConnection { upstairs_id: c29f404c-9fcb-4abe-97d9-77d5448a344e, session_id: d5786959-64ba-4a45-85e5-b539511f4f59, gen: 1 } connected, version 4, task: proc
The guest has requested activation
Sep 22 23:14:27.929 INFO [0] c29f404c-9fcb-4abe-97d9-77d5448a344e (d5786959-64ba-4a45-85e5-b539511f4f59) New New New ds_transition to WaitActive
Sep 22 23:14:27.929 INFO [0] Transition from New to WaitActive
Sep 22 23:14:27.929 INFO [1] c29f404c-9fcb-4abe-97d9-77d5448a344e (d5786959-64ba-4a45-85e5-b539511f4f59) WaitActive New New ds_transition to WaitActive
Sep 22 23:14:27.929 INFO [1] Transition from New to WaitActive
Sep 22 23:14:27.929 INFO [2] c29f404c-9fcb-4abe-97d9-77d5448a344e (d5786959-64ba-4a45-85e5-b539511f4f59) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:14:27.929 INFO [2] Transition from New to WaitActive
Sep 22 23:14:27.929 DEBG Write     :1000 deps:[] res:true
Sep 22 23:14:27.929 INFO c29f404c-9fcb-4abe-97d9-77d5448a344e active request set
Sep 22 23:14:27.929 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.929 INFO Opened existing region file "/tmp/downstairs-Cj3zW210/region.json"
Sep 22 23:14:27.929 INFO [0] received activate with gen 1
Sep 22 23:14:27.929 INFO Database read version 1
Sep 22 23:14:27.929 INFO [0] client got ds_active_rx, promote! session d5786959-64ba-4a45-85e5-b539511f4f59
Sep 22 23:14:27.929 INFO Database write version 1
Sep 22 23:14:27.929 INFO [1] received activate with gen 1
Sep 22 23:14:27.929 INFO [1] client got ds_active_rx, promote! session d5786959-64ba-4a45-85e5-b539511f4f59
Sep 22 23:14:27.929 INFO [2] received activate with gen 1
Sep 22 23:14:27.929 INFO [2] client got ds_active_rx, promote! session d5786959-64ba-4a45-85e5-b539511f4f59
Sep 22 23:14:27.929 DEBG Write     :1000 deps:[] res:true
Sep 22 23:14:27.929 INFO UpstairsConnection { upstairs_id: c29f404c-9fcb-4abe-97d9-77d5448a344e, session_id: d5786959-64ba-4a45-85e5-b539511f4f59, gen: 1 } is now active (read-write)
Sep 22 23:14:27.930 INFO UpstairsConnection { upstairs_id: c29f404c-9fcb-4abe-97d9-77d5448a344e, session_id: d5786959-64ba-4a45-85e5-b539511f4f59, gen: 1 } is now active (read-write)
Sep 22 23:14:27.930 INFO UpstairsConnection { upstairs_id: c29f404c-9fcb-4abe-97d9-77d5448a344e, session_id: d5786959-64ba-4a45-85e5-b539511f4f59, gen: 1 } is now active (read-write)
Sep 22 23:14:27.930 DEBG Write     :1000 deps:[] res:true
Sep 22 23:14:27.930 DEBG IO Read  1001 has deps [JobId(1000)]
Sep 22 23:14:27.930 INFO [0] downstairs client at 127.0.0.1:63499 has UUID 9a08289f-c73f-4f3d-af8e-b9eb4fe46f56
Sep 22 23:14:27.930 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 9a08289f-c73f-4f3d-af8e-b9eb4fe46f56, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.930 INFO c29f404c-9fcb-4abe-97d9-77d5448a344e WaitActive WaitActive WaitActive
Sep 22 23:14:27.930 INFO [1] downstairs client at 127.0.0.1:35263 has UUID 2f70c471-1dde-474e-a4ed-18909de1eaf6
Sep 22 23:14:27.930 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 2f70c471-1dde-474e-a4ed-18909de1eaf6, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.930 INFO c29f404c-9fcb-4abe-97d9-77d5448a344e WaitActive WaitActive WaitActive
Sep 22 23:14:27.930 INFO [2] downstairs client at 127.0.0.1:61372 has UUID 52a150fb-f4af-4b8f-9c89-001d2d0a9421
Sep 22 23:14:27.930 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 52a150fb-f4af-4b8f-9c89-001d2d0a9421, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.930 INFO c29f404c-9fcb-4abe-97d9-77d5448a344e WaitActive WaitActive WaitActive
Sep 22 23:14:27.931 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.931 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.931 DEBG Read      :1001 deps:[JobId(1000)] res:true
Sep 22 23:14:27.931 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.931 DEBG Read      :1001 deps:[JobId(1000)] res:true
Sep 22 23:14:27.931 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.931 DEBG Read      :1001 deps:[JobId(1000)] res:true
Sep 22 23:14:27.931 INFO UUID: 484c6303-bd26-472d-a8db-93e975b34d58
Sep 22 23:14:27.931 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.931 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.932 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.932 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.932 INFO Using address: 127.0.0.1:41379, task: main
Sep 22 23:14:27.932 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.932 INFO [0] c29f404c-9fcb-4abe-97d9-77d5448a344e (d5786959-64ba-4a45-85e5-b539511f4f59) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.932 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.932 WARN [0] new RM replaced this: None
Sep 22 23:14:27.932 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.932 INFO [0] Starts reconcile loop
Sep 22 23:14:27.932 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:52744, task: repair
Sep 22 23:14:27.932 INFO [1] c29f404c-9fcb-4abe-97d9-77d5448a344e (d5786959-64ba-4a45-85e5-b539511f4f59) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.932 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:52744, task: repair
Sep 22 23:14:27.932 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.932 WARN [1] new RM replaced this: None
Sep 22 23:14:27.932 INFO listening, local_addr: 127.0.0.1:52744, task: repair
Sep 22 23:14:27.932 INFO [1] Starts reconcile loop
Sep 22 23:14:27.932 DEBG [0] Read AckReady 1001, : downstairs
Sep 22 23:14:27.932 INFO [2] c29f404c-9fcb-4abe-97d9-77d5448a344e (d5786959-64ba-4a45-85e5-b539511f4f59) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.932 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:52744, task: repair
Sep 22 23:14:27.932 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.932 WARN [2] new RM replaced this: None
Sep 22 23:14:27.932 INFO Using repair address: 127.0.0.1:52744, task: main
Sep 22 23:14:27.932 INFO [2] Starts reconcile loop
Sep 22 23:14:27.932 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.932 INFO [0] 127.0.0.1:63499 task reports connection:true
Sep 22 23:14:27.932 INFO c29f404c-9fcb-4abe-97d9-77d5448a344e WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:14:27.932 INFO [0]R  flush_numbers: [0, 0]
Sep 22 23:14:27.932 INFO [0]R  generation: [0, 0]
Sep 22 23:14:27.932 INFO [0]R  dirty: [false, false]
Sep 22 23:14:27.932 INFO [1]R  flush_numbers: [0, 0]
Sep 22 23:14:27.932 INFO [1]R  generation: [0, 0]
Sep 22 23:14:27.932 INFO [1]R  dirty: [false, false]
Sep 22 23:14:27.932 INFO [2]R  flush_numbers: [0, 0]
Sep 22 23:14:27.932 INFO [2]R  generation: [0, 0]
Sep 22 23:14:27.932 DEBG [1] Read already AckReady 1001, : downstairs
Sep 22 23:14:27.932 INFO [2]R  dirty: [false, false]
Sep 22 23:14:27.933 INFO Max found gen is 1
Sep 22 23:14:27.933 INFO Generation requested: 1 >= found:1
Sep 22 23:14:27.933 INFO Next flush: 1
Sep 22 23:14:27.933 INFO Upstairs starts
Sep 22 23:14:27.933 INFO All extents match
Sep 22 23:14:27.933 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:14:27.933 INFO No downstairs repair required
Sep 22 23:14:27.933 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:14:27.933 INFO No initial repair work was required
Sep 22 23:14:27.933 INFO Set Downstairs and Upstairs active
Sep 22 23:14:27.933 INFO Crucible stats registered with UUID: f94aa62c-b9ca-4a22-b94a-357bd8d6b5ab
Sep 22 23:14:27.933 INFO Crucible f94aa62c-b9ca-4a22-b94a-357bd8d6b5ab has session id: 86fb1b98-1b29-4954-8159-960709b7882f
Sep 22 23:14:27.933 INFO c29f404c-9fcb-4abe-97d9-77d5448a344e is now active with session: d5786959-64ba-4a45-85e5-b539511f4f59
Sep 22 23:14:27.933 INFO c29f404c-9fcb-4abe-97d9-77d5448a344e Set Active after no repair
Sep 22 23:14:27.933 INFO Notify all downstairs, region set compare is done.
Sep 22 23:14:27.933 INFO Set check for repair
Sep 22 23:14:27.933 INFO [1] 127.0.0.1:35263 task reports connection:true
Sep 22 23:14:27.933 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.933 INFO c29f404c-9fcb-4abe-97d9-77d5448a344e Active Active Active
Sep 22 23:14:27.933 INFO Set check for repair
Sep 22 23:14:27.933 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.933 DEBG [2] Read already AckReady 1001, : downstairs
Sep 22 23:14:27.933 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.933 INFO [2] 127.0.0.1:61372 task reports connection:true
Sep 22 23:14:27.933 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.933 DEBG up_ds_listen was notified
Sep 22 23:14:27.933 INFO c29f404c-9fcb-4abe-97d9-77d5448a344e Active Active Active
Sep 22 23:14:27.933 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.933 INFO Set check for repair
Sep 22 23:14:27.933 DEBG up_ds_listen process 1001
Sep 22 23:14:27.933 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.933 DEBG [A] ack job 1001:2, : downstairs
Sep 22 23:14:27.933 INFO [0] connecting to 127.0.0.1:62077, looper: 0
Sep 22 23:14:27.933 INFO [0] received reconcile message
Sep 22 23:14:27.933 INFO [0] All repairs completed, exit
Sep 22 23:14:27.933 INFO [0] Starts cmd_loop
Sep 22 23:14:27.933 INFO [1] connecting to 127.0.0.1:55380, looper: 1
Sep 22 23:14:27.933 INFO [1] received reconcile message
Sep 22 23:14:27.933 INFO [2] connecting to 127.0.0.1:41467, looper: 2
Sep 22 23:14:27.933 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.933 INFO [1] All repairs completed, exit
Sep 22 23:14:27.933 INFO up_listen starts, task: up_listen
Sep 22 23:14:27.933 INFO Wait for all three downstairs to come online
Sep 22 23:14:27.933 INFO Flush timeout: 0.5
Sep 22 23:14:27.933 INFO [1] Starts cmd_loop
Sep 22 23:14:27.933 INFO [2] received reconcile message
Sep 22 23:14:27.933 INFO [2] All repairs completed, exit
Sep 22 23:14:27.933 INFO [2] Starts cmd_loop
Sep 22 23:14:27.933 INFO accepted connection from 127.0.0.1:33010, task: main
The guest has finished waiting for activation
Sep 22 23:14:27.934 INFO accepted connection from 127.0.0.1:48696, task: main
Sep 22 23:14:27.934 INFO accepted connection from 127.0.0.1:40107, task: main
Sep 22 23:14:27.934 DEBG IO Read  1001 has deps [JobId(1000)]
Sep 22 23:14:27.934 INFO [0] f94aa62c-b9ca-4a22-b94a-357bd8d6b5ab looper connected, looper: 0
Sep 22 23:14:27.934 INFO [0] Proc runs for 127.0.0.1:62077 in state New
Sep 22 23:14:27.934 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.934 INFO [1] f94aa62c-b9ca-4a22-b94a-357bd8d6b5ab looper connected, looper: 1
Sep 22 23:14:27.934 INFO [1] Proc runs for 127.0.0.1:55380 in state New
Sep 22 23:14:27.934 INFO Created new region file "/tmp/downstairs-aQ0wdqbq/region.json"
Sep 22 23:14:27.934 INFO [2] f94aa62c-b9ca-4a22-b94a-357bd8d6b5ab looper connected, looper: 2
Sep 22 23:14:27.934 INFO [2] Proc runs for 127.0.0.1:41467 in state New
Sep 22 23:14:27.934 INFO Upstairs starts
Sep 22 23:14:27.934 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:14:27.934 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:14:27.934 INFO Crucible stats registered with UUID: e61fa5a6-7bbf-4a27-8d67-7620a71d6c63
Sep 22 23:14:27.934 INFO Crucible e61fa5a6-7bbf-4a27-8d67-7620a71d6c63 has session id: af64e58b-86df-4d1e-a59b-7e30583b77a9
Sep 22 23:14:27.934 DEBG Read      :1001 deps:[JobId(1000)] res:true
Sep 22 23:14:27.934 INFO Connection request from f94aa62c-b9ca-4a22-b94a-357bd8d6b5ab with version 4, task: proc
Sep 22 23:14:27.934 INFO upstairs UpstairsConnection { upstairs_id: f94aa62c-b9ca-4a22-b94a-357bd8d6b5ab, session_id: 3b6d6787-7c4f-4800-a80e-7e17426e2512, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.934 INFO Connection request from f94aa62c-b9ca-4a22-b94a-357bd8d6b5ab with version 4, task: proc
Sep 22 23:14:27.935 INFO upstairs UpstairsConnection { upstairs_id: f94aa62c-b9ca-4a22-b94a-357bd8d6b5ab, session_id: 3b6d6787-7c4f-4800-a80e-7e17426e2512, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.935 INFO Connection request from f94aa62c-b9ca-4a22-b94a-357bd8d6b5ab with version 4, task: proc
Sep 22 23:14:27.935 INFO upstairs UpstairsConnection { upstairs_id: f94aa62c-b9ca-4a22-b94a-357bd8d6b5ab, session_id: 3b6d6787-7c4f-4800-a80e-7e17426e2512, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.935 INFO [0] connecting to 127.0.0.1:35764, looper: 0
Sep 22 23:14:27.935 DEBG Read      :1001 deps:[JobId(1000)] res:true
Sep 22 23:14:27.935 INFO [1] connecting to 127.0.0.1:43569, looper: 1
Sep 22 23:14:27.935 INFO [2] connecting to 127.0.0.1:41379, looper: 2
Sep 22 23:14:27.935 DEBG Read      :1001 deps:[JobId(1000)] res:true
Sep 22 23:14:27.935 INFO up_listen starts, task: up_listen
Sep 22 23:14:27.935 INFO Wait for all three downstairs to come online
Sep 22 23:14:27.935 INFO Flush timeout: 0.5
Sep 22 23:14:27.935 INFO accepted connection from 127.0.0.1:59309, task: main
Sep 22 23:14:27.935 INFO accepted connection from 127.0.0.1:47302, task: main
Sep 22 23:14:27.935 INFO accepted connection from 127.0.0.1:42539, task: main
Sep 22 23:14:27.936 INFO [0] f94aa62c-b9ca-4a22-b94a-357bd8d6b5ab (3b6d6787-7c4f-4800-a80e-7e17426e2512) New New New ds_transition to WaitActive
Sep 22 23:14:27.936 INFO [0] Transition from New to WaitActive
Sep 22 23:14:27.936 INFO [1] f94aa62c-b9ca-4a22-b94a-357bd8d6b5ab (3b6d6787-7c4f-4800-a80e-7e17426e2512) WaitActive New New ds_transition to WaitActive
Sep 22 23:14:27.936 DEBG [0] Read AckReady 1001, : downstairs
Sep 22 23:14:27.936 INFO [1] Transition from New to WaitActive
Sep 22 23:14:27.936 INFO [2] f94aa62c-b9ca-4a22-b94a-357bd8d6b5ab (3b6d6787-7c4f-4800-a80e-7e17426e2512) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:14:27.936 INFO [2] Transition from New to WaitActive
Sep 22 23:14:27.936 INFO [0] e61fa5a6-7bbf-4a27-8d67-7620a71d6c63 looper connected, looper: 0
Sep 22 23:14:27.936 INFO [0] Proc runs for 127.0.0.1:35764 in state New
Sep 22 23:14:27.936 INFO [1] e61fa5a6-7bbf-4a27-8d67-7620a71d6c63 looper connected, looper: 1
Sep 22 23:14:27.936 INFO [1] Proc runs for 127.0.0.1:43569 in state New
Sep 22 23:14:27.936 DEBG [1] Read already AckReady 1001, : downstairs
Sep 22 23:14:27.936 INFO [2] e61fa5a6-7bbf-4a27-8d67-7620a71d6c63 looper connected, looper: 2
Sep 22 23:14:27.936 INFO [2] Proc runs for 127.0.0.1:41379 in state New
Sep 22 23:14:27.936 DEBG [2] Read already AckReady 1001, : downstairs
Sep 22 23:14:27.936 DEBG up_ds_listen was notified
Sep 22 23:14:27.936 DEBG up_ds_listen process 1001
Sep 22 23:14:27.937 DEBG [A] ack job 1001:2, : downstairs
Sep 22 23:14:27.937 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.937 INFO Connection request from e61fa5a6-7bbf-4a27-8d67-7620a71d6c63 with version 4, task: proc
Sep 22 23:14:27.937 INFO upstairs UpstairsConnection { upstairs_id: e61fa5a6-7bbf-4a27-8d67-7620a71d6c63, session_id: 2024cbe9-f560-42ab-89c7-6b05d6eff703, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.937 INFO Connection request from e61fa5a6-7bbf-4a27-8d67-7620a71d6c63 with version 4, task: proc
Sep 22 23:14:27.937 INFO upstairs UpstairsConnection { upstairs_id: e61fa5a6-7bbf-4a27-8d67-7620a71d6c63, session_id: 2024cbe9-f560-42ab-89c7-6b05d6eff703, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.937 INFO Connection request from e61fa5a6-7bbf-4a27-8d67-7620a71d6c63 with version 4, task: proc
Sep 22 23:14:27.937 INFO upstairs UpstairsConnection { upstairs_id: e61fa5a6-7bbf-4a27-8d67-7620a71d6c63, session_id: 2024cbe9-f560-42ab-89c7-6b05d6eff703, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.937 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.937 INFO Opened existing region file "/tmp/downstairs-aQ0wdqbq/region.json"
Sep 22 23:14:27.937 INFO Database read version 1
Sep 22 23:14:27.937 INFO Database write version 1
Sep 22 23:14:27.937 INFO [0] e61fa5a6-7bbf-4a27-8d67-7620a71d6c63 (2024cbe9-f560-42ab-89c7-6b05d6eff703) New New New ds_transition to WaitActive
Sep 22 23:14:27.937 INFO [0] Transition from New to WaitActive
Sep 22 23:14:27.937 INFO [1] e61fa5a6-7bbf-4a27-8d67-7620a71d6c63 (2024cbe9-f560-42ab-89c7-6b05d6eff703) WaitActive New New ds_transition to WaitActive
Sep 22 23:14:27.937 INFO [1] Transition from New to WaitActive
Sep 22 23:14:27.937 INFO [2] e61fa5a6-7bbf-4a27-8d67-7620a71d6c63 (2024cbe9-f560-42ab-89c7-6b05d6eff703) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:14:27.937 INFO [2] Transition from New to WaitActive
Sep 22 23:14:27.937 DEBG IO Write 1002 has deps []
The guest has requested activation
Sep 22 23:14:27.938 DEBG up_ds_listen was notified
Sep 22 23:14:27.938 DEBG up_ds_listen process 1002
Sep 22 23:14:27.938 INFO f94aa62c-b9ca-4a22-b94a-357bd8d6b5ab active request set
Sep 22 23:14:27.938 DEBG [A] ack job 1002:3, : downstairs
Sep 22 23:14:27.938 INFO [0] received activate with gen 1
Sep 22 23:14:27.938 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.938 INFO [0] client got ds_active_rx, promote! session 3b6d6787-7c4f-4800-a80e-7e17426e2512
Sep 22 23:14:27.938 INFO [1] received activate with gen 1
Sep 22 23:14:27.938 INFO [1] client got ds_active_rx, promote! session 3b6d6787-7c4f-4800-a80e-7e17426e2512
Sep 22 23:14:27.938 INFO [2] received activate with gen 1
Sep 22 23:14:27.938 INFO [2] client got ds_active_rx, promote! session 3b6d6787-7c4f-4800-a80e-7e17426e2512
Sep 22 23:14:27.938 INFO UpstairsConnection { upstairs_id: f94aa62c-b9ca-4a22-b94a-357bd8d6b5ab, session_id: 3b6d6787-7c4f-4800-a80e-7e17426e2512, gen: 1 } is now active (read-write)
Sep 22 23:14:27.938 INFO Scrub check for bcb46126-f008-485e-8ce7-38ca9e1e6dc2
Sep 22 23:14:27.938 INFO Scrub for bcb46126-f008-485e-8ce7-38ca9e1e6dc2 begins
Sep 22 23:14:27.938 INFO UpstairsConnection { upstairs_id: f94aa62c-b9ca-4a22-b94a-357bd8d6b5ab, session_id: 3b6d6787-7c4f-4800-a80e-7e17426e2512, gen: 1 } is now active (read-write)
Sep 22 23:14:27.938 INFO Scrub with total_size:5120 block_size:512
Sep 22 23:14:27.938 INFO Scrubs from block 0 to 10 in (256) 131072 size IOs pm:0
Sep 22 23:14:27.938 INFO Adjust block_count to 10 at offset 0
Sep 22 23:14:27.938 INFO UpstairsConnection { upstairs_id: f94aa62c-b9ca-4a22-b94a-357bd8d6b5ab, session_id: 3b6d6787-7c4f-4800-a80e-7e17426e2512, gen: 1 } is now active (read-write)
Sep 22 23:14:27.939 INFO [0] downstairs client at 127.0.0.1:62077 has UUID da4b10d8-d31d-4fd9-811a-957d3b103709
Sep 22 23:14:27.939 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: da4b10d8-d31d-4fd9-811a-957d3b103709, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.939 INFO f94aa62c-b9ca-4a22-b94a-357bd8d6b5ab WaitActive WaitActive WaitActive
Sep 22 23:14:27.939 INFO [1] downstairs client at 127.0.0.1:55380 has UUID a11fec66-3ce4-4c3c-bad2-f8da3b3c9b3e
Sep 22 23:14:27.939 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: a11fec66-3ce4-4c3c-bad2-f8da3b3c9b3e, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.939 INFO f94aa62c-b9ca-4a22-b94a-357bd8d6b5ab WaitActive WaitActive WaitActive
Sep 22 23:14:27.939 INFO [2] downstairs client at 127.0.0.1:41467 has UUID 7d1add5d-5a13-414b-bc02-b8d357908557
Sep 22 23:14:27.939 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 7d1add5d-5a13-414b-bc02-b8d357908557, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.939 INFO f94aa62c-b9ca-4a22-b94a-357bd8d6b5ab WaitActive WaitActive WaitActive
Sep 22 23:14:27.939 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.939 INFO UUID: a9656490-6146-4f17-b472-6f478edd5fc4
Sep 22 23:14:27.939 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.939 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.939 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.939 INFO Using address: 127.0.0.1:58389, task: main
Sep 22 23:14:27.939 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.940 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.940 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.940 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:57311, task: repair
Sep 22 23:14:27.940 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:57311, task: repair
Sep 22 23:14:27.940 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.940 DEBG Write     :1002 deps:[] res:true
Sep 22 23:14:27.940 INFO listening, local_addr: 127.0.0.1:57311, task: repair
Sep 22 23:14:27.940 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:57311, task: repair
Sep 22 23:14:27.940 INFO Using repair address: 127.0.0.1:57311, task: main
Sep 22 23:14:27.940 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.940 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.940 DEBG Write     :1002 deps:[] res:true
Sep 22 23:14:27.941 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.941 DEBG Write     :1002 deps:[] res:true
Sep 22 23:14:27.941 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.941 INFO [0] f94aa62c-b9ca-4a22-b94a-357bd8d6b5ab (3b6d6787-7c4f-4800-a80e-7e17426e2512) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.941 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.941 WARN c29f404c-9fcb-4abe-97d9-77d5448a344e request to replace downstairs 127.0.0.1:63499 with 127.0.0.1:58389
Sep 22 23:14:27.941 WARN [0] new RM replaced this: None
Sep 22 23:14:27.941 INFO c29f404c-9fcb-4abe-97d9-77d5448a344e found old target: 127.0.0.1:63499 at 0
Sep 22 23:14:27.941 INFO c29f404c-9fcb-4abe-97d9-77d5448a344e replacing old: 127.0.0.1:63499 at 0
Sep 22 23:14:27.941 INFO [0] Starts reconcile loop
Sep 22 23:14:27.941 INFO [0] client skip 0 in process jobs because fault, : downstairs
Sep 22 23:14:27.941 INFO [0] changed 0 jobs to fault skipped, : downstairs
Sep 22 23:14:27.941 INFO [0] c29f404c-9fcb-4abe-97d9-77d5448a344e (d5786959-64ba-4a45-85e5-b539511f4f59) Active Active Active ds_transition to Replacing
Sep 22 23:14:27.941 INFO [0] Transition from Active to Replacing
Sep 22 23:14:27.941 INFO [1] f94aa62c-b9ca-4a22-b94a-357bd8d6b5ab (3b6d6787-7c4f-4800-a80e-7e17426e2512) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.941 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.941 WARN [1] new RM replaced this: None
Sep 22 23:14:27.941 WARN c29f404c-9fcb-4abe-97d9-77d5448a344e request to replace downstairs 127.0.0.1:63499 with 127.0.0.1:58389
Sep 22 23:14:27.941 INFO [1] Starts reconcile loop
Sep 22 23:14:27.941 INFO c29f404c-9fcb-4abe-97d9-77d5448a344e found new target: 127.0.0.1:58389 at 0
Sep 22 23:14:27.941 INFO [2] f94aa62c-b9ca-4a22-b94a-357bd8d6b5ab (3b6d6787-7c4f-4800-a80e-7e17426e2512) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.941 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.941 WARN [2] new RM replaced this: None
Sep 22 23:14:27.941 INFO [2] Starts reconcile loop
Sep 22 23:14:27.941 INFO [0] 127.0.0.1:62077 task reports connection:true
Sep 22 23:14:27.941 INFO f94aa62c-b9ca-4a22-b94a-357bd8d6b5ab WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:14:27.941 INFO [0]R  flush_numbers: [0, 0]
Sep 22 23:14:27.941 INFO [0]R  generation: [0, 0]
Sep 22 23:14:27.941 INFO [0]R  dirty: [false, false]
Sep 22 23:14:27.941 INFO [1]R  flush_numbers: [0, 0]
Sep 22 23:14:27.941 INFO [1]R  generation: [0, 0]
Sep 22 23:14:27.941 INFO [1]R  dirty: [false, false]
Sep 22 23:14:27.941 INFO [2]R  flush_numbers: [0, 0]
Sep 22 23:14:27.941 INFO [2]R  generation: [0, 0]
Sep 22 23:14:27.941 INFO [2]R  dirty: [false, false]
Sep 22 23:14:27.941 INFO Max found gen is 1
Sep 22 23:14:27.941 INFO Generation requested: 1 >= found:1
Sep 22 23:14:27.941 INFO Next flush: 1
Sep 22 23:14:27.942 INFO All extents match
Sep 22 23:14:27.942 INFO No downstairs repair required
Sep 22 23:14:27.942 INFO No initial repair work was required
Sep 22 23:14:27.942 INFO Set Downstairs and Upstairs active
Sep 22 23:14:27.942 INFO f94aa62c-b9ca-4a22-b94a-357bd8d6b5ab is now active with session: 3b6d6787-7c4f-4800-a80e-7e17426e2512
Sep 22 23:14:27.942 INFO f94aa62c-b9ca-4a22-b94a-357bd8d6b5ab Set Active after no repair
Sep 22 23:14:27.942 INFO Notify all downstairs, region set compare is done.
Sep 22 23:14:27.942 INFO Set check for repair
Sep 22 23:14:27.942 INFO [1] 127.0.0.1:55380 task reports connection:true
Sep 22 23:14:27.942 INFO f94aa62c-b9ca-4a22-b94a-357bd8d6b5ab Active Active Active
Sep 22 23:14:27.942 INFO Set check for repair
Sep 22 23:14:27.942 INFO [2] 127.0.0.1:41467 task reports connection:true
Sep 22 23:14:27.942 INFO f94aa62c-b9ca-4a22-b94a-357bd8d6b5ab Active Active Active
Sep 22 23:14:27.942 INFO Set check for repair
Sep 22 23:14:27.942 INFO [0] received reconcile message
Sep 22 23:14:27.942 INFO [0] All repairs completed, exit
Sep 22 23:14:27.942 INFO [0] Starts cmd_loop
Sep 22 23:14:27.942 INFO [1] received reconcile message
Sep 22 23:14:27.942 INFO [1] All repairs completed, exit
Sep 22 23:14:27.942 INFO [1] Starts cmd_loop
Sep 22 23:14:27.942 INFO [2] received reconcile message
Sep 22 23:14:27.942 INFO [2] All repairs completed, exit
Sep 22 23:14:27.942 INFO [2] Starts cmd_loop
The guest has finished waiting for activation
Sep 22 23:14:27.943 DEBG IO Write 1003 has deps [JobId(1002), JobId(1001), JobId(1000)]
The guest has requested activation
Sep 22 23:14:27.943 INFO e61fa5a6-7bbf-4a27-8d67-7620a71d6c63 active request set
Sep 22 23:14:27.943 INFO [0] received activate with gen 1
Sep 22 23:14:27.943 INFO [0] client got ds_active_rx, promote! session 2024cbe9-f560-42ab-89c7-6b05d6eff703
Sep 22 23:14:27.943 INFO [1] received activate with gen 1
Sep 22 23:14:27.943 INFO [1] client got ds_active_rx, promote! session 2024cbe9-f560-42ab-89c7-6b05d6eff703
Sep 22 23:14:27.943 INFO [2] received activate with gen 1
Sep 22 23:14:27.943 INFO [2] client got ds_active_rx, promote! session 2024cbe9-f560-42ab-89c7-6b05d6eff703
Sep 22 23:14:27.943 INFO UpstairsConnection { upstairs_id: e61fa5a6-7bbf-4a27-8d67-7620a71d6c63, session_id: 2024cbe9-f560-42ab-89c7-6b05d6eff703, gen: 1 } is now active (read-write)
Sep 22 23:14:27.943 INFO UpstairsConnection { upstairs_id: e61fa5a6-7bbf-4a27-8d67-7620a71d6c63, session_id: 2024cbe9-f560-42ab-89c7-6b05d6eff703, gen: 1 } is now active (read-write)
Sep 22 23:14:27.943 INFO UpstairsConnection { upstairs_id: e61fa5a6-7bbf-4a27-8d67-7620a71d6c63, session_id: 2024cbe9-f560-42ab-89c7-6b05d6eff703, gen: 1 } is now active (read-write)
Sep 22 23:14:27.944 INFO [0] downstairs client at 127.0.0.1:35764 has UUID 27e21158-4dda-4f80-89f0-fd433654b40d
Sep 22 23:14:27.944 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 27e21158-4dda-4f80-89f0-fd433654b40d, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.944 INFO e61fa5a6-7bbf-4a27-8d67-7620a71d6c63 WaitActive WaitActive WaitActive
Sep 22 23:14:27.944 INFO [1] downstairs client at 127.0.0.1:43569 has UUID c1d22958-e424-406b-a939-8b7b5f9f6177
Sep 22 23:14:27.944 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: c1d22958-e424-406b-a939-8b7b5f9f6177, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.944 INFO e61fa5a6-7bbf-4a27-8d67-7620a71d6c63 WaitActive WaitActive WaitActive
Sep 22 23:14:27.944 INFO [2] downstairs client at 127.0.0.1:41379 has UUID 484c6303-bd26-472d-a8db-93e975b34d58
Sep 22 23:14:27.944 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 484c6303-bd26-472d-a8db-93e975b34d58, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.944 INFO e61fa5a6-7bbf-4a27-8d67-7620a71d6c63 WaitActive WaitActive WaitActive
Sep 22 23:14:27.944 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.944 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.945 INFO Current flush_numbers [0..12]: [0, 0]
test test::integration_test_volume_twice_replace_downstairs ... ok
Sep 22 23:14:27.945 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.945 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.945 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.945 INFO Created new region file "/tmp/downstairs-wIUBV7jK/region.json"
Sep 22 23:14:27.945 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.945 INFO [0] e61fa5a6-7bbf-4a27-8d67-7620a71d6c63 (2024cbe9-f560-42ab-89c7-6b05d6eff703) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.945 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.945 WARN [0] new RM replaced this: None
Sep 22 23:14:27.945 INFO [0] Starts reconcile loop
Sep 22 23:14:27.945 INFO [1] e61fa5a6-7bbf-4a27-8d67-7620a71d6c63 (2024cbe9-f560-42ab-89c7-6b05d6eff703) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.945 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.945 WARN [1] new RM replaced this: None
Sep 22 23:14:27.945 INFO [1] Starts reconcile loop
Sep 22 23:14:27.945 INFO [2] e61fa5a6-7bbf-4a27-8d67-7620a71d6c63 (2024cbe9-f560-42ab-89c7-6b05d6eff703) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.945 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.945 WARN [2] new RM replaced this: None
Sep 22 23:14:27.945 INFO [2] Starts reconcile loop
Sep 22 23:14:27.945 INFO [0] 127.0.0.1:35764 task reports connection:true
Sep 22 23:14:27.945 INFO e61fa5a6-7bbf-4a27-8d67-7620a71d6c63 WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:14:27.946 INFO [0]R  flush_numbers: [0, 0]
Sep 22 23:14:27.946 INFO [0]R  generation: [0, 0]
Sep 22 23:14:27.946 INFO [0]R  dirty: [false, false]
Sep 22 23:14:27.946 INFO [1]R  flush_numbers: [0, 0]
Sep 22 23:14:27.946 INFO [1]R  generation: [0, 0]
Sep 22 23:14:27.946 INFO [1]R  dirty: [false, false]
Sep 22 23:14:27.946 INFO [2]R  flush_numbers: [0, 0]
Sep 22 23:14:27.946 INFO [2]R  generation: [0, 0]
Sep 22 23:14:27.946 INFO [2]R  dirty: [false, false]
Sep 22 23:14:27.946 INFO Max found gen is 1
Sep 22 23:14:27.946 INFO Generation requested: 1 >= found:1
Sep 22 23:14:27.946 INFO Next flush: 1
Sep 22 23:14:27.946 INFO All extents match
Sep 22 23:14:27.946 INFO No downstairs repair required
Sep 22 23:14:27.946 INFO No initial repair work was required
Sep 22 23:14:27.946 INFO Set Downstairs and Upstairs active
Sep 22 23:14:27.946 INFO e61fa5a6-7bbf-4a27-8d67-7620a71d6c63 is now active with session: 2024cbe9-f560-42ab-89c7-6b05d6eff703
Sep 22 23:14:27.946 INFO e61fa5a6-7bbf-4a27-8d67-7620a71d6c63 Set Active after no repair
Sep 22 23:14:27.946 INFO Notify all downstairs, region set compare is done.
Sep 22 23:14:27.946 INFO Set check for repair
Sep 22 23:14:27.946 INFO [1] 127.0.0.1:43569 task reports connection:true
Sep 22 23:14:27.946 INFO e61fa5a6-7bbf-4a27-8d67-7620a71d6c63 Active Active Active
Sep 22 23:14:27.946 INFO Set check for repair
Sep 22 23:14:27.946 INFO [2] 127.0.0.1:41379 task reports connection:true
Sep 22 23:14:27.946 INFO e61fa5a6-7bbf-4a27-8d67-7620a71d6c63 Active Active Active
Sep 22 23:14:27.946 INFO Set check for repair
Sep 22 23:14:27.946 INFO [0] received reconcile message
Sep 22 23:14:27.946 INFO [0] All repairs completed, exit
Sep 22 23:14:27.946 INFO [0] Starts cmd_loop
Sep 22 23:14:27.946 INFO [1] received reconcile message
Sep 22 23:14:27.946 INFO [1] All repairs completed, exit
Sep 22 23:14:27.946 INFO [1] Starts cmd_loop
Sep 22 23:14:27.946 INFO [2] received reconcile message
Sep 22 23:14:27.946 INFO [2] All repairs completed, exit
Sep 22 23:14:27.946 INFO [2] Starts cmd_loop
The guest has finished waiting for activation
Sep 22 23:14:27.946 DEBG up_ds_listen was notified
Sep 22 23:14:27.946 DEBG up_ds_listen process 1003
Sep 22 23:14:27.946 DEBG [A] ack job 1003:4, : downstairs
Sep 22 23:14:27.946 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.946 INFO Scrub at offset 10/10 sp:10
Sep 22 23:14:27.947 DEBG IO Write 1000 has deps []
Sep 22 23:14:27.947 DEBG up_ds_listen was notified
Sep 22 23:14:27.947 DEBG up_ds_listen process 1000
Sep 22 23:14:27.947 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:14:27.947 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.948 INFO Scrub bcb46126-f008-485e-8ce7-38ca9e1e6dc2 done in 0 seconds. Retries:0 scrub_size:5120 size:10 pause_milli:0
Sep 22 23:14:27.948 DEBG Write     :1000 deps:[] res:true
Sep 22 23:14:27.948 DEBG IO Flush 1004 has deps [JobId(1003), JobId(1002), JobId(1001), JobId(1000)]
Sep 22 23:14:27.948 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.948 INFO Opened existing region file "/tmp/downstairs-wIUBV7jK/region.json"
Sep 22 23:14:27.948 INFO Database read version 1
Sep 22 23:14:27.948 INFO Database write version 1
Sep 22 23:14:27.948 DEBG Write     :1000 deps:[] res:true
Sep 22 23:14:27.949 DEBG Write     :1000 deps:[] res:true
Sep 22 23:14:27.949 DEBG IO Write 1000 has deps []
Sep 22 23:14:27.949 DEBG up_ds_listen was notified
Sep 22 23:14:27.949 DEBG up_ds_listen process 1000
Sep 22 23:14:27.949 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:14:27.949 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.950 DEBG Flush     :1004 extent_limit None deps:[JobId(1003), JobId(1002), JobId(1001), JobId(1000)] res:true f:1 g:1
Sep 22 23:14:27.950 DEBG Flush     :1004 extent_limit None deps:[JobId(1003), JobId(1002), JobId(1001), JobId(1000)] res:true f:1 g:1
Sep 22 23:14:27.950 DEBG Flush     :1004 extent_limit None deps:[JobId(1003), JobId(1002), JobId(1001), JobId(1000)] res:true f:1 g:1
Sep 22 23:14:27.950 INFO UUID: 3b4bb3c0-2112-401e-871c-3ba820d1a25e
Sep 22 23:14:27.950 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.950 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.950 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.950 INFO Using address: 127.0.0.1:56875, task: main
Sep 22 23:14:27.950 DEBG up_ds_listen was notified
Sep 22 23:14:27.950 DEBG Write     :1000 deps:[] res:true
Sep 22 23:14:27.950 DEBG up_ds_listen process 1004
Sep 22 23:14:27.950 DEBG [A] ack job 1004:5, : downstairs
Sep 22 23:14:27.950 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.950 DEBG [rc] retire 1004 clears [JobId(1000), JobId(1001), JobId(1002), JobId(1003), JobId(1004)], : downstairs
Sep 22 23:14:27.950 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.950 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:46993, task: repair
Sep 22 23:14:27.950 DEBG Write     :1000 deps:[] res:true
Sep 22 23:14:27.950 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:46993, task: repair
Sep 22 23:14:27.950 DEBG IO Flush 1002 has deps [JobId(1001), JobId(1000)]
Sep 22 23:14:27.950 INFO listening, local_addr: 127.0.0.1:46993, task: repair
Sep 22 23:14:27.951 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:46993, task: repair
Sep 22 23:14:27.951 INFO Using repair address: 127.0.0.1:46993, task: main
Sep 22 23:14:27.951 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.951 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.951 INFO Created new region file "/tmp/downstairs-Qg6sZndH/region.json"
Sep 22 23:14:27.951 DEBG Write     :1000 deps:[] res:true
Sep 22 23:14:27.951 DEBG IO Read  1001 has deps [JobId(1000)]
Sep 22 23:14:27.951 DEBG Flush     :1002 extent_limit None deps:[JobId(1001), JobId(1000)] res:true f:1 g:1
Sep 22 23:14:27.952 DEBG Flush     :1002 extent_limit None deps:[JobId(1001), JobId(1000)] res:true f:1 g:1
Sep 22 23:14:27.952 DEBG Flush     :1002 extent_limit None deps:[JobId(1001), JobId(1000)] res:true f:1 g:1
Sep 22 23:14:27.952 DEBG up_ds_listen was notified
Sep 22 23:14:27.952 DEBG up_ds_listen process 1002
Sep 22 23:14:27.952 DEBG [A] ack job 1002:3, : downstairs
Sep 22 23:14:27.952 DEBG [rc] retire 1002 clears [JobId(1000), JobId(1001), JobId(1002)], : downstairs
Sep 22 23:14:27.952 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.952 DEBG Read      :1001 deps:[JobId(1000)] res:true
Sep 22 23:14:27.952 DEBG IO Read  1005 has deps []
Sep 22 23:14:27.952 DEBG Read      :1001 deps:[JobId(1000)] res:true
Sep 22 23:14:27.952 DEBG Read      :1001 deps:[JobId(1000)] res:true
Sep 22 23:14:27.953 DEBG Read      :1005 deps:[] res:true
Sep 22 23:14:27.953 DEBG Read      :1005 deps:[] res:true
Sep 22 23:14:27.953 DEBG [0] Read AckReady 1001, : downstairs
Sep 22 23:14:27.953 DEBG Read      :1005 deps:[] res:true
Sep 22 23:14:27.954 DEBG [1] Read already AckReady 1001, : downstairs
Sep 22 23:14:27.954 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.954 INFO Opened existing region file "/tmp/downstairs-Qg6sZndH/region.json"
Sep 22 23:14:27.954 INFO Database read version 1
Sep 22 23:14:27.954 INFO Database write version 1
Sep 22 23:14:27.954 DEBG [2] Read already AckReady 1001, : downstairs
Sep 22 23:14:27.954 DEBG up_ds_listen was notified
Sep 22 23:14:27.954 DEBG up_ds_listen process 1001
Sep 22 23:14:27.954 DEBG [A] ack job 1001:2, : downstairs
Sep 22 23:14:27.954 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.955 DEBG IO Read  1001 has deps [JobId(1000)]
Sep 22 23:14:27.955 DEBG Read      :1001 deps:[JobId(1000)] res:true
Sep 22 23:14:27.956 DEBG Read      :1001 deps:[JobId(1000)] res:true
Sep 22 23:14:27.956 INFO UUID: e2279262-aa71-4426-b934-e18abcaf8d84
Sep 22 23:14:27.956 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.956 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.956 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.956 INFO Using address: 127.0.0.1:59924, task: main
Sep 22 23:14:27.956 DEBG Read      :1001 deps:[JobId(1000)] res:true
Sep 22 23:14:27.956 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.956 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:58902, task: repair
Sep 22 23:14:27.956 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:58902, task: repair
Sep 22 23:14:27.956 INFO listening, local_addr: 127.0.0.1:58902, task: repair
Sep 22 23:14:27.956 DEBG [0] Read AckReady 1005, : downstairs
Sep 22 23:14:27.957 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:58902, task: repair
Sep 22 23:14:27.957 INFO Using repair address: 127.0.0.1:58902, task: main
Sep 22 23:14:27.957 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.957 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.957 DEBG [0] Read AckReady 1001, : downstairs
Sep 22 23:14:27.957 INFO Created new region file "/tmp/downstairs-gHHLudLJ/region.json"
Sep 22 23:14:27.957 DEBG [1] Read already AckReady 1001, : downstairs
Sep 22 23:14:27.957 DEBG [2] Read already AckReady 1001, : downstairs
Sep 22 23:14:27.957 DEBG up_ds_listen was notified
Sep 22 23:14:27.958 DEBG up_ds_listen process 1001
Sep 22 23:14:27.958 DEBG [A] ack job 1001:2, : downstairs
Sep 22 23:14:27.958 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.958 DEBG [1] Read already AckReady 1005, : downstairs
Sep 22 23:14:27.959 DEBG IO Write 1002 has deps []
Sep 22 23:14:27.959 DEBG up_ds_listen was notified
Sep 22 23:14:27.959 DEBG up_ds_listen process 1002
Sep 22 23:14:27.959 DEBG [A] ack job 1002:3, : downstairs
Sep 22 23:14:27.959 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.959 INFO Connection request from da9f63ff-0d34-4427-ba79-865cdebab321 with version 4, task: proc
Sep 22 23:14:27.959 INFO upstairs UpstairsConnection { upstairs_id: da9f63ff-0d34-4427-ba79-865cdebab321, session_id: 5502d8a4-3689-41e6-8dea-ec66f52f7613, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.959 INFO Connection request from da9f63ff-0d34-4427-ba79-865cdebab321 with version 4, task: proc
Sep 22 23:14:27.959 INFO upstairs UpstairsConnection { upstairs_id: da9f63ff-0d34-4427-ba79-865cdebab321, session_id: 5502d8a4-3689-41e6-8dea-ec66f52f7613, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.959 INFO Connection request from da9f63ff-0d34-4427-ba79-865cdebab321 with version 4, task: proc
Sep 22 23:14:27.959 INFO upstairs UpstairsConnection { upstairs_id: da9f63ff-0d34-4427-ba79-865cdebab321, session_id: 5502d8a4-3689-41e6-8dea-ec66f52f7613, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.960 INFO [1] da9f63ff-0d34-4427-ba79-865cdebab321 (5502d8a4-3689-41e6-8dea-ec66f52f7613) New New New ds_transition to WaitActive
Sep 22 23:14:27.960 INFO [1] Transition from New to WaitActive
Sep 22 23:14:27.960 DEBG [2] Read already AckReady 1005, : downstairs
Sep 22 23:14:27.960 INFO [0] da9f63ff-0d34-4427-ba79-865cdebab321 (5502d8a4-3689-41e6-8dea-ec66f52f7613) New WaitActive New ds_transition to WaitActive
Sep 22 23:14:27.960 DEBG up_ds_listen was notified
Sep 22 23:14:27.960 INFO [0] Transition from New to WaitActive
Sep 22 23:14:27.960 DEBG up_ds_listen process 1005
Sep 22 23:14:27.960 DEBG [A] ack job 1005:6, : downstairs
Sep 22 23:14:27.960 INFO [2] da9f63ff-0d34-4427-ba79-865cdebab321 (5502d8a4-3689-41e6-8dea-ec66f52f7613) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:14:27.960 INFO [2] Transition from New to WaitActive
Sep 22 23:14:27.960 DEBG Write     :1002 deps:[] res:true
Sep 22 23:14:27.960 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.960 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.960 INFO Opened existing region file "/tmp/downstairs-gHHLudLJ/region.json"
Sep 22 23:14:27.960 INFO Database read version 1
Sep 22 23:14:27.960 INFO Database write version 1
Sep 22 23:14:27.961 DEBG IO Read  1003 has deps []
Sep 22 23:14:27.961 DEBG Write     :1002 deps:[] res:true
Sep 22 23:14:27.961 DEBG Read      :1003 deps:[] res:true
Sep 22 23:14:27.961 DEBG Write     :1002 deps:[] res:true
The guest has requested activation
Sep 22 23:14:27.962 INFO da9f63ff-0d34-4427-ba79-865cdebab321 active request set
Sep 22 23:14:27.962 DEBG IO Write 1002 has deps []
Sep 22 23:14:27.962 INFO [0] received activate with gen 1
Sep 22 23:14:27.962 DEBG Read      :1003 deps:[] res:true
Sep 22 23:14:27.962 INFO [0] client got ds_active_rx, promote! session 5502d8a4-3689-41e6-8dea-ec66f52f7613
Sep 22 23:14:27.962 DEBG up_ds_listen was notified
Sep 22 23:14:27.962 INFO [1] received activate with gen 1
Sep 22 23:14:27.962 INFO [1] client got ds_active_rx, promote! session 5502d8a4-3689-41e6-8dea-ec66f52f7613
Sep 22 23:14:27.962 DEBG up_ds_listen process 1002
Sep 22 23:14:27.962 DEBG [A] ack job 1002:3, : downstairs
Sep 22 23:14:27.962 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.962 INFO [2] received activate with gen 1
Sep 22 23:14:27.962 INFO [2] client got ds_active_rx, promote! session 5502d8a4-3689-41e6-8dea-ec66f52f7613
Sep 22 23:14:27.962 DEBG Read      :1003 deps:[] res:true
Sep 22 23:14:27.962 INFO UpstairsConnection { upstairs_id: da9f63ff-0d34-4427-ba79-865cdebab321, session_id: 5502d8a4-3689-41e6-8dea-ec66f52f7613, gen: 1 } is now active (read-write)
Sep 22 23:14:27.962 INFO Scrub check for 22457d3a-4c86-4e03-b495-3e59aee64613
Sep 22 23:14:27.962 INFO UpstairsConnection { upstairs_id: da9f63ff-0d34-4427-ba79-865cdebab321, session_id: 5502d8a4-3689-41e6-8dea-ec66f52f7613, gen: 1 } is now active (read-write)
Sep 22 23:14:27.962 INFO Scrub for 22457d3a-4c86-4e03-b495-3e59aee64613 begins
Sep 22 23:14:27.962 INFO Scrub with total_size:7680 block_size:512
Sep 22 23:14:27.962 INFO Scrubs from block 0 to 15 in (256) 131072 size IOs pm:0
Sep 22 23:14:27.962 INFO Adjust block_count to 15 at offset 0
Sep 22 23:14:27.962 INFO UpstairsConnection { upstairs_id: da9f63ff-0d34-4427-ba79-865cdebab321, session_id: 5502d8a4-3689-41e6-8dea-ec66f52f7613, gen: 1 } is now active (read-write)
Sep 22 23:14:27.963 INFO [1] downstairs client at 127.0.0.1:46036 has UUID 5d23d6e0-25c6-4cbe-ad9c-d04399dce2ff
Sep 22 23:14:27.963 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 5d23d6e0-25c6-4cbe-ad9c-d04399dce2ff, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.963 INFO da9f63ff-0d34-4427-ba79-865cdebab321 WaitActive WaitActive WaitActive
Sep 22 23:14:27.963 INFO [0] downstairs client at 127.0.0.1:39253 has UUID 231089c9-45a4-4798-b0b3-3fef94d32b3b
Sep 22 23:14:27.963 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 231089c9-45a4-4798-b0b3-3fef94d32b3b, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.963 DEBG [0] Read AckReady 1003, : downstairs
Sep 22 23:14:27.963 INFO da9f63ff-0d34-4427-ba79-865cdebab321 WaitActive WaitActive WaitActive
Sep 22 23:14:27.963 INFO [2] downstairs client at 127.0.0.1:37718 has UUID e165f739-3242-442c-822a-5ed698df44b4
Sep 22 23:14:27.963 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: e165f739-3242-442c-822a-5ed698df44b4, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.963 INFO da9f63ff-0d34-4427-ba79-865cdebab321 WaitActive WaitActive WaitActive
Sep 22 23:14:27.963 INFO UUID: a97b4e0c-86a6-4238-838d-e198926e4567
Sep 22 23:14:27.963 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.963 DEBG [1] Read already AckReady 1003, : downstairs
Sep 22 23:14:27.963 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.963 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.964 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.964 INFO Using address: 127.0.0.1:44361, task: main
Sep 22 23:14:27.964 DEBG [2] Read already AckReady 1003, : downstairs
Sep 22 23:14:27.964 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.964 DEBG up_ds_listen was notified
Sep 22 23:14:27.964 DEBG up_ds_listen process 1003
Sep 22 23:14:27.964 DEBG [A] ack job 1003:4, : downstairs
Sep 22 23:14:27.964 DEBG Write     :1002 deps:[] res:true
Sep 22 23:14:27.964 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.964 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.964 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:40957, task: repair
Sep 22 23:14:27.964 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:40957, task: repair
Sep 22 23:14:27.964 INFO listening, local_addr: 127.0.0.1:40957, task: repair
Sep 22 23:14:27.964 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.964 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.964 DEBG Write     :1002 deps:[] res:true
Sep 22 23:14:27.964 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:40957, task: repair
Sep 22 23:14:27.964 INFO Using repair address: 127.0.0.1:40957, task: main
Sep 22 23:14:27.964 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.964 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.965 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.965 DEBG Write     :1002 deps:[] res:true
Sep 22 23:14:27.965 INFO [1] da9f63ff-0d34-4427-ba79-865cdebab321 (5502d8a4-3689-41e6-8dea-ec66f52f7613) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.965 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.965 WARN [1] new RM replaced this: None
Sep 22 23:14:27.965 INFO [1] Starts reconcile loop
Sep 22 23:14:27.965 INFO Upstairs starts
Sep 22 23:14:27.965 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:14:27.965 INFO [0] da9f63ff-0d34-4427-ba79-865cdebab321 (5502d8a4-3689-41e6-8dea-ec66f52f7613) WaitActive WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.965 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:14:27.965 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.965 WARN [0] new RM replaced this: None
Sep 22 23:14:27.965 INFO Crucible stats registered with UUID: 6759ea0c-86e6-48d3-9651-28d14390b7c1
Sep 22 23:14:27.965 INFO [0] Starts reconcile loop
Sep 22 23:14:27.965 INFO Crucible 6759ea0c-86e6-48d3-9651-28d14390b7c1 has session id: 35b1cde4-1665-4791-808b-231481a45998
Sep 22 23:14:27.965 INFO [2] da9f63ff-0d34-4427-ba79-865cdebab321 (5502d8a4-3689-41e6-8dea-ec66f52f7613) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.965 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.965 WARN [2] new RM replaced this: None
Sep 22 23:14:27.965 INFO [2] Starts reconcile loop
Sep 22 23:14:27.965 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.965 INFO [1] 127.0.0.1:46036 task reports connection:true
Sep 22 23:14:27.965 INFO da9f63ff-0d34-4427-ba79-865cdebab321 WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:14:27.965 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.965 INFO [0]R  flush_numbers: [0, 0]
Sep 22 23:14:27.965 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.965 INFO [0]R  generation: [0, 0]
Sep 22 23:14:27.965 INFO [0]R  dirty: [false, false]
Sep 22 23:14:27.965 INFO [1]R  flush_numbers: [0, 0]
Sep 22 23:14:27.965 INFO [1]R  generation: [0, 0]
Sep 22 23:14:27.965 INFO [0] connecting to 127.0.0.1:56875, looper: 0
Sep 22 23:14:27.965 INFO [1]R  dirty: [false, false]
Sep 22 23:14:27.965 INFO [2]R  flush_numbers: [0, 0]
Sep 22 23:14:27.965 INFO [2]R  generation: [0, 0]
Sep 22 23:14:27.965 INFO [2]R  dirty: [false, false]
Sep 22 23:14:27.966 INFO Max found gen is 1
Sep 22 23:14:27.966 INFO Generation requested: 1 >= found:1
Sep 22 23:14:27.966 INFO Next flush: 1
Sep 22 23:14:27.966 INFO All extents match
Sep 22 23:14:27.966 INFO No downstairs repair required
Sep 22 23:14:27.966 INFO No initial repair work was required
Sep 22 23:14:27.966 INFO Set Downstairs and Upstairs active
Sep 22 23:14:27.966 INFO [1] connecting to 127.0.0.1:59924, looper: 1
Sep 22 23:14:27.966 INFO da9f63ff-0d34-4427-ba79-865cdebab321 is now active with session: 5502d8a4-3689-41e6-8dea-ec66f52f7613
Sep 22 23:14:27.966 INFO da9f63ff-0d34-4427-ba79-865cdebab321 Set Active after no repair
Sep 22 23:14:27.966 INFO Notify all downstairs, region set compare is done.
Sep 22 23:14:27.966 INFO Set check for repair
Sep 22 23:14:27.966 INFO [0] 127.0.0.1:39253 task reports connection:true
Sep 22 23:14:27.966 INFO da9f63ff-0d34-4427-ba79-865cdebab321 Active Active Active
Sep 22 23:14:27.966 INFO Set check for repair
Sep 22 23:14:27.966 INFO [2] connecting to 127.0.0.1:44361, looper: 2
Sep 22 23:14:27.966 INFO [2] 127.0.0.1:37718 task reports connection:true
Sep 22 23:14:27.966 INFO da9f63ff-0d34-4427-ba79-865cdebab321 Active Active Active
Sep 22 23:14:27.966 INFO Set check for repair
Sep 22 23:14:27.966 INFO up_listen starts, task: up_listen
Sep 22 23:14:27.966 INFO [0] received reconcile message
Sep 22 23:14:27.966 INFO Wait for all three downstairs to come online
Sep 22 23:14:27.966 INFO [0] All repairs completed, exit
Sep 22 23:14:27.966 INFO Flush timeout: 0.5
Sep 22 23:14:27.966 INFO [0] Starts cmd_loop
Sep 22 23:14:27.966 INFO [1] received reconcile message
Sep 22 23:14:27.966 INFO [1] All repairs completed, exit
Sep 22 23:14:27.966 INFO [1] Starts cmd_loop
Sep 22 23:14:27.966 INFO [0] 6759ea0c-86e6-48d3-9651-28d14390b7c1 looper connected, looper: 0
Sep 22 23:14:27.966 INFO [2] received reconcile message
Sep 22 23:14:27.966 INFO [2] All repairs completed, exit
Sep 22 23:14:27.966 INFO [0] Proc runs for 127.0.0.1:56875 in state New
Sep 22 23:14:27.966 INFO [2] Starts cmd_loop
The guest has finished waiting for activation
Sep 22 23:14:27.966 INFO [1] 6759ea0c-86e6-48d3-9651-28d14390b7c1 looper connected, looper: 1
Sep 22 23:14:27.966 INFO [1] Proc runs for 127.0.0.1:59924 in state New
Sep 22 23:14:27.966 INFO [2] 6759ea0c-86e6-48d3-9651-28d14390b7c1 looper connected, looper: 2
Sep 22 23:14:27.966 INFO [2] Proc runs for 127.0.0.1:44361 in state New
Sep 22 23:14:27.967 INFO accepted connection from 127.0.0.1:58618, task: main
Sep 22 23:14:27.967 DEBG IO Read  1000 has deps []
Sep 22 23:14:27.967 INFO accepted connection from 127.0.0.1:46006, task: main
Sep 22 23:14:27.967 INFO accepted connection from 127.0.0.1:42838, task: main
Sep 22 23:14:27.967 DEBG IO Write 1003 has deps [JobId(1002), JobId(1001), JobId(1000)]
Sep 22 23:14:27.967 INFO Connection request from 6759ea0c-86e6-48d3-9651-28d14390b7c1 with version 4, task: proc
Sep 22 23:14:27.967 INFO upstairs UpstairsConnection { upstairs_id: 6759ea0c-86e6-48d3-9651-28d14390b7c1, session_id: 64036d42-d0be-4120-9d55-87badff38ec8, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.967 INFO Connection request from 6759ea0c-86e6-48d3-9651-28d14390b7c1 with version 4, task: proc
Sep 22 23:14:27.967 INFO upstairs UpstairsConnection { upstairs_id: 6759ea0c-86e6-48d3-9651-28d14390b7c1, session_id: 64036d42-d0be-4120-9d55-87badff38ec8, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.967 INFO Connection request from 6759ea0c-86e6-48d3-9651-28d14390b7c1 with version 4, task: proc
Sep 22 23:14:27.967 INFO upstairs UpstairsConnection { upstairs_id: 6759ea0c-86e6-48d3-9651-28d14390b7c1, session_id: 64036d42-d0be-4120-9d55-87badff38ec8, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.967 DEBG Read      :1000 deps:[] res:true
Sep 22 23:14:27.968 INFO [0] 6759ea0c-86e6-48d3-9651-28d14390b7c1 (64036d42-d0be-4120-9d55-87badff38ec8) New New New ds_transition to WaitActive
Sep 22 23:14:27.968 INFO [0] Transition from New to WaitActive
Sep 22 23:14:27.968 DEBG Read      :1000 deps:[] res:true
Sep 22 23:14:27.968 INFO [1] 6759ea0c-86e6-48d3-9651-28d14390b7c1 (64036d42-d0be-4120-9d55-87badff38ec8) WaitActive New New ds_transition to WaitActive
Sep 22 23:14:27.968 INFO [1] Transition from New to WaitActive
Sep 22 23:14:27.968 INFO [2] 6759ea0c-86e6-48d3-9651-28d14390b7c1 (64036d42-d0be-4120-9d55-87badff38ec8) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:14:27.968 INFO [2] Transition from New to WaitActive
The guest has requested activation
Sep 22 23:14:27.968 DEBG Read      :1000 deps:[] res:true
Sep 22 23:14:27.968 INFO 6759ea0c-86e6-48d3-9651-28d14390b7c1 active request set
Sep 22 23:14:27.968 INFO [0] received activate with gen 1
Sep 22 23:14:27.968 INFO [0] client got ds_active_rx, promote! session 64036d42-d0be-4120-9d55-87badff38ec8
Sep 22 23:14:27.968 INFO [1] received activate with gen 1
Sep 22 23:14:27.968 INFO [1] client got ds_active_rx, promote! session 64036d42-d0be-4120-9d55-87badff38ec8
Sep 22 23:14:27.968 INFO [2] received activate with gen 1
Sep 22 23:14:27.968 INFO [2] client got ds_active_rx, promote! session 64036d42-d0be-4120-9d55-87badff38ec8
Sep 22 23:14:27.968 INFO UpstairsConnection { upstairs_id: 6759ea0c-86e6-48d3-9651-28d14390b7c1, session_id: 64036d42-d0be-4120-9d55-87badff38ec8, gen: 1 } is now active (read-write)
Sep 22 23:14:27.969 INFO UpstairsConnection { upstairs_id: 6759ea0c-86e6-48d3-9651-28d14390b7c1, session_id: 64036d42-d0be-4120-9d55-87badff38ec8, gen: 1 } is now active (read-write)
Sep 22 23:14:27.969 INFO UpstairsConnection { upstairs_id: 6759ea0c-86e6-48d3-9651-28d14390b7c1, session_id: 64036d42-d0be-4120-9d55-87badff38ec8, gen: 1 } is now active (read-write)
Sep 22 23:14:27.969 DEBG [1] Read AckReady 1000, : downstairs
Sep 22 23:14:27.969 DEBG [0] Read already AckReady 1000, : downstairs
Sep 22 23:14:27.969 DEBG [2] Read already AckReady 1000, : downstairs
Sep 22 23:14:27.969 DEBG up_ds_listen was notified
Sep 22 23:14:27.969 DEBG up_ds_listen process 1000
Sep 22 23:14:27.969 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:14:27.969 INFO [0] downstairs client at 127.0.0.1:56875 has UUID 3b4bb3c0-2112-401e-871c-3ba820d1a25e
Sep 22 23:14:27.969 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 3b4bb3c0-2112-401e-871c-3ba820d1a25e, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.969 INFO 6759ea0c-86e6-48d3-9651-28d14390b7c1 WaitActive WaitActive WaitActive
Sep 22 23:14:27.969 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.969 INFO [1] downstairs client at 127.0.0.1:59924 has UUID e2279262-aa71-4426-b934-e18abcaf8d84
Sep 22 23:14:27.969 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: e2279262-aa71-4426-b934-e18abcaf8d84, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.969 INFO 6759ea0c-86e6-48d3-9651-28d14390b7c1 WaitActive WaitActive WaitActive
Sep 22 23:14:27.969 INFO [2] downstairs client at 127.0.0.1:44361 has UUID a97b4e0c-86a6-4238-838d-e198926e4567
Sep 22 23:14:27.969 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: a97b4e0c-86a6-4238-838d-e198926e4567, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.969 INFO 6759ea0c-86e6-48d3-9651-28d14390b7c1 WaitActive WaitActive WaitActive
Sep 22 23:14:27.970 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.970 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.971 DEBG IO Write 1001 has deps [JobId(1000)]
Sep 22 23:14:27.971 DEBG up_ds_listen was notified
Sep 22 23:14:27.971 DEBG up_ds_listen process 1001
Sep 22 23:14:27.971 DEBG [A] ack job 1001:2, : downstairs
Sep 22 23:14:27.971 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.971 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.971 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.971 INFO Current flush_numbers [0..12]: [0, 0]
test test::integration_test_volume_subvols_parent_scrub_sparse ... ok
Sep 22 23:14:27.972 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.972 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.972 INFO [0] 6759ea0c-86e6-48d3-9651-28d14390b7c1 (64036d42-d0be-4120-9d55-87badff38ec8) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.972 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.972 WARN [0] new RM replaced this: None
Sep 22 23:14:27.972 INFO [0] Starts reconcile loop
Sep 22 23:14:27.972 INFO Created new region file "/tmp/downstairs-CjWA24Kr/region.json"
Sep 22 23:14:27.972 INFO [1] 6759ea0c-86e6-48d3-9651-28d14390b7c1 (64036d42-d0be-4120-9d55-87badff38ec8) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.972 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.972 WARN [1] new RM replaced this: None
Sep 22 23:14:27.972 INFO [1] Starts reconcile loop
Sep 22 23:14:27.972 DEBG Write     :1001 deps:[JobId(1000)] res:true
Sep 22 23:14:27.972 INFO [2] 6759ea0c-86e6-48d3-9651-28d14390b7c1 (64036d42-d0be-4120-9d55-87badff38ec8) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.972 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.972 WARN [2] new RM replaced this: None
Sep 22 23:14:27.972 INFO [2] Starts reconcile loop
Sep 22 23:14:27.972 DEBG up_ds_listen was notified
Sep 22 23:14:27.972 INFO [0] 127.0.0.1:56875 task reports connection:true
Sep 22 23:14:27.972 INFO 6759ea0c-86e6-48d3-9651-28d14390b7c1 WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:14:27.972 DEBG up_ds_listen process 1003
Sep 22 23:14:27.972 INFO [0]R  flush_numbers: [0, 0]
Sep 22 23:14:27.972 INFO [0]R  generation: [0, 0]
Sep 22 23:14:27.972 DEBG [A] ack job 1003:4, : downstairs
Sep 22 23:14:27.972 INFO [0]R  dirty: [false, false]
Sep 22 23:14:27.972 INFO [1]R  flush_numbers: [0, 0]
Sep 22 23:14:27.972 INFO [1]R  generation: [0, 0]
Sep 22 23:14:27.972 INFO [1]R  dirty: [false, false]
Sep 22 23:14:27.972 INFO [2]R  flush_numbers: [0, 0]
Sep 22 23:14:27.972 INFO [2]R  generation: [0, 0]
Sep 22 23:14:27.972 INFO [2]R  dirty: [false, false]
Sep 22 23:14:27.972 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.972 INFO Max found gen is 1
Sep 22 23:14:27.972 INFO Generation requested: 1 >= found:1
Sep 22 23:14:27.972 INFO Next flush: 1
Sep 22 23:14:27.972 INFO All extents match
Sep 22 23:14:27.972 INFO No downstairs repair required
Sep 22 23:14:27.972 INFO No initial repair work was required
Sep 22 23:14:27.972 INFO Set Downstairs and Upstairs active
Sep 22 23:14:27.972 INFO 6759ea0c-86e6-48d3-9651-28d14390b7c1 is now active with session: 64036d42-d0be-4120-9d55-87badff38ec8
Sep 22 23:14:27.972 DEBG Write     :1001 deps:[JobId(1000)] res:true
Sep 22 23:14:27.972 INFO 6759ea0c-86e6-48d3-9651-28d14390b7c1 Set Active after no repair
Sep 22 23:14:27.972 INFO Notify all downstairs, region set compare is done.
Sep 22 23:14:27.972 INFO Set check for repair
Sep 22 23:14:27.972 INFO [1] 127.0.0.1:59924 task reports connection:true
Sep 22 23:14:27.972 INFO 6759ea0c-86e6-48d3-9651-28d14390b7c1 Active Active Active
Sep 22 23:14:27.972 INFO Set check for repair
Sep 22 23:14:27.973 INFO [2] 127.0.0.1:44361 task reports connection:true
Sep 22 23:14:27.973 INFO 6759ea0c-86e6-48d3-9651-28d14390b7c1 Active Active Active
Sep 22 23:14:27.973 INFO Set check for repair
Sep 22 23:14:27.973 INFO [0] received reconcile message
Sep 22 23:14:27.973 INFO [0] All repairs completed, exit
Sep 22 23:14:27.973 INFO [0] Starts cmd_loop
Sep 22 23:14:27.973 INFO [1] received reconcile message
Sep 22 23:14:27.973 INFO [1] All repairs completed, exit
Sep 22 23:14:27.973 INFO [1] Starts cmd_loop
Sep 22 23:14:27.973 INFO [2] received reconcile message
Sep 22 23:14:27.973 DEBG Write     :1001 deps:[JobId(1000)] res:true
Sep 22 23:14:27.973 INFO [2] All repairs completed, exit
Sep 22 23:14:27.973 INFO [2] Starts cmd_loop
The guest has finished waiting for activation
Sep 22 23:14:27.973 DEBG IO Read  1002 has deps [JobId(1001)]
Sep 22 23:14:27.974 DEBG IO Write 1003 has deps [JobId(1002), JobId(1001), JobId(1000)]
Sep 22 23:14:27.974 DEBG Read      :1002 deps:[JobId(1001)] res:true
Sep 22 23:14:27.974 DEBG Read      :1002 deps:[JobId(1001)] res:true
Sep 22 23:14:27.974 DEBG Read      :1002 deps:[JobId(1001)] res:true
Sep 22 23:14:27.975 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.975 INFO Opened existing region file "/tmp/downstairs-CjWA24Kr/region.json"
Sep 22 23:14:27.975 INFO Database read version 1
Sep 22 23:14:27.975 DEBG IO Write 1000 has deps []
Sep 22 23:14:27.975 INFO Database write version 1
Sep 22 23:14:27.975 DEBG up_ds_listen was notified
Sep 22 23:14:27.975 DEBG up_ds_listen process 1000
Sep 22 23:14:27.975 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:14:27.975 DEBG [1] Read AckReady 1002, : downstairs
Sep 22 23:14:27.975 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.975 DEBG [0] Read already AckReady 1002, : downstairs
Sep 22 23:14:27.976 DEBG [2] Read already AckReady 1002, : downstairs
Sep 22 23:14:27.976 DEBG up_ds_listen was notified
Sep 22 23:14:27.976 DEBG up_ds_listen process 1002
Sep 22 23:14:27.976 DEBG [A] ack job 1002:3, : downstairs
Sep 22 23:14:27.976 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.976 DEBG up_ds_listen was notified
Sep 22 23:14:27.976 DEBG up_ds_listen process 1003
Sep 22 23:14:27.976 DEBG [A] ack job 1003:4, : downstairs
Sep 22 23:14:27.976 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.977 INFO Scrub at offset 15/15 sp:15
Sep 22 23:14:27.977 INFO UUID: 989c51b5-eaa4-4274-9bd1-0900d0b855e1
Sep 22 23:14:27.977 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.977 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.977 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.977 INFO Using address: 127.0.0.1:62045, task: main
Sep 22 23:14:27.977 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.977 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:44279, task: repair
Sep 22 23:14:27.977 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:44279, task: repair
Sep 22 23:14:27.977 INFO listening, local_addr: 127.0.0.1:44279, task: repair
Sep 22 23:14:27.977 DEBG Write     :1000 deps:[] res:true
Sep 22 23:14:27.977 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:44279, task: repair
Sep 22 23:14:27.978 INFO Using repair address: 127.0.0.1:44279, task: main
Sep 22 23:14:27.978 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.978 INFO Scrub 22457d3a-4c86-4e03-b495-3e59aee64613 done in 0 seconds. Retries:0 scrub_size:7680 size:15 pause_milli:0
Sep 22 23:14:27.978 DEBG IO Flush 1004 has deps [JobId(1003), JobId(1002), JobId(1001), JobId(1000)]
Sep 22 23:14:27.978 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.978 INFO Created new region file "/tmp/downstairs-rUNKYH5o/region.json"
Sep 22 23:14:27.978 DEBG Write     :1000 deps:[] res:true
Sep 22 23:14:27.979 DEBG Write     :1000 deps:[] res:true
Sep 22 23:14:27.979 DEBG IO Read  1001 has deps [JobId(1000)]
Sep 22 23:14:27.979 DEBG Flush     :1004 extent_limit None deps:[JobId(1003), JobId(1002), JobId(1001), JobId(1000)] res:true f:1 g:1
Sep 22 23:14:27.979 DEBG Flush     :1004 extent_limit None deps:[JobId(1003), JobId(1002), JobId(1001), JobId(1000)] res:true f:1 g:1
Sep 22 23:14:27.979 DEBG Flush     :1004 extent_limit None deps:[JobId(1003), JobId(1002), JobId(1001), JobId(1000)] res:true f:1 g:1
Sep 22 23:14:27.980 DEBG up_ds_listen was notified
Sep 22 23:14:27.980 DEBG up_ds_listen process 1004
Sep 22 23:14:27.980 DEBG [A] ack job 1004:5, : downstairs
Sep 22 23:14:27.980 DEBG [rc] retire 1004 clears [JobId(1000), JobId(1001), JobId(1002), JobId(1003), JobId(1004)], : downstairs
Sep 22 23:14:27.980 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.980 DEBG IO Flush 1004 has deps [JobId(1003), JobId(1002), JobId(1001), JobId(1000)]
Sep 22 23:14:27.980 DEBG Read      :1001 deps:[JobId(1000)] res:true
Sep 22 23:14:27.981 DEBG Read      :1001 deps:[JobId(1000)] res:true
Sep 22 23:14:27.982 DEBG Read      :1001 deps:[JobId(1000)] res:true
Sep 22 23:14:27.982 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.982 INFO Opened existing region file "/tmp/downstairs-rUNKYH5o/region.json"
Sep 22 23:14:27.982 INFO Database read version 1
Sep 22 23:14:27.982 INFO Database write version 1
Sep 22 23:14:27.982 DEBG Flush     :1004 extent_limit None deps:[JobId(1003), JobId(1002), JobId(1001), JobId(1000)] res:true f:1 g:1
Sep 22 23:14:27.982 DEBG Flush     :1004 extent_limit None deps:[JobId(1003), JobId(1002), JobId(1001), JobId(1000)] res:true f:1 g:1
Sep 22 23:14:27.982 DEBG Flush     :1004 extent_limit None deps:[JobId(1003), JobId(1002), JobId(1001), JobId(1000)] res:true f:1 g:1
Sep 22 23:14:27.983 DEBG up_ds_listen was notified
Sep 22 23:14:27.983 DEBG up_ds_listen process 1004
Sep 22 23:14:27.983 DEBG [A] ack job 1004:5, : downstairs
Sep 22 23:14:27.983 DEBG [rc] retire 1004 clears [JobId(1000), JobId(1001), JobId(1002), JobId(1003), JobId(1004)], : downstairs
Sep 22 23:14:27.983 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.983 DEBG IO Read  1005 has deps []
Sep 22 23:14:27.983 INFO UUID: 0dbe2b78-0264-4d8b-ad44-c519f3295e35
Sep 22 23:14:27.983 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.983 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
test test::integration_test_url ... ok
Sep 22 23:14:27.984 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.984 INFO Using address: 127.0.0.1:37528, task: main
Sep 22 23:14:27.984 DEBG Read      :1005 deps:[] res:true
Sep 22 23:14:27.984 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.984 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.984 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:57545, task: repair
Sep 22 23:14:27.984 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:57545, task: repair
Sep 22 23:14:27.984 INFO Created new region file "/tmp/downstairs-ILagCh4E/region.json"
Sep 22 23:14:27.984 INFO listening, local_addr: 127.0.0.1:57545, task: repair
Sep 22 23:14:27.984 DEBG Read      :1005 deps:[] res:true
Sep 22 23:14:27.984 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:57545, task: repair
Sep 22 23:14:27.984 INFO Using repair address: 127.0.0.1:57545, task: main
Sep 22 23:14:27.984 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.985 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.985 DEBG [0] Read AckReady 1001, : downstairs
Sep 22 23:14:27.985 DEBG Read      :1005 deps:[] res:true
Sep 22 23:14:27.985 INFO Created new region file "/tmp/downstairs-G1TA2uOj/region.json"
Sep 22 23:14:27.986 DEBG [1] Read already AckReady 1001, : downstairs
Sep 22 23:14:27.988 DEBG [0] Read AckReady 1005, : downstairs
Sep 22 23:14:27.988 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.988 INFO Opened existing region file "/tmp/downstairs-ILagCh4E/region.json"
Sep 22 23:14:27.988 INFO Database read version 1
Sep 22 23:14:27.988 INFO Database write version 1
Sep 22 23:14:27.988 DEBG [2] Read already AckReady 1001, : downstairs
Sep 22 23:14:27.988 DEBG up_ds_listen was notified
Sep 22 23:14:27.988 DEBG up_ds_listen process 1001
Sep 22 23:14:27.988 DEBG [A] ack job 1001:2, : downstairs
Sep 22 23:14:27.988 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.989 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.989 INFO Opened existing region file "/tmp/downstairs-G1TA2uOj/region.json"
Sep 22 23:14:27.989 INFO Database read version 1
Sep 22 23:14:27.989 INFO Database write version 1
Sep 22 23:14:27.990 DEBG [1] Read already AckReady 1005, : downstairs
Sep 22 23:14:27.990 DEBG IO Write 1002 has deps [JobId(1001), JobId(1000)]
Sep 22 23:14:27.991 INFO UUID: 177b9038-d4da-4321-bb6f-83528c5df2bb
Sep 22 23:14:27.991 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.991 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.991 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.991 INFO Using address: 127.0.0.1:53498, task: main
Sep 22 23:14:27.991 INFO UUID: 8d12fb73-3125-4add-9524-088d1d513bcf
Sep 22 23:14:27.991 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.991 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.991 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.991 INFO Using address: 127.0.0.1:59233, task: main
Sep 22 23:14:27.991 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.991 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:33162, task: repair
Sep 22 23:14:27.991 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:33162, task: repair
Sep 22 23:14:27.991 INFO listening, local_addr: 127.0.0.1:33162, task: repair
Sep 22 23:14:27.991 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.991 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:60052, task: repair
Sep 22 23:14:27.991 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:60052, task: repair
Sep 22 23:14:27.991 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:33162, task: repair
Sep 22 23:14:27.991 INFO listening, local_addr: 127.0.0.1:60052, task: repair
Sep 22 23:14:27.991 INFO Using repair address: 127.0.0.1:33162, task: main
Sep 22 23:14:27.991 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.992 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:60052, task: repair
Sep 22 23:14:27.992 INFO Using repair address: 127.0.0.1:60052, task: main
Sep 22 23:14:27.992 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.992 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.992 INFO Created new region file "/tmp/downstairs-8mPvIugn/region.json"
Sep 22 23:14:27.992 DEBG [2] Read already AckReady 1005, : downstairs
Sep 22 23:14:27.992 INFO Upstairs starts
Sep 22 23:14:27.992 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:14:27.992 DEBG up_ds_listen was notified
Sep 22 23:14:27.992 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:14:27.992 DEBG up_ds_listen process 1005
Sep 22 23:14:27.992 INFO Crucible stats registered with UUID: d294d936-1c70-474e-8080-a4dab42d6778
Sep 22 23:14:27.992 DEBG [A] ack job 1005:6, : downstairs
Sep 22 23:14:27.992 INFO Crucible d294d936-1c70-474e-8080-a4dab42d6778 has session id: 16d0bb5f-a970-4c72-8e04-70e4a93316bd
Sep 22 23:14:27.992 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.992 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.992 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:27.992 INFO [0] connecting to 127.0.0.1:62045, looper: 0
Sep 22 23:14:27.992 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.992 INFO [1] connecting to 127.0.0.1:37528, looper: 1
Sep 22 23:14:27.992 INFO [2] connecting to 127.0.0.1:59233, looper: 2
Sep 22 23:14:27.993 INFO up_listen starts, task: up_listen
Sep 22 23:14:27.993 INFO Wait for all three downstairs to come online
Sep 22 23:14:27.993 DEBG IO Read  1005 has deps []
Sep 22 23:14:27.993 INFO Flush timeout: 0.5
Sep 22 23:14:27.993 INFO accepted connection from 127.0.0.1:41831, task: main
Sep 22 23:14:27.993 INFO accepted connection from 127.0.0.1:54062, task: main
Sep 22 23:14:27.993 INFO [0] d294d936-1c70-474e-8080-a4dab42d6778 looper connected, looper: 0
Sep 22 23:14:27.993 INFO [0] Proc runs for 127.0.0.1:62045 in state New
Sep 22 23:14:27.993 INFO accepted connection from 127.0.0.1:57271, task: main
Sep 22 23:14:27.993 INFO [1] d294d936-1c70-474e-8080-a4dab42d6778 looper connected, looper: 1
Sep 22 23:14:27.993 INFO [1] Proc runs for 127.0.0.1:37528 in state New
Sep 22 23:14:27.993 INFO [2] d294d936-1c70-474e-8080-a4dab42d6778 looper connected, looper: 2
Sep 22 23:14:27.993 INFO [2] Proc runs for 127.0.0.1:59233 in state New
Sep 22 23:14:27.993 INFO Connection request from d294d936-1c70-474e-8080-a4dab42d6778 with version 4, task: proc
Sep 22 23:14:27.993 INFO upstairs UpstairsConnection { upstairs_id: d294d936-1c70-474e-8080-a4dab42d6778, session_id: eff1e9a9-bbb5-477e-acde-f6036995e381, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.993 INFO Connection request from d294d936-1c70-474e-8080-a4dab42d6778 with version 4, task: proc
Sep 22 23:14:27.993 INFO upstairs UpstairsConnection { upstairs_id: d294d936-1c70-474e-8080-a4dab42d6778, session_id: eff1e9a9-bbb5-477e-acde-f6036995e381, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.994 INFO Connection request from d294d936-1c70-474e-8080-a4dab42d6778 with version 4, task: proc
Sep 22 23:14:27.994 DEBG Read      :1005 deps:[] res:true
Sep 22 23:14:27.994 INFO upstairs UpstairsConnection { upstairs_id: d294d936-1c70-474e-8080-a4dab42d6778, session_id: eff1e9a9-bbb5-477e-acde-f6036995e381, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:27.994 DEBG up_ds_listen was notified
Sep 22 23:14:27.994 DEBG up_ds_listen process 1002
Sep 22 23:14:27.994 DEBG [A] ack job 1002:3, : downstairs
Sep 22 23:14:27.994 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:27.994 INFO [0] d294d936-1c70-474e-8080-a4dab42d6778 (eff1e9a9-bbb5-477e-acde-f6036995e381) New New New ds_transition to WaitActive
Sep 22 23:14:27.994 INFO [0] Transition from New to WaitActive
Sep 22 23:14:27.994 INFO [1] d294d936-1c70-474e-8080-a4dab42d6778 (eff1e9a9-bbb5-477e-acde-f6036995e381) WaitActive New New ds_transition to WaitActive
Sep 22 23:14:27.994 INFO [1] Transition from New to WaitActive
Sep 22 23:14:27.994 INFO [2] d294d936-1c70-474e-8080-a4dab42d6778 (eff1e9a9-bbb5-477e-acde-f6036995e381) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:14:27.994 INFO [2] Transition from New to WaitActive
Sep 22 23:14:27.994 DEBG Read      :1005 deps:[] res:true
Sep 22 23:14:27.994 DEBG IO Read  1003 has deps [JobId(1002), JobId(1000)]
The guest has requested activation
Sep 22 23:14:27.994 INFO d294d936-1c70-474e-8080-a4dab42d6778 active request set
Sep 22 23:14:27.994 INFO [0] received activate with gen 1
Sep 22 23:14:27.994 INFO [0] client got ds_active_rx, promote! session eff1e9a9-bbb5-477e-acde-f6036995e381
Sep 22 23:14:27.995 INFO [1] received activate with gen 1
Sep 22 23:14:27.995 INFO [1] client got ds_active_rx, promote! session eff1e9a9-bbb5-477e-acde-f6036995e381
Sep 22 23:14:27.995 DEBG Read      :1005 deps:[] res:true
Sep 22 23:14:27.995 INFO [2] received activate with gen 1
Sep 22 23:14:27.995 INFO [2] client got ds_active_rx, promote! session eff1e9a9-bbb5-477e-acde-f6036995e381
Sep 22 23:14:27.995 INFO UpstairsConnection { upstairs_id: d294d936-1c70-474e-8080-a4dab42d6778, session_id: eff1e9a9-bbb5-477e-acde-f6036995e381, gen: 1 } is now active (read-write)
Sep 22 23:14:27.995 INFO UpstairsConnection { upstairs_id: d294d936-1c70-474e-8080-a4dab42d6778, session_id: eff1e9a9-bbb5-477e-acde-f6036995e381, gen: 1 } is now active (read-write)
Sep 22 23:14:27.995 INFO UpstairsConnection { upstairs_id: d294d936-1c70-474e-8080-a4dab42d6778, session_id: eff1e9a9-bbb5-477e-acde-f6036995e381, gen: 1 } is now active (read-write)
Sep 22 23:14:27.995 DEBG Read      :1003 deps:[JobId(1002), JobId(1000)] res:true
Sep 22 23:14:27.995 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.995 INFO Opened existing region file "/tmp/downstairs-8mPvIugn/region.json"
Sep 22 23:14:27.995 INFO Database read version 1
Sep 22 23:14:27.995 INFO Database write version 1
Sep 22 23:14:27.995 INFO [0] downstairs client at 127.0.0.1:62045 has UUID 989c51b5-eaa4-4274-9bd1-0900d0b855e1
Sep 22 23:14:27.995 DEBG Read      :1003 deps:[JobId(1002), JobId(1000)] res:true
Sep 22 23:14:27.996 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 989c51b5-eaa4-4274-9bd1-0900d0b855e1, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.996 INFO d294d936-1c70-474e-8080-a4dab42d6778 WaitActive WaitActive WaitActive
Sep 22 23:14:27.996 INFO [1] downstairs client at 127.0.0.1:37528 has UUID 0dbe2b78-0264-4d8b-ad44-c519f3295e35
Sep 22 23:14:27.996 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 0dbe2b78-0264-4d8b-ad44-c519f3295e35, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.996 INFO d294d936-1c70-474e-8080-a4dab42d6778 WaitActive WaitActive WaitActive
Sep 22 23:14:27.996 INFO [2] downstairs client at 127.0.0.1:59233 has UUID 8d12fb73-3125-4add-9524-088d1d513bcf
Sep 22 23:14:27.996 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 8d12fb73-3125-4add-9524-088d1d513bcf, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:27.996 INFO d294d936-1c70-474e-8080-a4dab42d6778 WaitActive WaitActive WaitActive
Sep 22 23:14:27.996 DEBG Read      :1003 deps:[JobId(1002), JobId(1000)] res:true
Sep 22 23:14:27.996 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.996 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.997 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.997 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.997 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:27.997 INFO UUID: 7e66c791-8f61-4b2b-80e3-9850920f90c5
Sep 22 23:14:27.997 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:27.997 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:27.997 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:27.997 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:27.997 INFO Using address: 127.0.0.1:35121, task: main
Sep 22 23:14:27.997 INFO [0] d294d936-1c70-474e-8080-a4dab42d6778 (eff1e9a9-bbb5-477e-acde-f6036995e381) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.997 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.997 WARN [0] new RM replaced this: None
Sep 22 23:14:27.997 INFO [0] Starts reconcile loop
Sep 22 23:14:27.997 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:27.997 INFO [1] d294d936-1c70-474e-8080-a4dab42d6778 (eff1e9a9-bbb5-477e-acde-f6036995e381) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.997 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:33106, task: repair
Sep 22 23:14:27.997 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.997 WARN [1] new RM replaced this: None
Sep 22 23:14:27.997 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:33106, task: repair
Sep 22 23:14:27.998 INFO [1] Starts reconcile loop
Sep 22 23:14:27.998 INFO listening, local_addr: 127.0.0.1:33106, task: repair
Sep 22 23:14:27.998 INFO [2] d294d936-1c70-474e-8080-a4dab42d6778 (eff1e9a9-bbb5-477e-acde-f6036995e381) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:14:27.998 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:14:27.998 WARN [2] new RM replaced this: None
Sep 22 23:14:27.998 INFO [2] Starts reconcile loop
Sep 22 23:14:27.998 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:33106, task: repair
Sep 22 23:14:27.998 INFO Using repair address: 127.0.0.1:33106, task: main
Sep 22 23:14:27.998 INFO [0] 127.0.0.1:62045 task reports connection:true
Sep 22 23:14:27.998 INFO No SSL acceptor configured, task: main
Sep 22 23:14:27.998 INFO d294d936-1c70-474e-8080-a4dab42d6778 WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:14:27.998 INFO [0]R  flush_numbers: [0, 0]
Sep 22 23:14:27.998 INFO [0]R  generation: [0, 0]
Sep 22 23:14:27.998 DEBG [0] Read AckReady 1005, : downstairs
Sep 22 23:14:27.998 INFO [0]R  dirty: [false, false]
Sep 22 23:14:27.998 INFO [1]R  flush_numbers: [0, 0]
Sep 22 23:14:27.998 INFO [1]R  generation: [0, 0]
Sep 22 23:14:27.998 INFO [1]R  dirty: [false, false]
Sep 22 23:14:27.998 INFO [2]R  flush_numbers: [0, 0]
Sep 22 23:14:27.998 INFO [2]R  generation: [0, 0]
Sep 22 23:14:27.998 INFO [2]R  dirty: [false, false]
Sep 22 23:14:27.998 INFO Max found gen is 1
Sep 22 23:14:27.998 INFO Generation requested: 1 >= found:1
Sep 22 23:14:27.998 INFO Next flush: 1
Sep 22 23:14:27.998 INFO All extents match
Sep 22 23:14:27.998 INFO No downstairs repair required
Sep 22 23:14:27.998 INFO No initial repair work was required
Sep 22 23:14:27.998 INFO Set Downstairs and Upstairs active
Sep 22 23:14:27.998 INFO d294d936-1c70-474e-8080-a4dab42d6778 is now active with session: eff1e9a9-bbb5-477e-acde-f6036995e381
Sep 22 23:14:27.998 INFO d294d936-1c70-474e-8080-a4dab42d6778 Set Active after no repair
Sep 22 23:14:27.998 INFO Notify all downstairs, region set compare is done.
Sep 22 23:14:27.998 INFO Set check for repair
Sep 22 23:14:27.998 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:27.998 INFO [1] 127.0.0.1:37528 task reports connection:true
Sep 22 23:14:27.998 INFO d294d936-1c70-474e-8080-a4dab42d6778 Active Active Active
Sep 22 23:14:27.998 INFO Created new region file "/tmp/downstairs-HbhwudPm/region.json"
Sep 22 23:14:27.998 INFO Set check for repair
Sep 22 23:14:27.998 INFO [2] 127.0.0.1:59233 task reports connection:true
Sep 22 23:14:27.998 INFO d294d936-1c70-474e-8080-a4dab42d6778 Active Active Active
Sep 22 23:14:27.998 INFO Set check for repair
Sep 22 23:14:27.998 INFO [0] received reconcile message
Sep 22 23:14:27.998 INFO [0] All repairs completed, exit
Sep 22 23:14:27.998 INFO [0] Starts cmd_loop
Sep 22 23:14:27.999 INFO [1] received reconcile message
Sep 22 23:14:27.999 INFO [1] All repairs completed, exit
Sep 22 23:14:27.999 INFO [1] Starts cmd_loop
Sep 22 23:14:27.999 INFO [2] received reconcile message
Sep 22 23:14:27.999 INFO [2] All repairs completed, exit
Sep 22 23:14:27.999 INFO [2] Starts cmd_loop
The guest has finished waiting for activation
Sep 22 23:14:28.000 DEBG [1] Read already AckReady 1005, : downstairs
Sep 22 23:14:28.000 DEBG [0] Read AckReady 1003, : downstairs
Sep 22 23:14:28.001 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:28.001 INFO Opened existing region file "/tmp/downstairs-HbhwudPm/region.json"
Sep 22 23:14:28.001 INFO Database read version 1
Sep 22 23:14:28.001 INFO Database write version 1
Sep 22 23:14:28.001 DEBG [2] Read already AckReady 1005, : downstairs
Sep 22 23:14:28.001 DEBG up_ds_listen was notified
Sep 22 23:14:28.001 DEBG up_ds_listen process 1005
Sep 22 23:14:28.002 DEBG [A] ack job 1005:6, : downstairs
Sep 22 23:14:28.002 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:28.002 DEBG IO Write 1000 has deps []
Sep 22 23:14:28.003 DEBG [1] Read already AckReady 1003, : downstairs
Sep 22 23:14:28.003 INFO UUID: 1c71ccd3-6e2a-48af-bbbe-c8c45ccbac25
Sep 22 23:14:28.003 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:28.003 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:28.003 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:28.003 INFO Using address: 127.0.0.1:58813, task: main
Sep 22 23:14:28.004 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:28.004 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:49899, task: repair
Sep 22 23:14:28.004 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:49899, task: repair
Sep 22 23:14:28.004 INFO listening, local_addr: 127.0.0.1:49899, task: repair
Sep 22 23:14:28.004 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:49899, task: repair
Sep 22 23:14:28.004 INFO Using repair address: 127.0.0.1:49899, task: main
Sep 22 23:14:28.004 INFO No SSL acceptor configured, task: main
Sep 22 23:14:28.004 INFO Upstairs starts
Sep 22 23:14:28.004 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:14:28.004 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:14:28.005 INFO Crucible stats registered with UUID: eb72ed16-1c89-4d06-98f5-0c7000baa884
Sep 22 23:14:28.005 INFO Crucible eb72ed16-1c89-4d06-98f5-0c7000baa884 has session id: 2487323e-caaf-4119-8b10-1ac34246a659
Sep 22 23:14:28.005 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:28.005 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:28.005 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:28.005 INFO [0] connecting to 127.0.0.1:53498, looper: 0
Sep 22 23:14:28.005 INFO [1] connecting to 127.0.0.1:35121, looper: 1
Sep 22 23:14:28.005 INFO [2] connecting to 127.0.0.1:58813, looper: 2
Sep 22 23:14:28.005 DEBG [2] Read already AckReady 1003, : downstairs
Sep 22 23:14:28.005 INFO up_listen starts, task: up_listen
Sep 22 23:14:28.005 INFO Wait for all three downstairs to come online
Sep 22 23:14:28.005 INFO Flush timeout: 0.5
Sep 22 23:14:28.005 DEBG up_ds_listen was notified
Sep 22 23:14:28.005 DEBG up_ds_listen process 1003
Sep 22 23:14:28.005 DEBG [A] ack job 1003:4, : downstairs
Sep 22 23:14:28.006 INFO [0] eb72ed16-1c89-4d06-98f5-0c7000baa884 looper connected, looper: 0
Sep 22 23:14:28.006 INFO [0] Proc runs for 127.0.0.1:53498 in state New
Sep 22 23:14:28.006 INFO [2] eb72ed16-1c89-4d06-98f5-0c7000baa884 looper connected, looper: 2
Sep 22 23:14:28.006 INFO [2] Proc runs for 127.0.0.1:58813 in state New
Sep 22 23:14:28.006 INFO [1] eb72ed16-1c89-4d06-98f5-0c7000baa884 looper connected, looper: 1
Sep 22 23:14:28.006 INFO [1] Proc runs for 127.0.0.1:35121 in state New
Sep 22 23:14:28.006 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:28.006 INFO accepted connection from 127.0.0.1:59909, task: main
Sep 22 23:14:28.006 INFO accepted connection from 127.0.0.1:56686, task: main
Sep 22 23:14:28.006 INFO accepted connection from 127.0.0.1:59801, task: main
Sep 22 23:14:28.006 INFO Connection request from eb72ed16-1c89-4d06-98f5-0c7000baa884 with version 4, task: proc
Sep 22 23:14:28.006 INFO upstairs UpstairsConnection { upstairs_id: eb72ed16-1c89-4d06-98f5-0c7000baa884, session_id: 69193141-8135-481f-a830-960928a8721b, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:28.007 INFO Connection request from eb72ed16-1c89-4d06-98f5-0c7000baa884 with version 4, task: proc
Sep 22 23:14:28.007 INFO upstairs UpstairsConnection { upstairs_id: eb72ed16-1c89-4d06-98f5-0c7000baa884, session_id: 69193141-8135-481f-a830-960928a8721b, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:28.007 INFO Connection request from eb72ed16-1c89-4d06-98f5-0c7000baa884 with version 4, task: proc
Sep 22 23:14:28.007 INFO upstairs UpstairsConnection { upstairs_id: eb72ed16-1c89-4d06-98f5-0c7000baa884, session_id: 69193141-8135-481f-a830-960928a8721b, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:28.007 INFO [0] eb72ed16-1c89-4d06-98f5-0c7000baa884 (69193141-8135-481f-a830-960928a8721b) New New New ds_transition to WaitActive
Sep 22 23:14:28.007 INFO [0] Transition from New to WaitActive
Sep 22 23:14:28.007 INFO [2] eb72ed16-1c89-4d06-98f5-0c7000baa884 (69193141-8135-481f-a830-960928a8721b) WaitActive New New ds_transition to WaitActive
Sep 22 23:14:28.007 INFO [2] Transition from New to WaitActive
Sep 22 23:14:28.007 INFO [1] eb72ed16-1c89-4d06-98f5-0c7000baa884 (69193141-8135-481f-a830-960928a8721b) WaitActive New WaitActive ds_transition to WaitActive
Sep 22 23:14:28.007 INFO [1] Transition from New to WaitActive
The guest has requested activation
Sep 22 23:14:28.007 INFO eb72ed16-1c89-4d06-98f5-0c7000baa884 active request set
Sep 22 23:14:28.008 INFO [0] received activate with gen 1
Sep 22 23:14:28.008 INFO [0] client got ds_active_rx, promote! session 69193141-8135-481f-a830-960928a8721b
Sep 22 23:14:28.008 INFO [1] received activate with gen 1
Sep 22 23:14:28.008 INFO [1] client got ds_active_rx, promote! session 69193141-8135-481f-a830-960928a8721b
Sep 22 23:14:28.008 INFO [2] received activate with gen 1
Sep 22 23:14:28.008 INFO [2] client got ds_active_rx, promote! session 69193141-8135-481f-a830-960928a8721b
Sep 22 23:14:28.008 INFO UpstairsConnection { upstairs_id: eb72ed16-1c89-4d06-98f5-0c7000baa884, session_id: 69193141-8135-481f-a830-960928a8721b, gen: 1 } is now active (read-write)
Sep 22 23:14:28.008 INFO UpstairsConnection { upstairs_id: eb72ed16-1c89-4d06-98f5-0c7000baa884, session_id: 69193141-8135-481f-a830-960928a8721b, gen: 1 } is now active (read-write)
Sep 22 23:14:28.008 INFO UpstairsConnection { upstairs_id: eb72ed16-1c89-4d06-98f5-0c7000baa884, session_id: 69193141-8135-481f-a830-960928a8721b, gen: 1 } is now active (read-write)
Sep 22 23:14:28.009 INFO [0] downstairs client at 127.0.0.1:53498 has UUID 177b9038-d4da-4321-bb6f-83528c5df2bb
Sep 22 23:14:28.009 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 177b9038-d4da-4321-bb6f-83528c5df2bb, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:28.009 INFO eb72ed16-1c89-4d06-98f5-0c7000baa884 WaitActive WaitActive WaitActive
Sep 22 23:14:28.009 INFO [2] downstairs client at 127.0.0.1:58813 has UUID 1c71ccd3-6e2a-48af-bbbe-c8c45ccbac25
Sep 22 23:14:28.009 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 1c71ccd3-6e2a-48af-bbbe-c8c45ccbac25, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:28.009 INFO eb72ed16-1c89-4d06-98f5-0c7000baa884 WaitActive WaitActive WaitActive
Sep 22 23:14:28.009 INFO [1] downstairs client at 127.0.0.1:35121 has UUID 7e66c791-8f61-4b2b-80e3-9850920f90c5
Sep 22 23:14:28.009 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 7e66c791-8f61-4b2b-80e3-9850920f90c5, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:28.009 INFO eb72ed16-1c89-4d06-98f5-0c7000baa884 WaitActive WaitActive WaitActive
test test::integration_test_volume_write_unwritten_1 ... ok
Sep 22 23:14:28.010 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:28.010 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:28.010 INFO Created new region file "/tmp/downstairs-GWyoXIoW/region.json"
Sep 22 23:14:28.010 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:28.011 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:28.011 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:28.011 DEBG up_ds_listen was notified
test test::integration_test_volume_subvols_parent_scrub_sparse_2 ... Sep 22 23:14:28.011 DEBG up_ds_listen process 1000
ok
Sep 22 23:14:28.011 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:14:28.011 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:28.011 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:28.011 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:28.011 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:28.011 INFO [0] eb72ed16-1c89-4d06-98f5-0c7000baa884 (69193141-8135-481f-a830-960928a8721b) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:28.011 INFO Created new region file "/tmp/downstairs-OIXCD2JG/region.json"
Sep 22 23:14:28.011 DEBG IO Read  1001 has deps [JobId(1000)]
Sep 22 23:14:28.011 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:14:28.011 WARN [0] new RM replaced this: None
Sep 22 23:14:28.011 INFO [0] Starts reconcile loop
Sep 22 23:14:28.011 INFO [2] eb72ed16-1c89-4d06-98f5-0c7000baa884 (69193141-8135-481f-a830-960928a8721b) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:28.011 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:14:28.011 WARN [2] new RM replaced this: None
Sep 22 23:14:28.011 INFO [2] Starts reconcile loop
Sep 22 23:14:28.012 INFO [1] eb72ed16-1c89-4d06-98f5-0c7000baa884 (69193141-8135-481f-a830-960928a8721b) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum
Sep 22 23:14:28.012 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:14:28.012 WARN [1] new RM replaced this: None
Sep 22 23:14:28.012 INFO [1] Starts reconcile loop
Sep 22 23:14:28.012 INFO [0] 127.0.0.1:53498 task reports connection:true
Sep 22 23:14:28.012 INFO eb72ed16-1c89-4d06-98f5-0c7000baa884 WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:14:28.012 INFO [0]R  flush_numbers: [0, 0]
Sep 22 23:14:28.012 INFO [0]R  generation: [0, 0]
Sep 22 23:14:28.012 INFO [0]R  dirty: [false, false]
Sep 22 23:14:28.012 INFO [1]R  flush_numbers: [0, 0]
Sep 22 23:14:28.012 INFO [1]R  generation: [0, 0]
Sep 22 23:14:28.012 INFO [1]R  dirty: [false, false]
Sep 22 23:14:28.012 INFO [2]R  flush_numbers: [0, 0]
Sep 22 23:14:28.012 INFO [2]R  generation: [0, 0]
Sep 22 23:14:28.012 INFO [2]R  dirty: [false, false]
Sep 22 23:14:28.012 INFO Max found gen is 1
Sep 22 23:14:28.012 INFO Generation requested: 1 >= found:1
Sep 22 23:14:28.012 INFO Next flush: 1
Sep 22 23:14:28.012 INFO All extents match
Sep 22 23:14:28.012 INFO No downstairs repair required
Sep 22 23:14:28.012 INFO No initial repair work was required
Sep 22 23:14:28.012 INFO Set Downstairs and Upstairs active
Sep 22 23:14:28.012 INFO eb72ed16-1c89-4d06-98f5-0c7000baa884 is now active with session: 69193141-8135-481f-a830-960928a8721b
Sep 22 23:14:28.012 INFO eb72ed16-1c89-4d06-98f5-0c7000baa884 Set Active after no repair
Sep 22 23:14:28.012 INFO Notify all downstairs, region set compare is done.
Sep 22 23:14:28.012 INFO Set check for repair
Sep 22 23:14:28.012 INFO [2] 127.0.0.1:58813 task reports connection:true
Sep 22 23:14:28.012 INFO eb72ed16-1c89-4d06-98f5-0c7000baa884 Active Active Active
Sep 22 23:14:28.012 DEBG Read      :1001 deps:[JobId(1000)] res:true
Sep 22 23:14:28.012 INFO Set check for repair
Sep 22 23:14:28.012 INFO [1] 127.0.0.1:35121 task reports connection:true
Sep 22 23:14:28.012 INFO eb72ed16-1c89-4d06-98f5-0c7000baa884 Active Active Active
Sep 22 23:14:28.012 INFO Set check for repair
Sep 22 23:14:28.012 INFO [0] received reconcile message
Sep 22 23:14:28.012 INFO [0] All repairs completed, exit
Sep 22 23:14:28.012 INFO [0] Starts cmd_loop
Sep 22 23:14:28.013 INFO [1] received reconcile message
Sep 22 23:14:28.013 INFO [1] All repairs completed, exit
Sep 22 23:14:28.013 INFO [1] Starts cmd_loop
Sep 22 23:14:28.013 DEBG Read      :1001 deps:[JobId(1000)] res:true
Sep 22 23:14:28.013 INFO [2] received reconcile message
Sep 22 23:14:28.013 INFO [2] All repairs completed, exit
Sep 22 23:14:28.013 INFO [2] Starts cmd_loop
The guest has finished waiting for activation
Sep 22 23:14:28.013 DEBG Read      :1001 deps:[JobId(1000)] res:true
Sep 22 23:14:28.013 DEBG IO Write 1000 has deps []
Sep 22 23:14:28.013 DEBG up_ds_listen was notified
Sep 22 23:14:28.014 DEBG up_ds_listen process 1000
Sep 22 23:14:28.014 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:14:28.014 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:28.014 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:28.014 INFO Opened existing region file "/tmp/downstairs-GWyoXIoW/region.json"
Sep 22 23:14:28.014 INFO Database read version 1
Sep 22 23:14:28.014 INFO Database write version 1
Sep 22 23:14:28.015 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:28.015 INFO Opened existing region file "/tmp/downstairs-OIXCD2JG/region.json"
Sep 22 23:14:28.015 INFO Database read version 1
Sep 22 23:14:28.015 INFO Database write version 1
Sep 22 23:14:28.015 DEBG Write     :1000 deps:[] res:true
Sep 22 23:14:28.015 DEBG Write     :1000 deps:[] res:true
Sep 22 23:14:28.016 DEBG [0] Read AckReady 1001, : downstairs
Sep 22 23:14:28.016 DEBG Write     :1000 deps:[] res:true
Sep 22 23:14:28.016 INFO UUID: 07752cff-eba4-4cda-9fc8-25c01941ef38
Sep 22 23:14:28.016 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:28.016 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:28.017 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:28.017 INFO Using address: 127.0.0.1:39201, task: main
Sep 22 23:14:28.017 INFO UUID: 4653359c-1409-4b81-8a59-94f85d2bcf29
Sep 22 23:14:28.017 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:28.017 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:28.017 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:28.017 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:33789, task: repair
Sep 22 23:14:28.017 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:28.017 INFO Using address: 127.0.0.1:43937, task: main
Sep 22 23:14:28.017 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:33789, task: repair
Sep 22 23:14:28.017 INFO listening, local_addr: 127.0.0.1:33789, task: repair
Sep 22 23:14:28.017 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:33789, task: repair
Sep 22 23:14:28.017 INFO Using repair address: 127.0.0.1:33789, task: main
Sep 22 23:14:28.017 INFO No SSL acceptor configured, task: main
Sep 22 23:14:28.017 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:28.017 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:57722, task: repair
Sep 22 23:14:28.018 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:57722, task: repair
Sep 22 23:14:28.018 INFO listening, local_addr: 127.0.0.1:57722, task: repair
Sep 22 23:14:28.018 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:28.018 DEBG [1] Read already AckReady 1001, : downstairs
Sep 22 23:14:28.018 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:57722, task: repair
Sep 22 23:14:28.018 INFO Using repair address: 127.0.0.1:57722, task: main
Sep 22 23:14:28.018 INFO Created new region file "/tmp/downstairs-F0TZn2dN/region.json"
Sep 22 23:14:28.018 INFO No SSL acceptor configured, task: main
Sep 22 23:14:28.018 DEBG IO Write 1001 has deps [JobId(1000)]
Sep 22 23:14:28.018 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:28.018 INFO Created new region file "/tmp/downstairs-rQ43necs/region.json"
Sep 22 23:14:28.019 DEBG [2] Read already AckReady 1001, : downstairs
Sep 22 23:14:28.020 DEBG up_ds_listen was notified
Sep 22 23:14:28.020 DEBG up_ds_listen process 1001
Sep 22 23:14:28.020 DEBG [A] ack job 1001:2, : downstairs
Sep 22 23:14:28.020 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:28.022 DEBG IO Write 1002 has deps [JobId(1001), JobId(1000)]
Sep 22 23:14:28.022 DEBG up_ds_listen was notified
Sep 22 23:14:28.022 DEBG up_ds_listen process 1001
Sep 22 23:14:28.022 DEBG [A] ack job 1001:2, : downstairs
Sep 22 23:14:28.022 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:28.022 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:28.022 DEBG IO Read  1002 has deps [JobId(1001), JobId(1000)]
Sep 22 23:14:28.022 INFO Opened existing region file "/tmp/downstairs-F0TZn2dN/region.json"
Sep 22 23:14:28.022 INFO Database read version 1
Sep 22 23:14:28.022 INFO Database write version 1
Sep 22 23:14:28.022 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:28.022 INFO Opened existing region file "/tmp/downstairs-rQ43necs/region.json"
Sep 22 23:14:28.023 INFO Database read version 1
Sep 22 23:14:28.023 INFO Database write version 1
Sep 22 23:14:28.023 DEBG Read      :1002 deps:[JobId(1001), JobId(1000)] res:true
Sep 22 23:14:28.023 DEBG Read      :1002 deps:[JobId(1001), JobId(1000)] res:true
Sep 22 23:14:28.023 DEBG Read      :1002 deps:[JobId(1001), JobId(1000)] res:true
Sep 22 23:14:28.024 DEBG up_ds_listen was notified
Sep 22 23:14:28.024 DEBG up_ds_listen process 1002
Sep 22 23:14:28.024 DEBG [A] ack job 1002:3, : downstairs
Sep 22 23:14:28.024 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:28.025 DEBG IO Read  1003 has deps [JobId(1002), JobId(1000)]
Sep 22 23:14:28.025 INFO UUID: 8a58fc4b-e2e1-4a7f-8747-0d8dd2cd6f65
Sep 22 23:14:28.025 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:28.025 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:28.025 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:28.025 INFO Using address: 127.0.0.1:39321, task: main
Sep 22 23:14:28.025 DEBG Read      :1003 deps:[JobId(1002), JobId(1000)] res:true
Sep 22 23:14:28.025 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:28.025 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:36320, task: repair
Sep 22 23:14:28.025 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:36320, task: repair
Sep 22 23:14:28.025 DEBG Read      :1003 deps:[JobId(1002), JobId(1000)] res:true
Sep 22 23:14:28.025 INFO listening, local_addr: 127.0.0.1:36320, task: repair
Sep 22 23:14:28.026 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:36320, task: repair
Sep 22 23:14:28.026 INFO UUID: 3f0b274b-a55a-4121-96c8-a689e27a459a
Sep 22 23:14:28.026 INFO Using repair address: 127.0.0.1:36320, task: main
Sep 22 23:14:28.026 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:28.026 INFO No SSL acceptor configured, task: main
Sep 22 23:14:28.026 DEBG Read      :1003 deps:[JobId(1002), JobId(1000)] res:true
Sep 22 23:14:28.026 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:28.026 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:28.026 INFO Using address: 127.0.0.1:38253, task: main
Sep 22 23:14:28.026 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:28.026 INFO Created new region file "/tmp/downstairs-eKrOivsr/region.json"
Sep 22 23:14:28.026 DEBG [0] Read AckReady 1002, : downstairs
Sep 22 23:14:28.026 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:28.026 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:36529, task: repair
Sep 22 23:14:28.026 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:36529, task: repair
Sep 22 23:14:28.026 INFO listening, local_addr: 127.0.0.1:36529, task: repair
Sep 22 23:14:28.027 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:36529, task: repair
Sep 22 23:14:28.027 INFO Using repair address: 127.0.0.1:36529, task: main
Sep 22 23:14:28.027 INFO No SSL acceptor configured, task: main
Sep 22 23:14:28.027 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:28.027 INFO Created new region file "/tmp/downstairs-SPkLZJb1/region.json"
Sep 22 23:14:28.028 DEBG [2] Read already AckReady 1002, : downstairs
Sep 22 23:14:28.029 DEBG [0] Read AckReady 1003, : downstairs
Sep 22 23:14:28.030 DEBG [1] Read already AckReady 1002, : downstairs
Sep 22 23:14:28.030 DEBG up_ds_listen was notified
Sep 22 23:14:28.030 DEBG up_ds_listen process 1002
Sep 22 23:14:28.030 DEBG [A] ack job 1002:3, : downstairs
Sep 22 23:14:28.030 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:28.030 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:28.030 INFO Opened existing region file "/tmp/downstairs-eKrOivsr/region.json"
Sep 22 23:14:28.030 INFO Database read version 1
Sep 22 23:14:28.030 INFO Database write version 1
Sep 22 23:14:28.030 DEBG [1] Read already AckReady 1003, : downstairs
Sep 22 23:14:28.032 DEBG [2] Read already AckReady 1003, : downstairs
Sep 22 23:14:28.032 DEBG up_ds_listen was notified
Sep 22 23:14:28.032 DEBG up_ds_listen process 1003
Sep 22 23:14:28.032 DEBG [A] ack job 1003:4, : downstairs
Sep 22 23:14:28.033 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:28.033 INFO UUID: 3f976252-a8b2-44c0-aea9-ba9b4e4fedb3
Sep 22 23:14:28.033 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:28.033 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:28.033 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:28.033 INFO Using address: 127.0.0.1:48350, task: main
test test::integration_test_volume_write_unwritten_sparse ... ok
Sep 22 23:14:28.033 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:28.033 INFO Opened existing region file "/tmp/downstairs-SPkLZJb1/region.json"
Sep 22 23:14:28.033 INFO Database read version 1
Sep 22 23:14:28.033 INFO Database write version 1
Sep 22 23:14:28.033 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:28.033 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:53088, task: repair
Sep 22 23:14:28.033 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:53088, task: repair
Sep 22 23:14:28.033 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:28.033 INFO listening, local_addr: 127.0.0.1:53088, task: repair
Sep 22 23:14:28.034 INFO Created new region file "/tmp/downstairs-5PKewOgB/region.json"
Sep 22 23:14:28.034 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:53088, task: repair
Sep 22 23:14:28.034 INFO Using repair address: 127.0.0.1:53088, task: main
Sep 22 23:14:28.034 INFO No SSL acceptor configured, task: main
Sep 22 23:14:28.034 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:28.034 INFO Created new region file "/tmp/downstairs-aiQZQRd8/region.json"
Sep 22 23:14:28.036 INFO UUID: 9a0e7cc4-1eb0-42dc-b42e-9dc1e52fa851
Sep 22 23:14:28.036 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:28.036 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:28.036 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:28.036 INFO Using address: 127.0.0.1:32937, task: main
Sep 22 23:14:28.036 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:28.036 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:63724, task: repair
Sep 22 23:14:28.037 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:63724, task: repair
Sep 22 23:14:28.037 INFO listening, local_addr: 127.0.0.1:63724, task: repair
Sep 22 23:14:28.037 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:63724, task: repair
Sep 22 23:14:28.037 INFO Using repair address: 127.0.0.1:63724, task: main
Sep 22 23:14:28.037 INFO No SSL acceptor configured, task: main
Sep 22 23:14:28.037 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:28.037 INFO Created new region file "/tmp/downstairs-WSynRapI/region.json"
test test::integration_test_volume_write_unwritten_2 ... ok
Sep 22 23:14:28.038 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:28.038 INFO Created new region file "/tmp/downstairs-U5vIRi38/region.json"
Sep 22 23:14:28.038 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:28.038 INFO Opened existing region file "/tmp/downstairs-5PKewOgB/region.json"
Sep 22 23:14:28.038 INFO Database read version 1
Sep 22 23:14:28.038 INFO Database write version 1
Sep 22 23:14:28.040 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:28.040 INFO Opened existing region file "/tmp/downstairs-aiQZQRd8/region.json"
Sep 22 23:14:28.040 INFO Database read version 1
Sep 22 23:14:28.040 INFO Database write version 1
Sep 22 23:14:28.041 INFO UUID: e867ee61-247c-4a29-967e-48e3c07ee764
Sep 22 23:14:28.041 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:28.041 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:28.042 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:28.042 INFO Using address: 127.0.0.1:46618, task: main
Sep 22 23:14:28.042 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:28.042 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:58733, task: repair
Sep 22 23:14:28.042 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:58733, task: repair
Sep 22 23:14:28.042 INFO listening, local_addr: 127.0.0.1:58733, task: repair
Sep 22 23:14:28.042 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:58733, task: repair
Sep 22 23:14:28.042 INFO Using repair address: 127.0.0.1:58733, task: main
Sep 22 23:14:28.042 INFO No SSL acceptor configured, task: main
Sep 22 23:14:28.043 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:28.043 INFO Opened existing region file "/tmp/downstairs-WSynRapI/region.json"
Sep 22 23:14:28.043 INFO Database read version 1
Sep 22 23:14:28.043 INFO Database write version 1
Sep 22 23:14:28.043 INFO UUID: dfa11e22-89c3-48d2-ac68-b5822a655ba0
Sep 22 23:14:28.043 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:28.043 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:28.043 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:28.043 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:28.043 INFO Opened existing region file "/tmp/downstairs-U5vIRi38/region.json"
Sep 22 23:14:28.043 INFO Database read version 1
Sep 22 23:14:28.043 INFO Created new region file "/tmp/downstairs-yyVBLW4f/region.json"
Sep 22 23:14:28.043 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:28.043 INFO Database write version 1
Sep 22 23:14:28.043 INFO Using address: 127.0.0.1:53594, task: main
Sep 22 23:14:28.044 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:28.044 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:50289, task: repair
Sep 22 23:14:28.044 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:50289, task: repair
Sep 22 23:14:28.044 INFO listening, local_addr: 127.0.0.1:50289, task: repair
Sep 22 23:14:28.044 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:50289, task: repair
Sep 22 23:14:28.044 INFO Using repair address: 127.0.0.1:50289, task: main
Sep 22 23:14:28.044 INFO No SSL acceptor configured, task: main
Sep 22 23:14:28.045 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:28.045 INFO Created new region file "/tmp/downstairs-a5Y9rIyO/region.json"
Sep 22 23:14:28.046 INFO UUID: b7c099f2-5446-4f38-bad7-8d884c3a5016
Sep 22 23:14:28.046 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:28.046 INFO UUID: 6fddf8da-d152-4d38-8108-a3c9271fb31e
Sep 22 23:14:28.046 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:28.046 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:28.046 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:28.046 INFO Using address: 127.0.0.1:53693, task: main
Sep 22 23:14:28.047 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:28.047 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:28.047 INFO Using address: 127.0.0.1:56487, task: main
Sep 22 23:14:28.047 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:28.047 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:28.047 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:40471, task: repair
Sep 22 23:14:28.047 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:42158, task: repair
Sep 22 23:14:28.047 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:42158, task: repair
Sep 22 23:14:28.047 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:40471, task: repair
Sep 22 23:14:28.047 INFO listening, local_addr: 127.0.0.1:42158, task: repair
Sep 22 23:14:28.047 INFO listening, local_addr: 127.0.0.1:40471, task: repair
Sep 22 23:14:28.047 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:42158, task: repair
Sep 22 23:14:28.047 INFO Using repair address: 127.0.0.1:42158, task: main
Sep 22 23:14:28.047 INFO No SSL acceptor configured, task: main
Sep 22 23:14:28.047 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:40471, task: repair
Sep 22 23:14:28.047 INFO Using repair address: 127.0.0.1:40471, task: main
Sep 22 23:14:28.047 INFO No SSL acceptor configured, task: main
Sep 22 23:14:28.048 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:28.048 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:28.048 INFO Created new region file "/tmp/downstairs-kBuYI5cw/region.json"
Sep 22 23:14:28.048 INFO Created new region file "/tmp/downstairs-oWPWb9nU/region.json"
Sep 22 23:14:28.049 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:28.049 INFO Opened existing region file "/tmp/downstairs-yyVBLW4f/region.json"
Sep 22 23:14:28.049 INFO Database read version 1
Sep 22 23:14:28.049 INFO Database write version 1
Sep 22 23:14:28.053 INFO UUID: 01a6d79b-23e4-44c1-a6f8-9804b08fb8c2
Sep 22 23:14:28.053 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:28.053 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:28.053 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:28.053 INFO Using address: 127.0.0.1:49265, task: main
Sep 22 23:14:28.053 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:28.053 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:38123, task: repair
Sep 22 23:14:28.053 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:38123, task: repair
Sep 22 23:14:28.053 INFO listening, local_addr: 127.0.0.1:38123, task: repair
Sep 22 23:14:28.053 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:38123, task: repair
Sep 22 23:14:28.053 INFO Using repair address: 127.0.0.1:38123, task: main
Sep 22 23:14:28.053 INFO No SSL acceptor configured, task: main
Sep 22 23:14:28.054 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:28.054 INFO Opened existing region file "/tmp/downstairs-kBuYI5cw/region.json"
Sep 22 23:14:28.054 INFO Database read version 1
Sep 22 23:14:28.054 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:28.054 INFO Database write version 1
Sep 22 23:14:28.054 INFO Created new region file "/tmp/downstairs-qpN1lPF1/region.json"
Sep 22 23:14:28.054 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:28.054 INFO Opened existing region file "/tmp/downstairs-a5Y9rIyO/region.json"
Sep 22 23:14:28.054 INFO Database read version 1
Sep 22 23:14:28.054 INFO Database write version 1
Sep 22 23:14:28.057 INFO UUID: 2fe4896e-5871-495b-8a9e-a2c5b4173cd5
Sep 22 23:14:28.057 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:28.057 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:28.057 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:28.057 INFO Using address: 127.0.0.1:49049, task: main
Sep 22 23:14:28.058 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:28.058 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:57494, task: repair
Sep 22 23:14:28.058 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:57494, task: repair
Sep 22 23:14:28.058 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:28.058 INFO listening, local_addr: 127.0.0.1:57494, task: repair
Sep 22 23:14:28.058 INFO Opened existing region file "/tmp/downstairs-oWPWb9nU/region.json"
Sep 22 23:14:28.058 INFO Database read version 1
Sep 22 23:14:28.058 INFO Database write version 1
Sep 22 23:14:28.058 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:57494, task: repair
Sep 22 23:14:28.058 INFO Using repair address: 127.0.0.1:57494, task: main
Sep 22 23:14:28.058 INFO No SSL acceptor configured, task: main
Sep 22 23:14:28.058 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:28.058 INFO Created new region file "/tmp/downstairs-NfNZrfMD/region.json"
Sep 22 23:14:28.059 INFO UUID: 106a66d5-99df-45fc-8e19-905c44b58eb8
Sep 22 23:14:28.059 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:28.059 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:28.059 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:28.059 INFO Using address: 127.0.0.1:50602, task: main
Sep 22 23:14:28.059 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:28.059 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:60375, task: repair
Sep 22 23:14:28.059 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:60375, task: repair
Sep 22 23:14:28.059 INFO listening, local_addr: 127.0.0.1:60375, task: repair
Sep 22 23:14:28.059 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:28.060 INFO Opened existing region file "/tmp/downstairs-qpN1lPF1/region.json"
Sep 22 23:14:28.060 INFO Database read version 1
Sep 22 23:14:28.060 INFO Database write version 1
Sep 22 23:14:28.060 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:60375, task: repair
Sep 22 23:14:28.060 INFO Using repair address: 127.0.0.1:60375, task: main
Sep 22 23:14:28.060 INFO No SSL acceptor configured, task: main
Sep 22 23:14:28.060 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:28.060 INFO Created new region file "/tmp/downstairs-tMrbAOzK/region.json"
Sep 22 23:14:28.062 INFO UUID: a532462e-5feb-4b2d-aa58-e3da9a570973
Sep 22 23:14:28.062 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:28.062 INFO UUID: 4b9378e8-0a63-4886-801a-4023859bd2fc
Sep 22 23:14:28.062 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:28.062 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:28.062 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:28.062 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:28.062 INFO Using address: 127.0.0.1:43883, task: main
Sep 22 23:14:28.062 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:28.062 INFO Using address: 127.0.0.1:52057, task: main
Sep 22 23:14:28.063 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:28.063 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:28.063 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:63277, task: repair
Sep 22 23:14:28.063 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:63277, task: repair
Sep 22 23:14:28.063 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:56173, task: repair
Sep 22 23:14:28.063 INFO listening, local_addr: 127.0.0.1:63277, task: repair
Sep 22 23:14:28.063 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:56173, task: repair
Sep 22 23:14:28.063 INFO listening, local_addr: 127.0.0.1:56173, task: repair
Sep 22 23:14:28.063 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:63277, task: repair
Sep 22 23:14:28.063 INFO Using repair address: 127.0.0.1:63277, task: main
Sep 22 23:14:28.063 INFO No SSL acceptor configured, task: main
Sep 22 23:14:28.063 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:56173, task: repair
Sep 22 23:14:28.063 INFO Using repair address: 127.0.0.1:56173, task: main
Sep 22 23:14:28.063 INFO No SSL acceptor configured, task: main
Sep 22 23:14:28.063 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:28.063 INFO Created new region file "/tmp/downstairs-GX64VDLV/region.json"
Sep 22 23:14:28.064 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:28.065 INFO Created new region file "/tmp/downstairs-lzKcgOE9/region.json"
Sep 22 23:14:28.065 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:28.065 INFO Opened existing region file "/tmp/downstairs-NfNZrfMD/region.json"
Sep 22 23:14:28.065 INFO Database read version 1
Sep 22 23:14:28.065 INFO Database write version 1
Sep 22 23:14:28.066 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:28.066 INFO Opened existing region file "/tmp/downstairs-tMrbAOzK/region.json"
Sep 22 23:14:28.066 INFO Database read version 1
Sep 22 23:14:28.066 INFO Database write version 1
Sep 22 23:14:28.068 INFO UUID: adbabfb8-1e80-4905-a6ee-914c6aaac853
Sep 22 23:14:28.068 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:28.068 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:28.068 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:28.068 INFO Using address: 127.0.0.1:57827, task: main
Sep 22 23:14:28.068 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:28.068 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:41816, task: repair
Sep 22 23:14:28.068 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:41816, task: repair
Sep 22 23:14:28.068 INFO listening, local_addr: 127.0.0.1:41816, task: repair
Sep 22 23:14:28.068 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:41816, task: repair
Sep 22 23:14:28.068 INFO Using repair address: 127.0.0.1:41816, task: main
Sep 22 23:14:28.068 INFO No SSL acceptor configured, task: main
Sep 22 23:14:28.069 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:28.069 INFO Opened existing region file "/tmp/downstairs-GX64VDLV/region.json"
Sep 22 23:14:28.069 INFO Database read version 1
Sep 22 23:14:28.069 INFO Database write version 1
note: configured to log to "/dev/stdout"
Sep 22 23:14:28.070 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:28.070 INFO Opened existing region file "/tmp/downstairs-lzKcgOE9/region.json"
Sep 22 23:14:28.070 INFO Database read version 1
Sep 22 23:14:28.070 INFO Database write version 1
Sep 22 23:14:28.070 INFO UUID: 087cb3f4-dc4b-491c-947f-013a8f45b507
Sep 22 23:14:28.070 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:28.070 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:28.070 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:28.070 INFO Using address: 127.0.0.1:53970, task: main
Sep 22 23:14:28.070 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:28.070 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:52685, task: repair
Sep 22 23:14:28.070 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:52685, task: repair
Sep 22 23:14:28.070 INFO listening, local_addr: 127.0.0.1:52685, task: repair
Sep 22 23:14:28.070 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:52685, task: repair
Sep 22 23:14:28.070 INFO Using repair address: 127.0.0.1:52685, task: main
Sep 22 23:14:28.070 INFO No SSL acceptor configured, task: main
Sep 22 23:14:28.071 INFO Upstairs starts
Sep 22 23:14:28.071 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:14:28.071 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:14:28.071 INFO Crucible stats registered with UUID: 5a917d88-e6fb-4057-be02-fbcd45b1ea14
Sep 22 23:14:28.071 INFO Crucible 5a917d88-e6fb-4057-be02-fbcd45b1ea14 has session id: eee82ed7-e118-43b0-ba47-830f213fb9e4
Sep 22 23:14:28.071 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:28.071 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:28.071 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:28.071 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:28.071 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:28.071 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:28.071 INFO [0] connecting to 127.0.0.1:39201, looper: 0
Sep 22 23:14:28.071 INFO [1] connecting to 127.0.0.1:39321, looper: 1
Sep 22 23:14:28.072 INFO [2] connecting to 127.0.0.1:48350, looper: 2
Sep 22 23:14:28.072 INFO up_listen starts, task: up_listen
Sep 22 23:14:28.072 INFO Wait for all three downstairs to come online
Sep 22 23:14:28.072 INFO Flush timeout: 0.5
Sep 22 23:14:28.072 INFO accepted connection from 127.0.0.1:42419, task: main
Sep 22 23:14:28.072 INFO accepted connection from 127.0.0.1:41895, task: main
Sep 22 23:14:28.072 INFO accepted connection from 127.0.0.1:46108, task: main
{"msg":"listening","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.072643455Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"local_addr":"127.0.0.1:40823","component":"dropshot"}
{"msg":"listen IP: 127.0.0.1:40823","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.07273504Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
Sep 22 23:14:28.072 INFO [0] 5a917d88-e6fb-4057-be02-fbcd45b1ea14 looper connected, looper: 0
Sep 22 23:14:28.072 INFO [0] Proc runs for 127.0.0.1:39201 in state New
Sep 22 23:14:28.072 INFO [1] 5a917d88-e6fb-4057-be02-fbcd45b1ea14 looper connected, looper: 1
Sep 22 23:14:28.073 INFO [1] Proc runs for 127.0.0.1:39321 in state New
Sep 22 23:14:28.073 INFO UUID: b901037e-6e64-43d6-a637-75008a7adf81
Sep 22 23:14:28.073 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:28.073 INFO [2] 5a917d88-e6fb-4057-be02-fbcd45b1ea14 looper connected, looper: 2
Sep 22 23:14:28.073 INFO [2] Proc runs for 127.0.0.1:48350 in state New
Sep 22 23:14:28.073 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:28.073 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:28.073 INFO Using address: 127.0.0.1:35251, task: main
Sep 22 23:14:28.073 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:28.073 INFO Upstairs starts
Sep 22 23:14:28.073 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:14:28.073 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:14:28.073 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:48552, task: repair
Sep 22 23:14:28.073 INFO Crucible stats registered with UUID: 93b7973b-49e5-4363-943a-5bcc4e5bcda7
Sep 22 23:14:28.073 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:48552, task: repair
Sep 22 23:14:28.073 INFO Crucible 93b7973b-49e5-4363-943a-5bcc4e5bcda7 has session id: f8ce71f3-e79f-43d9-adf6-7772f1e05e28
Sep 22 23:14:28.073 INFO listening, local_addr: 127.0.0.1:48552, task: repair
Sep 22 23:14:28.073 INFO UUID: 0d1a8812-63e7-43b3-ae0e-38daa05e774f
Sep 22 23:14:28.073 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:28.073 INFO Connection request from 5a917d88-e6fb-4057-be02-fbcd45b1ea14 with version 4, task: proc
Sep 22 23:14:28.073 INFO upstairs UpstairsConnection { upstairs_id: 5a917d88-e6fb-4057-be02-fbcd45b1ea14, session_id: 75cf63e3-c4b7-4d25-80fe-726953cbb3fa, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:28.073 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:48552, task: repair
Sep 22 23:14:28.073 INFO Using repair address: 127.0.0.1:48552, task: main
Sep 22 23:14:28.073 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:28.073 INFO No SSL acceptor configured, task: main
Sep 22 23:14:28.074 INFO Connection request from 5a917d88-e6fb-4057-be02-fbcd45b1ea14 with version 4, task: proc
Sep 22 23:14:28.074 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:28.074 INFO Using address: 127.0.0.1:37204, task: main
Sep 22 23:14:28.074 INFO upstairs UpstairsConnection { upstairs_id: 5a917d88-e6fb-4057-be02-fbcd45b1ea14, session_id: 75cf63e3-c4b7-4d25-80fe-726953cbb3fa, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:28.074 INFO Connection request from 5a917d88-e6fb-4057-be02-fbcd45b1ea14 with version 4, task: proc
Sep 22 23:14:28.074 INFO upstairs UpstairsConnection { upstairs_id: 5a917d88-e6fb-4057-be02-fbcd45b1ea14, session_id: 75cf63e3-c4b7-4d25-80fe-726953cbb3fa, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:28.074 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:28.074 INFO [0] connecting to 127.0.0.1:53594, looper: 0
Sep 22 23:14:28.074 INFO Created new region file "/tmp/downstairs-YhnRrOOA/region.json"
Sep 22 23:14:28.074 INFO [1] connecting to 127.0.0.1:50602, looper: 1
Sep 22 23:14:28.074 INFO [2] connecting to 127.0.0.1:53970, looper: 2
Sep 22 23:14:28.074 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:28.074 INFO up_listen starts, task: up_listen
Sep 22 23:14:28.074 INFO Wait for all three downstairs to come online
Sep 22 23:14:28.074 INFO Flush timeout: 0.5
Sep 22 23:14:28.074 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:53439, task: repair
Sep 22 23:14:28.074 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:53439, task: repair
Sep 22 23:14:28.074 INFO listening, local_addr: 127.0.0.1:53439, task: repair
Sep 22 23:14:28.074 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:53439, task: repair
Sep 22 23:14:28.074 INFO Using repair address: 127.0.0.1:53439, task: main
Sep 22 23:14:28.074 INFO accepted connection from 127.0.0.1:55652, task: main
Sep 22 23:14:28.074 INFO No SSL acceptor configured, task: main
Sep 22 23:14:28.074 INFO accepted connection from 127.0.0.1:54461, task: main
Sep 22 23:14:28.075 INFO [1] 93b7973b-49e5-4363-943a-5bcc4e5bcda7 looper connected, looper: 1
Sep 22 23:14:28.075 INFO [1] Proc runs for 127.0.0.1:50602 in state New
Sep 22 23:14:28.075 INFO [0] 93b7973b-49e5-4363-943a-5bcc4e5bcda7 looper connected, looper: 0
Sep 22 23:14:28.075 INFO [0] Proc runs for 127.0.0.1:53594 in state New
Sep 22 23:14:28.075 INFO accepted connection from 127.0.0.1:65326, task: main
Sep 22 23:14:28.075 INFO Upstairs starts
Sep 22 23:14:28.075 INFO [0] 5a917d88-e6fb-4057-be02-fbcd45b1ea14 (75cf63e3-c4b7-4d25-80fe-726953cbb3fa) New New New ds_transition to WaitActive
Sep 22 23:14:28.075 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:14:28.075 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:14:28.075 INFO [0] Transition from New to WaitActive
Sep 22 23:14:28.075 INFO Crucible stats registered with UUID: 20e42765-410e-433b-9d48-11eb6e978e6e
Sep 22 23:14:28.075 INFO Crucible 20e42765-410e-433b-9d48-11eb6e978e6e has session id: 10f391b6-c49b-41f7-bbae-ab24f53d2215
Sep 22 23:14:28.075 INFO [1] 5a917d88-e6fb-4057-be02-fbcd45b1ea14 (75cf63e3-c4b7-4d25-80fe-726953cbb3fa) WaitActive New New ds_transition to WaitActive
Sep 22 23:14:28.075 INFO [1] Transition from New to WaitActive
Sep 22 23:14:28.075 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:28.075 INFO [2] 5a917d88-e6fb-4057-be02-fbcd45b1ea14 (75cf63e3-c4b7-4d25-80fe-726953cbb3fa) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:14:28.075 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:28.075 INFO [2] Transition from New to WaitActive
Sep 22 23:14:28.075 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:28.075 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:28.075 INFO [2] 93b7973b-49e5-4363-943a-5bcc4e5bcda7 looper connected, looper: 2
Sep 22 23:14:28.075 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:28.075 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:28.075 INFO [2] Proc runs for 127.0.0.1:53970 in state New
Sep 22 23:14:28.075 INFO [0] connecting to 127.0.0.1:43937, looper: 0
Sep 22 23:14:28.075 INFO [1] connecting to 127.0.0.1:38253, looper: 1
Sep 22 23:14:28.076 INFO [2] connecting to 127.0.0.1:32937, looper: 2
Sep 22 23:14:28.076 INFO Connection request from 93b7973b-49e5-4363-943a-5bcc4e5bcda7 with version 4, task: proc
Sep 22 23:14:28.076 INFO up_listen starts, task: up_listen
Sep 22 23:14:28.076 INFO Wait for all three downstairs to come online
Sep 22 23:14:28.076 INFO upstairs UpstairsConnection { upstairs_id: 93b7973b-49e5-4363-943a-5bcc4e5bcda7, session_id: d6ec8a5d-f8e6-4e07-ae83-e77d818c990e, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:28.076 INFO Flush timeout: 0.5
Sep 22 23:14:28.076 INFO Connection request from 93b7973b-49e5-4363-943a-5bcc4e5bcda7 with version 4, task: proc
Sep 22 23:14:28.076 INFO upstairs UpstairsConnection { upstairs_id: 93b7973b-49e5-4363-943a-5bcc4e5bcda7, session_id: d6ec8a5d-f8e6-4e07-ae83-e77d818c990e, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:28.076 INFO accepted connection from 127.0.0.1:59670, task: main
Sep 22 23:14:28.076 INFO Connection request from 93b7973b-49e5-4363-943a-5bcc4e5bcda7 with version 4, task: proc
Sep 22 23:14:28.076 INFO upstairs UpstairsConnection { upstairs_id: 93b7973b-49e5-4363-943a-5bcc4e5bcda7, session_id: d6ec8a5d-f8e6-4e07-ae83-e77d818c990e, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:28.076 INFO accepted connection from 127.0.0.1:58743, task: main
Sep 22 23:14:28.076 INFO accepted connection from 127.0.0.1:56193, task: main
Sep 22 23:14:28.076 INFO [0] 20e42765-410e-433b-9d48-11eb6e978e6e looper connected, looper: 0
Sep 22 23:14:28.076 INFO [0] Proc runs for 127.0.0.1:43937 in state New
Sep 22 23:14:28.076 INFO [1] 93b7973b-49e5-4363-943a-5bcc4e5bcda7 (d6ec8a5d-f8e6-4e07-ae83-e77d818c990e) New New New ds_transition to WaitActive
Sep 22 23:14:28.076 INFO [1] Transition from New to WaitActive
Sep 22 23:14:28.076 INFO [1] 20e42765-410e-433b-9d48-11eb6e978e6e looper connected, looper: 1
Sep 22 23:14:28.076 INFO [1] Proc runs for 127.0.0.1:38253 in state New
Sep 22 23:14:28.076 INFO [0] 93b7973b-49e5-4363-943a-5bcc4e5bcda7 (d6ec8a5d-f8e6-4e07-ae83-e77d818c990e) New WaitActive New ds_transition to WaitActive
Sep 22 23:14:28.076 INFO [0] Transition from New to WaitActive
Sep 22 23:14:28.076 INFO [2] 20e42765-410e-433b-9d48-11eb6e978e6e looper connected, looper: 2
Sep 22 23:14:28.076 INFO [2] Proc runs for 127.0.0.1:32937 in state New
Sep 22 23:14:28.076 INFO [2] 93b7973b-49e5-4363-943a-5bcc4e5bcda7 (d6ec8a5d-f8e6-4e07-ae83-e77d818c990e) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:14:28.076 INFO [2] Transition from New to WaitActive
The guest has requested activation
Sep 22 23:14:28.077 INFO 5a917d88-e6fb-4057-be02-fbcd45b1ea14 active request set
Sep 22 23:14:28.077 INFO Upstairs starts
Sep 22 23:14:28.077 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:14:28.077 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:14:28.077 INFO [0] received activate with gen 1
Sep 22 23:14:28.077 INFO Crucible stats registered with UUID: 668b284a-1cf2-4b38-b8f6-907605c718a3
Sep 22 23:14:28.077 INFO [0] client got ds_active_rx, promote! session 75cf63e3-c4b7-4d25-80fe-726953cbb3fa
Sep 22 23:14:28.077 INFO Crucible 668b284a-1cf2-4b38-b8f6-907605c718a3 has session id: 38d842b8-b294-4d07-9341-5554639614f2
Sep 22 23:14:28.077 INFO [1] received activate with gen 1
Sep 22 23:14:28.077 INFO [1] client got ds_active_rx, promote! session 75cf63e3-c4b7-4d25-80fe-726953cbb3fa
Sep 22 23:14:28.077 INFO Connection request from 20e42765-410e-433b-9d48-11eb6e978e6e with version 4, task: proc
Sep 22 23:14:28.077 INFO [2] received activate with gen 1
Sep 22 23:14:28.077 INFO upstairs UpstairsConnection { upstairs_id: 20e42765-410e-433b-9d48-11eb6e978e6e, session_id: 08079beb-4388-43ba-85d1-274eac4beb70, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:28.077 INFO [2] client got ds_active_rx, promote! session 75cf63e3-c4b7-4d25-80fe-726953cbb3fa
Sep 22 23:14:28.077 INFO Connection request from 20e42765-410e-433b-9d48-11eb6e978e6e with version 4, task: proc
Sep 22 23:14:28.077 INFO upstairs UpstairsConnection { upstairs_id: 20e42765-410e-433b-9d48-11eb6e978e6e, session_id: 08079beb-4388-43ba-85d1-274eac4beb70, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:28.077 INFO UpstairsConnection { upstairs_id: 5a917d88-e6fb-4057-be02-fbcd45b1ea14, session_id: 75cf63e3-c4b7-4d25-80fe-726953cbb3fa, gen: 1 } is now active (read-write)
Sep 22 23:14:28.077 INFO Connection request from 20e42765-410e-433b-9d48-11eb6e978e6e with version 4, task: proc
Sep 22 23:14:28.077 INFO upstairs UpstairsConnection { upstairs_id: 20e42765-410e-433b-9d48-11eb6e978e6e, session_id: 08079beb-4388-43ba-85d1-274eac4beb70, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:28.077 INFO [0] connecting to 127.0.0.1:53693, looper: 0
Sep 22 23:14:28.077 INFO UpstairsConnection { upstairs_id: 5a917d88-e6fb-4057-be02-fbcd45b1ea14, session_id: 75cf63e3-c4b7-4d25-80fe-726953cbb3fa, gen: 1 } is now active (read-write)
Sep 22 23:14:28.077 INFO [1] connecting to 127.0.0.1:43883, looper: 1
Sep 22 23:14:28.077 INFO [2] connecting to 127.0.0.1:37204, looper: 2
Sep 22 23:14:28.077 INFO UpstairsConnection { upstairs_id: 5a917d88-e6fb-4057-be02-fbcd45b1ea14, session_id: 75cf63e3-c4b7-4d25-80fe-726953cbb3fa, gen: 1 } is now active (read-write)
Sep 22 23:14:28.077 INFO up_listen starts, task: up_listen
Sep 22 23:14:28.077 INFO Wait for all three downstairs to come online
Sep 22 23:14:28.077 INFO Flush timeout: 0.5
Sep 22 23:14:28.078 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:28.078 INFO accepted connection from 127.0.0.1:46626, task: main
Sep 22 23:14:28.078 INFO Opened existing region file "/tmp/downstairs-YhnRrOOA/region.json"
Sep 22 23:14:28.078 INFO Database read version 1
Sep 22 23:14:28.078 INFO Database write version 1
Sep 22 23:14:28.078 INFO accepted connection from 127.0.0.1:45892, task: main
Sep 22 23:14:28.078 INFO accepted connection from 127.0.0.1:42215, task: main
Sep 22 23:14:28.078 INFO [0] 20e42765-410e-433b-9d48-11eb6e978e6e (08079beb-4388-43ba-85d1-274eac4beb70) New New New ds_transition to WaitActive
Sep 22 23:14:28.078 INFO [0] Transition from New to WaitActive
Sep 22 23:14:28.078 INFO [0] downstairs client at 127.0.0.1:39201 has UUID 07752cff-eba4-4cda-9fc8-25c01941ef38
Sep 22 23:14:28.078 INFO [1] 20e42765-410e-433b-9d48-11eb6e978e6e (08079beb-4388-43ba-85d1-274eac4beb70) WaitActive New New ds_transition to WaitActive
Sep 22 23:14:28.078 INFO [1] Transition from New to WaitActive
Sep 22 23:14:28.078 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 07752cff-eba4-4cda-9fc8-25c01941ef38, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:28.078 INFO [2] 20e42765-410e-433b-9d48-11eb6e978e6e (08079beb-4388-43ba-85d1-274eac4beb70) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:14:28.078 INFO 5a917d88-e6fb-4057-be02-fbcd45b1ea14 WaitActive WaitActive WaitActive
Sep 22 23:14:28.078 INFO [2] Transition from New to WaitActive
Sep 22 23:14:28.078 INFO [0] 668b284a-1cf2-4b38-b8f6-907605c718a3 looper connected, looper: 0
Sep 22 23:14:28.078 INFO [0] Proc runs for 127.0.0.1:53693 in state New
Sep 22 23:14:28.078 INFO [1] downstairs client at 127.0.0.1:39321 has UUID 8a58fc4b-e2e1-4a7f-8747-0d8dd2cd6f65
Sep 22 23:14:28.078 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 8a58fc4b-e2e1-4a7f-8747-0d8dd2cd6f65, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:28.078 INFO [1] 668b284a-1cf2-4b38-b8f6-907605c718a3 looper connected, looper: 1
Sep 22 23:14:28.078 INFO 5a917d88-e6fb-4057-be02-fbcd45b1ea14 WaitActive WaitActive WaitActive
Sep 22 23:14:28.078 INFO [1] Proc runs for 127.0.0.1:43883 in state New
Sep 22 23:14:28.078 INFO [2] downstairs client at 127.0.0.1:48350 has UUID 3f976252-a8b2-44c0-aea9-ba9b4e4fedb3
Sep 22 23:14:28.078 INFO [2] 668b284a-1cf2-4b38-b8f6-907605c718a3 looper connected, looper: 2
Sep 22 23:14:28.078 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 3f976252-a8b2-44c0-aea9-ba9b4e4fedb3, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:28.078 INFO [2] Proc runs for 127.0.0.1:37204 in state New
Sep 22 23:14:28.078 INFO 5a917d88-e6fb-4057-be02-fbcd45b1ea14 WaitActive WaitActive WaitActive
Sep 22 23:14:28.078 INFO Connection request from 668b284a-1cf2-4b38-b8f6-907605c718a3 with version 4, task: proc
Sep 22 23:14:28.078 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:28.078 INFO upstairs UpstairsConnection { upstairs_id: 668b284a-1cf2-4b38-b8f6-907605c718a3, session_id: 5cfe049d-d730-4ad8-841b-451298810994, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:28.079 INFO Connection request from 668b284a-1cf2-4b38-b8f6-907605c718a3 with version 4, task: proc
Sep 22 23:14:28.079 INFO upstairs UpstairsConnection { upstairs_id: 668b284a-1cf2-4b38-b8f6-907605c718a3, session_id: 5cfe049d-d730-4ad8-841b-451298810994, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:28.079 INFO Connection request from 668b284a-1cf2-4b38-b8f6-907605c718a3 with version 4, task: proc
Sep 22 23:14:28.079 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:28.079 INFO upstairs UpstairsConnection { upstairs_id: 668b284a-1cf2-4b38-b8f6-907605c718a3, session_id: 5cfe049d-d730-4ad8-841b-451298810994, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:28.079 INFO [0] 668b284a-1cf2-4b38-b8f6-907605c718a3 (5cfe049d-d730-4ad8-841b-451298810994) New New New ds_transition to WaitActive
Sep 22 23:14:28.079 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:28.079 INFO [0] Transition from New to WaitActive
Sep 22 23:14:28.079 INFO [1] 668b284a-1cf2-4b38-b8f6-907605c718a3 (5cfe049d-d730-4ad8-841b-451298810994) WaitActive New New ds_transition to WaitActive
Sep 22 23:14:28.079 INFO [1] Transition from New to WaitActive
Sep 22 23:14:28.079 INFO [2] 668b284a-1cf2-4b38-b8f6-907605c718a3 (5cfe049d-d730-4ad8-841b-451298810994) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:14:28.079 INFO [2] Transition from New to WaitActive
Sep 22 23:14:28.079 INFO Downstairs has completed Negotiation, task: proc
The guest has requested activation
Sep 22 23:14:28.079 INFO 20e42765-410e-433b-9d48-11eb6e978e6e active request set
Sep 22 23:14:28.079 INFO [0] received activate with gen 1
Sep 22 23:14:28.079 INFO [0] client got ds_active_rx, promote! session 08079beb-4388-43ba-85d1-274eac4beb70
Sep 22 23:14:28.079 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:28.079 INFO [1] received activate with gen 1
Sep 22 23:14:28.079 INFO [1] client got ds_active_rx, promote! session 08079beb-4388-43ba-85d1-274eac4beb70
Sep 22 23:14:28.079 INFO [2] received activate with gen 1
Sep 22 23:14:28.079 INFO [2] client got ds_active_rx, promote! session 08079beb-4388-43ba-85d1-274eac4beb70
Sep 22 23:14:28.079 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:28.079 INFO UpstairsConnection { upstairs_id: 20e42765-410e-433b-9d48-11eb6e978e6e, session_id: 08079beb-4388-43ba-85d1-274eac4beb70, gen: 1 } is now active (read-write)
Sep 22 23:14:28.079 INFO UUID: b54ba00b-56b7-4fec-8b06-0390fa4a34ff
Sep 22 23:14:28.079 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:28.080 INFO UpstairsConnection { upstairs_id: 20e42765-410e-433b-9d48-11eb6e978e6e, session_id: 08079beb-4388-43ba-85d1-274eac4beb70, gen: 1 } is now active (read-write)
Sep 22 23:14:28.080 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:28.080 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:28.080 INFO Using address: 127.0.0.1:33061, task: main
Sep 22 23:14:28.080 INFO [0] 5a917d88-e6fb-4057-be02-fbcd45b1ea14 (75cf63e3-c4b7-4d25-80fe-726953cbb3fa) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:28.080 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:14:28.080 INFO UpstairsConnection { upstairs_id: 20e42765-410e-433b-9d48-11eb6e978e6e, session_id: 08079beb-4388-43ba-85d1-274eac4beb70, gen: 1 } is now active (read-write)
Sep 22 23:14:28.080 WARN [0] new RM replaced this: None
Sep 22 23:14:28.080 INFO [0] Starts reconcile loop
Sep 22 23:14:28.080 INFO [1] 5a917d88-e6fb-4057-be02-fbcd45b1ea14 (75cf63e3-c4b7-4d25-80fe-726953cbb3fa) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:28.080 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:14:28.080 WARN [1] new RM replaced this: None
Sep 22 23:14:28.080 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:28.080 INFO [1] Starts reconcile loop
Sep 22 23:14:28.080 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:58530, task: repair
Sep 22 23:14:28.080 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:58530, task: repair
Sep 22 23:14:28.080 INFO [2] 5a917d88-e6fb-4057-be02-fbcd45b1ea14 (75cf63e3-c4b7-4d25-80fe-726953cbb3fa) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:14:28.080 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:14:28.080 INFO listening, local_addr: 127.0.0.1:58530, task: repair
Sep 22 23:14:28.080 WARN [2] new RM replaced this: None
Sep 22 23:14:28.080 INFO [0] downstairs client at 127.0.0.1:43937 has UUID 4653359c-1409-4b81-8a59-94f85d2bcf29
Sep 22 23:14:28.080 INFO [2] Starts reconcile loop
Sep 22 23:14:28.080 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 4653359c-1409-4b81-8a59-94f85d2bcf29, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:28.080 INFO 20e42765-410e-433b-9d48-11eb6e978e6e WaitActive WaitActive WaitActive
Sep 22 23:14:28.080 INFO [0] 127.0.0.1:39201 task reports connection:true
Sep 22 23:14:28.080 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:58530, task: repair
Sep 22 23:14:28.080 INFO 5a917d88-e6fb-4057-be02-fbcd45b1ea14 WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:14:28.080 INFO Using repair address: 127.0.0.1:58530, task: main
Sep 22 23:14:28.080 INFO [1] downstairs client at 127.0.0.1:38253 has UUID 3f0b274b-a55a-4121-96c8-a689e27a459a
Sep 22 23:14:28.080 INFO No SSL acceptor configured, task: main
Sep 22 23:14:28.080 INFO [0]R  flush_numbers: [0, 0]
Sep 22 23:14:28.080 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 3f0b274b-a55a-4121-96c8-a689e27a459a, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:28.080 INFO [0]R  generation: [0, 0]
Sep 22 23:14:28.080 INFO [0]R  dirty: [false, false]
Sep 22 23:14:28.080 INFO [1]R  flush_numbers: [0, 0]
Sep 22 23:14:28.080 INFO 20e42765-410e-433b-9d48-11eb6e978e6e WaitActive WaitActive WaitActive
Sep 22 23:14:28.080 INFO [1]R  generation: [0, 0]
Sep 22 23:14:28.080 INFO [1]R  dirty: [false, false]
Sep 22 23:14:28.080 INFO [2]R  flush_numbers: [0, 0]
Sep 22 23:14:28.080 INFO [2] downstairs client at 127.0.0.1:32937 has UUID 9a0e7cc4-1eb0-42dc-b42e-9dc1e52fa851
Sep 22 23:14:28.080 INFO [2]R  generation: [0, 0]
Sep 22 23:14:28.080 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 9a0e7cc4-1eb0-42dc-b42e-9dc1e52fa851, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:28.080 INFO [2]R  dirty: [false, false]
Sep 22 23:14:28.080 INFO Max found gen is 1
Sep 22 23:14:28.080 INFO Generation requested: 1 >= found:1
Sep 22 23:14:28.080 INFO 20e42765-410e-433b-9d48-11eb6e978e6e WaitActive WaitActive WaitActive
Sep 22 23:14:28.080 INFO Next flush: 1
Sep 22 23:14:28.080 INFO All extents match
Sep 22 23:14:28.080 INFO No downstairs repair required
Sep 22 23:14:28.080 INFO No initial repair work was required
Sep 22 23:14:28.080 INFO Set Downstairs and Upstairs active
Sep 22 23:14:28.080 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:28.080 INFO 5a917d88-e6fb-4057-be02-fbcd45b1ea14 is now active with session: 75cf63e3-c4b7-4d25-80fe-726953cbb3fa
Sep 22 23:14:28.080 INFO 5a917d88-e6fb-4057-be02-fbcd45b1ea14 Set Active after no repair
Sep 22 23:14:28.080 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:28.080 INFO Notify all downstairs, region set compare is done.
Sep 22 23:14:28.080 INFO Created new region file "/tmp/downstairs-OkVHThGJ/region.json"
Sep 22 23:14:28.080 INFO Set check for repair
Sep 22 23:14:28.081 INFO [1] 127.0.0.1:39321 task reports connection:true
Sep 22 23:14:28.081 INFO 5a917d88-e6fb-4057-be02-fbcd45b1ea14 Active Active Active
Sep 22 23:14:28.081 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:28.081 INFO Set check for repair
Sep 22 23:14:28.081 INFO [2] 127.0.0.1:48350 task reports connection:true
Sep 22 23:14:28.081 INFO 5a917d88-e6fb-4057-be02-fbcd45b1ea14 Active Active Active
Sep 22 23:14:28.081 INFO Set check for repair
Sep 22 23:14:28.081 INFO [0] received reconcile message
Sep 22 23:14:28.081 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:28.081 INFO [0] All repairs completed, exit
Sep 22 23:14:28.081 INFO [0] Starts cmd_loop
Sep 22 23:14:28.081 INFO [1] received reconcile message
Sep 22 23:14:28.081 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:28.081 INFO [1] All repairs completed, exit
Sep 22 23:14:28.081 INFO [1] Starts cmd_loop
Sep 22 23:14:28.081 INFO [2] received reconcile message
Sep 22 23:14:28.081 INFO [2] All repairs completed, exit
Sep 22 23:14:28.081 INFO [2] Starts cmd_loop
Sep 22 23:14:28.081 INFO Current flush_numbers [0..12]: [0, 0]
The guest has finished waiting for activation
Sep 22 23:14:28.081 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:28.081 INFO [0] 20e42765-410e-433b-9d48-11eb6e978e6e (08079beb-4388-43ba-85d1-274eac4beb70) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:28.081 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:14:28.082 WARN [0] new RM replaced this: None
The guest has requested activation
Sep 22 23:14:28.082 INFO [0] Starts reconcile loop
Sep 22 23:14:28.082 INFO 93b7973b-49e5-4363-943a-5bcc4e5bcda7 active request set
Sep 22 23:14:28.082 INFO [1] 20e42765-410e-433b-9d48-11eb6e978e6e (08079beb-4388-43ba-85d1-274eac4beb70) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:28.082 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:14:28.082 WARN [1] new RM replaced this: None
Sep 22 23:14:28.082 INFO [0] received activate with gen 1
Sep 22 23:14:28.082 INFO [1] Starts reconcile loop
Sep 22 23:14:28.082 INFO [0] client got ds_active_rx, promote! session d6ec8a5d-f8e6-4e07-ae83-e77d818c990e
Sep 22 23:14:28.082 INFO [2] 20e42765-410e-433b-9d48-11eb6e978e6e (08079beb-4388-43ba-85d1-274eac4beb70) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:14:28.082 INFO [1] received activate with gen 1
Sep 22 23:14:28.082 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:14:28.082 INFO [1] client got ds_active_rx, promote! session d6ec8a5d-f8e6-4e07-ae83-e77d818c990e
Sep 22 23:14:28.082 WARN [2] new RM replaced this: None
Sep 22 23:14:28.082 INFO [2] Starts reconcile loop
Sep 22 23:14:28.082 INFO [2] received activate with gen 1
Sep 22 23:14:28.082 INFO [2] client got ds_active_rx, promote! session d6ec8a5d-f8e6-4e07-ae83-e77d818c990e
Sep 22 23:14:28.082 INFO [0] 127.0.0.1:43937 task reports connection:true
Sep 22 23:14:28.082 INFO 20e42765-410e-433b-9d48-11eb6e978e6e WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:14:28.082 INFO [0]R  flush_numbers: [0, 0]
Sep 22 23:14:28.082 INFO [0]R  generation: [0, 0]
Sep 22 23:14:28.082 INFO [0]R  dirty: [false, false]
Sep 22 23:14:28.082 INFO [1]R  flush_numbers: [0, 0]
Sep 22 23:14:28.082 INFO [1]R  generation: [0, 0]
Sep 22 23:14:28.082 INFO [1]R  dirty: [false, false]
Sep 22 23:14:28.082 INFO [2]R  flush_numbers: [0, 0]
Sep 22 23:14:28.082 INFO UpstairsConnection { upstairs_id: 93b7973b-49e5-4363-943a-5bcc4e5bcda7, session_id: d6ec8a5d-f8e6-4e07-ae83-e77d818c990e, gen: 1 } is now active (read-write)
Sep 22 23:14:28.082 INFO [2]R  generation: [0, 0]
Sep 22 23:14:28.082 INFO [2]R  dirty: [false, false]
Sep 22 23:14:28.082 INFO Max found gen is 1
Sep 22 23:14:28.082 INFO Generation requested: 1 >= found:1
Sep 22 23:14:28.082 INFO Next flush: 1
Sep 22 23:14:28.082 INFO All extents match
Sep 22 23:14:28.082 INFO No downstairs repair required
Sep 22 23:14:28.082 INFO UpstairsConnection { upstairs_id: 93b7973b-49e5-4363-943a-5bcc4e5bcda7, session_id: d6ec8a5d-f8e6-4e07-ae83-e77d818c990e, gen: 1 } is now active (read-write)
Sep 22 23:14:28.082 INFO No initial repair work was required
Sep 22 23:14:28.082 INFO Set Downstairs and Upstairs active
Sep 22 23:14:28.082 INFO 20e42765-410e-433b-9d48-11eb6e978e6e is now active with session: 08079beb-4388-43ba-85d1-274eac4beb70
Sep 22 23:14:28.082 INFO 20e42765-410e-433b-9d48-11eb6e978e6e Set Active after no repair
Sep 22 23:14:28.082 INFO Notify all downstairs, region set compare is done.
Sep 22 23:14:28.082 INFO Set check for repair
Sep 22 23:14:28.082 INFO UpstairsConnection { upstairs_id: 93b7973b-49e5-4363-943a-5bcc4e5bcda7, session_id: d6ec8a5d-f8e6-4e07-ae83-e77d818c990e, gen: 1 } is now active (read-write)
Sep 22 23:14:28.082 INFO [1] 127.0.0.1:38253 task reports connection:true
Sep 22 23:14:28.082 INFO 20e42765-410e-433b-9d48-11eb6e978e6e Active Active Active
Sep 22 23:14:28.082 INFO Set check for repair
Sep 22 23:14:28.082 INFO [2] 127.0.0.1:32937 task reports connection:true
Sep 22 23:14:28.082 INFO 20e42765-410e-433b-9d48-11eb6e978e6e Active Active Active
Sep 22 23:14:28.082 INFO Set check for repair
Sep 22 23:14:28.082 INFO [0] received reconcile message
Sep 22 23:14:28.082 INFO [0] All repairs completed, exit
Sep 22 23:14:28.082 INFO [0] Starts cmd_loop
Sep 22 23:14:28.082 INFO [1] received reconcile message
Sep 22 23:14:28.082 INFO [1] All repairs completed, exit
Sep 22 23:14:28.082 INFO [1] Starts cmd_loop
Sep 22 23:14:28.082 INFO [1] downstairs client at 127.0.0.1:50602 has UUID 106a66d5-99df-45fc-8e19-905c44b58eb8
Sep 22 23:14:28.082 INFO [2] received reconcile message
Sep 22 23:14:28.082 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 106a66d5-99df-45fc-8e19-905c44b58eb8, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:28.082 INFO [2] All repairs completed, exit
Sep 22 23:14:28.082 INFO 93b7973b-49e5-4363-943a-5bcc4e5bcda7 WaitActive WaitActive WaitActive
Sep 22 23:14:28.083 INFO [0] downstairs client at 127.0.0.1:53594 has UUID dfa11e22-89c3-48d2-ac68-b5822a655ba0
Sep 22 23:14:28.083 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: dfa11e22-89c3-48d2-ac68-b5822a655ba0, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:28.083 INFO [2] Starts cmd_loop
Sep 22 23:14:28.083 INFO 93b7973b-49e5-4363-943a-5bcc4e5bcda7 WaitActive WaitActive WaitActive
The guest has finished waiting for activation
Sep 22 23:14:28.083 INFO [2] downstairs client at 127.0.0.1:53970 has UUID 087cb3f4-dc4b-491c-947f-013a8f45b507
Sep 22 23:14:28.083 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 087cb3f4-dc4b-491c-947f-013a8f45b507, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:28.083 INFO 93b7973b-49e5-4363-943a-5bcc4e5bcda7 WaitActive WaitActive WaitActive
The guest has requested activation
Sep 22 23:14:28.083 INFO 668b284a-1cf2-4b38-b8f6-907605c718a3 active request set
Sep 22 23:14:28.083 INFO [0] received activate with gen 1
Sep 22 23:14:28.083 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:28.083 INFO [0] client got ds_active_rx, promote! session 5cfe049d-d730-4ad8-841b-451298810994
Sep 22 23:14:28.083 INFO [1] received activate with gen 1
Sep 22 23:14:28.083 INFO [1] client got ds_active_rx, promote! session 5cfe049d-d730-4ad8-841b-451298810994
Sep 22 23:14:28.083 INFO [2] received activate with gen 1
Sep 22 23:14:28.083 INFO [2] client got ds_active_rx, promote! session 5cfe049d-d730-4ad8-841b-451298810994
Sep 22 23:14:28.083 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:28.083 INFO UpstairsConnection { upstairs_id: 668b284a-1cf2-4b38-b8f6-907605c718a3, session_id: 5cfe049d-d730-4ad8-841b-451298810994, gen: 1 } is now active (read-write)
Sep 22 23:14:28.083 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:28.083 INFO UpstairsConnection { upstairs_id: 668b284a-1cf2-4b38-b8f6-907605c718a3, session_id: 5cfe049d-d730-4ad8-841b-451298810994, gen: 1 } is now active (read-write)
Sep 22 23:14:28.083 INFO UpstairsConnection { upstairs_id: 668b284a-1cf2-4b38-b8f6-907605c718a3, session_id: 5cfe049d-d730-4ad8-841b-451298810994, gen: 1 } is now active (read-write)
Sep 22 23:14:28.084 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:28.084 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:28.084 INFO [0] downstairs client at 127.0.0.1:53693 has UUID 6fddf8da-d152-4d38-8108-a3c9271fb31e
Sep 22 23:14:28.084 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 6fddf8da-d152-4d38-8108-a3c9271fb31e, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:28.084 INFO 668b284a-1cf2-4b38-b8f6-907605c718a3 WaitActive WaitActive WaitActive
Sep 22 23:14:28.084 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:28.084 INFO [1] downstairs client at 127.0.0.1:43883 has UUID a532462e-5feb-4b2d-aa58-e3da9a570973
Sep 22 23:14:28.084 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: a532462e-5feb-4b2d-aa58-e3da9a570973, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:28.084 INFO 668b284a-1cf2-4b38-b8f6-907605c718a3 WaitActive WaitActive WaitActive
Sep 22 23:14:28.084 INFO [2] downstairs client at 127.0.0.1:37204 has UUID 0d1a8812-63e7-43b3-ae0e-38daa05e774f
Sep 22 23:14:28.084 INFO [1] 93b7973b-49e5-4363-943a-5bcc4e5bcda7 (d6ec8a5d-f8e6-4e07-ae83-e77d818c990e) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:28.084 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 0d1a8812-63e7-43b3-ae0e-38daa05e774f, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:28.084 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:14:28.084 WARN [1] new RM replaced this: None
Sep 22 23:14:28.084 INFO 668b284a-1cf2-4b38-b8f6-907605c718a3 WaitActive WaitActive WaitActive
Sep 22 23:14:28.084 INFO [1] Starts reconcile loop
Sep 22 23:14:28.084 INFO [0] 93b7973b-49e5-4363-943a-5bcc4e5bcda7 (d6ec8a5d-f8e6-4e07-ae83-e77d818c990e) WaitActive WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:14:28.084 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:14:28.084 WARN [0] new RM replaced this: None
Sep 22 23:14:28.084 INFO [0] Starts reconcile loop
Sep 22 23:14:28.084 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:28.084 INFO [2] 93b7973b-49e5-4363-943a-5bcc4e5bcda7 (d6ec8a5d-f8e6-4e07-ae83-e77d818c990e) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:14:28.084 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:14:28.084 WARN [2] new RM replaced this: None
Sep 22 23:14:28.084 INFO [2] Starts reconcile loop
Sep 22 23:14:28.084 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:28.084 INFO Opened existing region file "/tmp/downstairs-OkVHThGJ/region.json"
Sep 22 23:14:28.084 INFO [1] 127.0.0.1:50602 task reports connection:true
Sep 22 23:14:28.084 INFO Database read version 1
Sep 22 23:14:28.084 INFO Database write version 1
Sep 22 23:14:28.084 INFO 93b7973b-49e5-4363-943a-5bcc4e5bcda7 WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:14:28.084 INFO [0]R  flush_numbers: [0, 0]
Sep 22 23:14:28.084 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:28.084 INFO [0]R  generation: [0, 0]
Sep 22 23:14:28.084 INFO [0]R  dirty: [false, false]
Sep 22 23:14:28.084 INFO [1]R  flush_numbers: [0, 0]
Sep 22 23:14:28.084 INFO [1]R  generation: [0, 0]
Sep 22 23:14:28.084 INFO [1]R  dirty: [false, false]
Sep 22 23:14:28.084 INFO [2]R  flush_numbers: [0, 0]
Sep 22 23:14:28.084 INFO [2]R  generation: [0, 0]
Sep 22 23:14:28.084 INFO [2]R  dirty: [false, false]
Sep 22 23:14:28.084 INFO Max found gen is 1
Sep 22 23:14:28.084 INFO Generation requested: 1 >= found:1
Sep 22 23:14:28.084 INFO Next flush: 1
Sep 22 23:14:28.084 INFO All extents match
Sep 22 23:14:28.085 INFO No downstairs repair required
Sep 22 23:14:28.085 INFO No initial repair work was required
Sep 22 23:14:28.085 INFO Set Downstairs and Upstairs active
Sep 22 23:14:28.085 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:28.085 INFO 93b7973b-49e5-4363-943a-5bcc4e5bcda7 is now active with session: d6ec8a5d-f8e6-4e07-ae83-e77d818c990e
Sep 22 23:14:28.085 INFO 93b7973b-49e5-4363-943a-5bcc4e5bcda7 Set Active after no repair
Sep 22 23:14:28.085 INFO Notify all downstairs, region set compare is done.
Sep 22 23:14:28.085 INFO Set check for repair
Sep 22 23:14:28.085 INFO [0] 127.0.0.1:53594 task reports connection:true
Sep 22 23:14:28.085 INFO 93b7973b-49e5-4363-943a-5bcc4e5bcda7 Active Active Active
Sep 22 23:14:28.085 INFO Set check for repair
Sep 22 23:14:28.085 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:28.085 INFO [2] 127.0.0.1:53970 task reports connection:true
Sep 22 23:14:28.085 INFO 93b7973b-49e5-4363-943a-5bcc4e5bcda7 Active Active Active
Sep 22 23:14:28.085 INFO Set check for repair
Sep 22 23:14:28.085 INFO [0] received reconcile message
Sep 22 23:14:28.085 INFO [0] All repairs completed, exit
Sep 22 23:14:28.085 INFO [0] Starts cmd_loop
Sep 22 23:14:28.085 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:28.085 INFO [1] received reconcile message
Sep 22 23:14:28.085 INFO [1] All repairs completed, exit
Sep 22 23:14:28.085 INFO [1] Starts cmd_loop
Sep 22 23:14:28.085 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:28.085 INFO [2] received reconcile message
Sep 22 23:14:28.085 INFO [2] All repairs completed, exit
Sep 22 23:14:28.085 INFO [2] Starts cmd_loop
The guest has finished waiting for activation
Sep 22 23:14:28.085 INFO [0] 668b284a-1cf2-4b38-b8f6-907605c718a3 (5cfe049d-d730-4ad8-841b-451298810994) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:28.085 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:14:28.085 WARN [0] new RM replaced this: None
Sep 22 23:14:28.085 INFO [0] Starts reconcile loop
Sep 22 23:14:28.085 INFO [1] 668b284a-1cf2-4b38-b8f6-907605c718a3 (5cfe049d-d730-4ad8-841b-451298810994) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:28.085 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:14:28.085 WARN [1] new RM replaced this: None
Sep 22 23:14:28.085 INFO [1] Starts reconcile loop
Sep 22 23:14:28.085 INFO [2] 668b284a-1cf2-4b38-b8f6-907605c718a3 (5cfe049d-d730-4ad8-841b-451298810994) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:14:28.085 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:14:28.085 WARN [2] new RM replaced this: None
Sep 22 23:14:28.085 INFO [2] Starts reconcile loop
Sep 22 23:14:28.086 INFO [0] 127.0.0.1:53693 task reports connection:true
Sep 22 23:14:28.086 INFO 668b284a-1cf2-4b38-b8f6-907605c718a3 WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:14:28.086 INFO [0]R  flush_numbers: [0, 0]
Sep 22 23:14:28.086 INFO [0]R  generation: [0, 0]
Sep 22 23:14:28.086 INFO [0]R  dirty: [false, false]
Sep 22 23:14:28.086 INFO [1]R  flush_numbers: [0, 0]
Sep 22 23:14:28.086 INFO [1]R  generation: [0, 0]
Sep 22 23:14:28.086 INFO [1]R  dirty: [false, false]
Sep 22 23:14:28.086 INFO [2]R  flush_numbers: [0, 0]
Sep 22 23:14:28.086 INFO [2]R  generation: [0, 0]
Sep 22 23:14:28.086 INFO [2]R  dirty: [false, false]
Sep 22 23:14:28.086 INFO Max found gen is 1
Sep 22 23:14:28.086 INFO Generation requested: 1 >= found:1
Sep 22 23:14:28.086 INFO Next flush: 1
Sep 22 23:14:28.086 INFO All extents match
Sep 22 23:14:28.086 INFO No downstairs repair required
Sep 22 23:14:28.086 INFO No initial repair work was required
Sep 22 23:14:28.086 INFO Set Downstairs and Upstairs active
Sep 22 23:14:28.086 INFO 668b284a-1cf2-4b38-b8f6-907605c718a3 is now active with session: 5cfe049d-d730-4ad8-841b-451298810994
Sep 22 23:14:28.086 INFO 668b284a-1cf2-4b38-b8f6-907605c718a3 Set Active after no repair
Sep 22 23:14:28.086 INFO Notify all downstairs, region set compare is done.
Sep 22 23:14:28.086 INFO Set check for repair
Sep 22 23:14:28.086 INFO [1] 127.0.0.1:43883 task reports connection:true
Sep 22 23:14:28.086 INFO 668b284a-1cf2-4b38-b8f6-907605c718a3 Active Active Active
Sep 22 23:14:28.086 INFO Set check for repair
Sep 22 23:14:28.086 INFO [2] 127.0.0.1:37204 task reports connection:true
Sep 22 23:14:28.086 INFO 668b284a-1cf2-4b38-b8f6-907605c718a3 Active Active Active
Sep 22 23:14:28.086 INFO Set check for repair
Sep 22 23:14:28.086 INFO [0] received reconcile message
Sep 22 23:14:28.086 INFO [0] All repairs completed, exit
Sep 22 23:14:28.086 INFO [0] Starts cmd_loop
Sep 22 23:14:28.086 INFO [1] received reconcile message
Sep 22 23:14:28.086 INFO [1] All repairs completed, exit
Sep 22 23:14:28.086 INFO [1] Starts cmd_loop
Sep 22 23:14:28.086 INFO [2] received reconcile message
Sep 22 23:14:28.086 INFO [2] All repairs completed, exit
Sep 22 23:14:28.086 INFO [2] Starts cmd_loop
The guest has finished waiting for activation
Sep 22 23:14:28.086 INFO UUID: d07ff0ba-0bf5-4b77-9ec5-a7938bb906f6
Sep 22 23:14:28.086 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:28.086 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:28.086 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:28.086 INFO Using address: 127.0.0.1:61465, task: main
Sep 22 23:14:28.087 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:28.087 DEBG IO Write 1000 has deps []
Sep 22 23:14:28.087 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:52938, task: repair
Sep 22 23:14:28.087 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:52938, task: repair
Sep 22 23:14:28.087 INFO listening, local_addr: 127.0.0.1:52938, task: repair
Sep 22 23:14:28.087 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:52938, task: repair
Sep 22 23:14:28.087 INFO Using repair address: 127.0.0.1:52938, task: main
Sep 22 23:14:28.087 INFO No SSL acceptor configured, task: main
Sep 22 23:14:28.087 INFO Upstairs starts
Sep 22 23:14:28.087 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:14:28.087 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:14:28.087 INFO Crucible stats registered with UUID: 8cdc62de-7a87-47c6-9c8c-02c45f1c00bf
Sep 22 23:14:28.087 INFO Crucible 8cdc62de-7a87-47c6-9c8c-02c45f1c00bf has session id: 8b3cf6dc-0c13-435e-aadb-56c831fe7679
Sep 22 23:14:28.088 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:28.088 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:28.088 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:28.088 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:28.088 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:28.088 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:28.088 INFO [0] connecting to 127.0.0.1:46618, looper: 0
Sep 22 23:14:28.088 INFO [1] connecting to 127.0.0.1:49265, looper: 1
Sep 22 23:14:28.088 DEBG IO Write 1000 has deps []
Sep 22 23:14:28.088 INFO [2] connecting to 127.0.0.1:52057, looper: 2
Sep 22 23:14:28.088 INFO up_listen starts, task: up_listen
Sep 22 23:14:28.088 INFO Wait for all three downstairs to come online
Sep 22 23:14:28.088 INFO Flush timeout: 0.5
Sep 22 23:14:28.088 INFO accepted connection from 127.0.0.1:34129, task: main
Sep 22 23:14:28.088 INFO accepted connection from 127.0.0.1:39645, task: main
Sep 22 23:14:28.088 INFO accepted connection from 127.0.0.1:37248, task: main
Sep 22 23:14:28.089 INFO [0] 8cdc62de-7a87-47c6-9c8c-02c45f1c00bf looper connected, looper: 0
Sep 22 23:14:28.089 INFO [0] Proc runs for 127.0.0.1:46618 in state New
Sep 22 23:14:28.089 INFO [2] 8cdc62de-7a87-47c6-9c8c-02c45f1c00bf looper connected, looper: 2
Sep 22 23:14:28.089 INFO [2] Proc runs for 127.0.0.1:52057 in state New
Sep 22 23:14:28.089 INFO [1] 8cdc62de-7a87-47c6-9c8c-02c45f1c00bf looper connected, looper: 1
Sep 22 23:14:28.089 DEBG up_ds_listen was notified
Sep 22 23:14:28.089 INFO [1] Proc runs for 127.0.0.1:49265 in state New
Sep 22 23:14:28.089 DEBG up_ds_listen process 1000
Sep 22 23:14:28.089 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:14:28.089 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:28.089 INFO Upstairs starts
Sep 22 23:14:28.089 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:14:28.089 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:14:28.089 INFO Crucible stats registered with UUID: 23d5a1a6-b740-4ec5-ba77-4864ba7d7d72
Sep 22 23:14:28.089 DEBG IO Write 1000 has deps []
Sep 22 23:14:28.089 INFO Crucible 23d5a1a6-b740-4ec5-ba77-4864ba7d7d72 has session id: 64f5e595-7972-4cba-a861-ca60f3d9d8a6
Sep 22 23:14:28.089 INFO Connection request from 8cdc62de-7a87-47c6-9c8c-02c45f1c00bf with version 4, task: proc
Sep 22 23:14:28.089 INFO upstairs UpstairsConnection { upstairs_id: 8cdc62de-7a87-47c6-9c8c-02c45f1c00bf, session_id: f4dd55be-06a0-4424-99ef-c92da9671dc7, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:28.089 INFO Connection request from 8cdc62de-7a87-47c6-9c8c-02c45f1c00bf with version 4, task: proc
Sep 22 23:14:28.090 INFO upstairs UpstairsConnection { upstairs_id: 8cdc62de-7a87-47c6-9c8c-02c45f1c00bf, session_id: f4dd55be-06a0-4424-99ef-c92da9671dc7, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:28.090 INFO Connection request from 8cdc62de-7a87-47c6-9c8c-02c45f1c00bf with version 4, task: proc
Sep 22 23:14:28.090 INFO upstairs UpstairsConnection { upstairs_id: 8cdc62de-7a87-47c6-9c8c-02c45f1c00bf, session_id: f4dd55be-06a0-4424-99ef-c92da9671dc7, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:28.090 INFO [0] connecting to 127.0.0.1:35251, looper: 0
Sep 22 23:14:28.090 INFO [1] connecting to 127.0.0.1:33061, looper: 1
Sep 22 23:14:28.090 INFO [2] connecting to 127.0.0.1:61465, looper: 2
Sep 22 23:14:28.090 INFO up_listen starts, task: up_listen
Sep 22 23:14:28.090 INFO Wait for all three downstairs to come online
Sep 22 23:14:28.090 INFO Flush timeout: 0.5
Sep 22 23:14:28.090 INFO accepted connection from 127.0.0.1:54798, task: main
Sep 22 23:14:28.090 INFO accepted connection from 127.0.0.1:65146, task: main
Sep 22 23:14:28.091 INFO accepted connection from 127.0.0.1:56088, task: main
Sep 22 23:14:28.091 INFO [0] 8cdc62de-7a87-47c6-9c8c-02c45f1c00bf (f4dd55be-06a0-4424-99ef-c92da9671dc7) New New New ds_transition to WaitActive
Sep 22 23:14:28.091 INFO [0] Transition from New to WaitActive
Sep 22 23:14:28.091 INFO [2] 8cdc62de-7a87-47c6-9c8c-02c45f1c00bf (f4dd55be-06a0-4424-99ef-c92da9671dc7) WaitActive New New ds_transition to WaitActive
Sep 22 23:14:28.091 INFO [2] Transition from New to WaitActive
Sep 22 23:14:28.091 INFO [1] 8cdc62de-7a87-47c6-9c8c-02c45f1c00bf (f4dd55be-06a0-4424-99ef-c92da9671dc7) WaitActive New WaitActive ds_transition to WaitActive
Sep 22 23:14:28.091 INFO [1] Transition from New to WaitActive
Sep 22 23:14:28.091 INFO [0] 23d5a1a6-b740-4ec5-ba77-4864ba7d7d72 looper connected, looper: 0
Sep 22 23:14:28.091 INFO [0] Proc runs for 127.0.0.1:35251 in state New
Sep 22 23:14:28.091 INFO [1] 23d5a1a6-b740-4ec5-ba77-4864ba7d7d72 looper connected, looper: 1
Sep 22 23:14:28.091 INFO [1] Proc runs for 127.0.0.1:33061 in state New
Sep 22 23:14:28.091 INFO [2] 23d5a1a6-b740-4ec5-ba77-4864ba7d7d72 looper connected, looper: 2
Sep 22 23:14:28.091 INFO [2] Proc runs for 127.0.0.1:61465 in state New
Sep 22 23:14:28.091 DEBG up_ds_listen was notified
Sep 22 23:14:28.092 DEBG up_ds_listen process 1000
Sep 22 23:14:28.092 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:14:28.092 INFO Connection request from 23d5a1a6-b740-4ec5-ba77-4864ba7d7d72 with version 4, task: proc
Sep 22 23:14:28.092 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:28.092 INFO upstairs UpstairsConnection { upstairs_id: 23d5a1a6-b740-4ec5-ba77-4864ba7d7d72, session_id: b557ec33-3655-4c40-910a-6ea3a4aafaac, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:28.092 INFO Connection request from 23d5a1a6-b740-4ec5-ba77-4864ba7d7d72 with version 4, task: proc
Sep 22 23:14:28.092 INFO upstairs UpstairsConnection { upstairs_id: 23d5a1a6-b740-4ec5-ba77-4864ba7d7d72, session_id: b557ec33-3655-4c40-910a-6ea3a4aafaac, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:28.092 INFO Connection request from 23d5a1a6-b740-4ec5-ba77-4864ba7d7d72 with version 4, task: proc
Sep 22 23:14:28.092 INFO upstairs UpstairsConnection { upstairs_id: 23d5a1a6-b740-4ec5-ba77-4864ba7d7d72, session_id: b557ec33-3655-4c40-910a-6ea3a4aafaac, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:28.092 INFO [0] 23d5a1a6-b740-4ec5-ba77-4864ba7d7d72 (b557ec33-3655-4c40-910a-6ea3a4aafaac) New New New ds_transition to WaitActive
Sep 22 23:14:28.092 INFO [0] Transition from New to WaitActive
Sep 22 23:14:28.092 INFO [1] 23d5a1a6-b740-4ec5-ba77-4864ba7d7d72 (b557ec33-3655-4c40-910a-6ea3a4aafaac) WaitActive New New ds_transition to WaitActive
Sep 22 23:14:28.092 INFO [1] Transition from New to WaitActive
Sep 22 23:14:28.092 INFO [2] 23d5a1a6-b740-4ec5-ba77-4864ba7d7d72 (b557ec33-3655-4c40-910a-6ea3a4aafaac) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:14:28.092 INFO [2] Transition from New to WaitActive
The guest has requested activation
Sep 22 23:14:28.093 INFO 8cdc62de-7a87-47c6-9c8c-02c45f1c00bf active request set
Sep 22 23:14:28.093 INFO [0] received activate with gen 1
Sep 22 23:14:28.093 INFO [0] client got ds_active_rx, promote! session f4dd55be-06a0-4424-99ef-c92da9671dc7
Sep 22 23:14:28.093 INFO [1] received activate with gen 1
Sep 22 23:14:28.093 INFO [1] client got ds_active_rx, promote! session f4dd55be-06a0-4424-99ef-c92da9671dc7
Sep 22 23:14:28.093 INFO [2] received activate with gen 1
Sep 22 23:14:28.093 INFO [2] client got ds_active_rx, promote! session f4dd55be-06a0-4424-99ef-c92da9671dc7
Sep 22 23:14:28.093 INFO UpstairsConnection { upstairs_id: 8cdc62de-7a87-47c6-9c8c-02c45f1c00bf, session_id: f4dd55be-06a0-4424-99ef-c92da9671dc7, gen: 1 } is now active (read-write)
Sep 22 23:14:28.093 INFO UpstairsConnection { upstairs_id: 8cdc62de-7a87-47c6-9c8c-02c45f1c00bf, session_id: f4dd55be-06a0-4424-99ef-c92da9671dc7, gen: 1 } is now active (read-write)
Sep 22 23:14:28.093 INFO UpstairsConnection { upstairs_id: 8cdc62de-7a87-47c6-9c8c-02c45f1c00bf, session_id: f4dd55be-06a0-4424-99ef-c92da9671dc7, gen: 1 } is now active (read-write)
Sep 22 23:14:28.093 DEBG up_ds_listen was notified
Sep 22 23:14:28.093 DEBG up_ds_listen process 1000
Sep 22 23:14:28.094 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:14:28.094 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:28.094 INFO [0] downstairs client at 127.0.0.1:46618 has UUID e867ee61-247c-4a29-967e-48e3c07ee764
Sep 22 23:14:28.094 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: e867ee61-247c-4a29-967e-48e3c07ee764, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:28.094 INFO 8cdc62de-7a87-47c6-9c8c-02c45f1c00bf WaitActive WaitActive WaitActive
Sep 22 23:14:28.094 INFO [2] downstairs client at 127.0.0.1:52057 has UUID 4b9378e8-0a63-4886-801a-4023859bd2fc
Sep 22 23:14:28.094 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 4b9378e8-0a63-4886-801a-4023859bd2fc, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:28.094 INFO 8cdc62de-7a87-47c6-9c8c-02c45f1c00bf WaitActive WaitActive WaitActive
Sep 22 23:14:28.094 INFO [1] downstairs client at 127.0.0.1:49265 has UUID 01a6d79b-23e4-44c1-a6f8-9804b08fb8c2
Sep 22 23:14:28.094 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 01a6d79b-23e4-44c1-a6f8-9804b08fb8c2, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:28.094 INFO 8cdc62de-7a87-47c6-9c8c-02c45f1c00bf WaitActive WaitActive WaitActive
Sep 22 23:14:28.094 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:28.094 DEBG IO Write 1001 has deps [JobId(1000)]
Sep 22 23:14:28.095 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:28.095 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:28.095 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:28.095 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:28.095 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:28.096 INFO [0] 8cdc62de-7a87-47c6-9c8c-02c45f1c00bf (f4dd55be-06a0-4424-99ef-c92da9671dc7) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:28.096 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:14:28.096 WARN [0] new RM replaced this: None
Sep 22 23:14:28.096 INFO [0] Starts reconcile loop
Sep 22 23:14:28.096 INFO [2] 8cdc62de-7a87-47c6-9c8c-02c45f1c00bf (f4dd55be-06a0-4424-99ef-c92da9671dc7) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:28.096 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:14:28.096 WARN [2] new RM replaced this: None
Sep 22 23:14:28.096 INFO [2] Starts reconcile loop
Sep 22 23:14:28.096 INFO [1] 8cdc62de-7a87-47c6-9c8c-02c45f1c00bf (f4dd55be-06a0-4424-99ef-c92da9671dc7) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum
Sep 22 23:14:28.096 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:14:28.096 WARN [1] new RM replaced this: None
Sep 22 23:14:28.096 INFO [1] Starts reconcile loop
Sep 22 23:14:28.096 INFO [0] 127.0.0.1:46618 task reports connection:true
Sep 22 23:14:28.096 INFO 8cdc62de-7a87-47c6-9c8c-02c45f1c00bf WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:14:28.096 INFO [0]R  flush_numbers: [0, 0]
Sep 22 23:14:28.096 INFO [0]R  generation: [0, 0]
Sep 22 23:14:28.096 INFO [0]R  dirty: [false, false]
Sep 22 23:14:28.096 DEBG IO Write 1000 has deps []
Sep 22 23:14:28.096 INFO [1]R  flush_numbers: [0, 0]
Sep 22 23:14:28.096 INFO [1]R  generation: [0, 0]
Sep 22 23:14:28.096 INFO [1]R  dirty: [false, false]
Sep 22 23:14:28.096 INFO [2]R  flush_numbers: [0, 0]
Sep 22 23:14:28.096 INFO [2]R  generation: [0, 0]
Sep 22 23:14:28.096 INFO [2]R  dirty: [false, false]
Sep 22 23:14:28.096 INFO Max found gen is 1
Sep 22 23:14:28.096 INFO Generation requested: 1 >= found:1
Sep 22 23:14:28.096 INFO Next flush: 1
Sep 22 23:14:28.096 INFO All extents match
Sep 22 23:14:28.096 INFO No downstairs repair required
Sep 22 23:14:28.096 INFO No initial repair work was required
Sep 22 23:14:28.096 INFO Set Downstairs and Upstairs active
Sep 22 23:14:28.096 INFO 8cdc62de-7a87-47c6-9c8c-02c45f1c00bf is now active with session: f4dd55be-06a0-4424-99ef-c92da9671dc7
Sep 22 23:14:28.096 INFO 8cdc62de-7a87-47c6-9c8c-02c45f1c00bf Set Active after no repair
Sep 22 23:14:28.096 INFO Notify all downstairs, region set compare is done.
Sep 22 23:14:28.096 INFO Set check for repair
Sep 22 23:14:28.096 INFO [2] 127.0.0.1:52057 task reports connection:true
Sep 22 23:14:28.096 INFO 8cdc62de-7a87-47c6-9c8c-02c45f1c00bf Active Active Active
Sep 22 23:14:28.096 INFO Set check for repair
Sep 22 23:14:28.096 INFO [1] 127.0.0.1:49265 task reports connection:true
Sep 22 23:14:28.096 INFO 8cdc62de-7a87-47c6-9c8c-02c45f1c00bf Active Active Active
Sep 22 23:14:28.096 INFO Set check for repair
Sep 22 23:14:28.096 INFO [0] received reconcile message
Sep 22 23:14:28.096 INFO [0] All repairs completed, exit
Sep 22 23:14:28.096 INFO [0] Starts cmd_loop
Sep 22 23:14:28.096 INFO [1] received reconcile message
Sep 22 23:14:28.096 INFO [1] All repairs completed, exit
Sep 22 23:14:28.096 INFO [1] Starts cmd_loop
Sep 22 23:14:28.096 INFO [2] received reconcile message
Sep 22 23:14:28.096 INFO [2] All repairs completed, exit
Sep 22 23:14:28.097 INFO [2] Starts cmd_loop
The guest has finished waiting for activation
The guest has requested activation
Sep 22 23:14:28.097 INFO 23d5a1a6-b740-4ec5-ba77-4864ba7d7d72 active request set
Sep 22 23:14:28.097 INFO [0] received activate with gen 1
Sep 22 23:14:28.097 INFO [0] client got ds_active_rx, promote! session b557ec33-3655-4c40-910a-6ea3a4aafaac
Sep 22 23:14:28.097 INFO [1] received activate with gen 1
Sep 22 23:14:28.097 INFO [1] client got ds_active_rx, promote! session b557ec33-3655-4c40-910a-6ea3a4aafaac
Sep 22 23:14:28.097 INFO [2] received activate with gen 1
Sep 22 23:14:28.097 INFO [2] client got ds_active_rx, promote! session b557ec33-3655-4c40-910a-6ea3a4aafaac
Sep 22 23:14:28.097 INFO UpstairsConnection { upstairs_id: 23d5a1a6-b740-4ec5-ba77-4864ba7d7d72, session_id: b557ec33-3655-4c40-910a-6ea3a4aafaac, gen: 1 } is now active (read-write)
Sep 22 23:14:28.097 INFO UpstairsConnection { upstairs_id: 23d5a1a6-b740-4ec5-ba77-4864ba7d7d72, session_id: b557ec33-3655-4c40-910a-6ea3a4aafaac, gen: 1 } is now active (read-write)
Sep 22 23:14:28.097 INFO UpstairsConnection { upstairs_id: 23d5a1a6-b740-4ec5-ba77-4864ba7d7d72, session_id: b557ec33-3655-4c40-910a-6ea3a4aafaac, gen: 1 } is now active (read-write)
Sep 22 23:14:28.098 INFO [0] downstairs client at 127.0.0.1:35251 has UUID b901037e-6e64-43d6-a637-75008a7adf81
Sep 22 23:14:28.098 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: b901037e-6e64-43d6-a637-75008a7adf81, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:28.098 INFO 23d5a1a6-b740-4ec5-ba77-4864ba7d7d72 WaitActive WaitActive WaitActive
Sep 22 23:14:28.098 INFO [1] downstairs client at 127.0.0.1:33061 has UUID b54ba00b-56b7-4fec-8b06-0390fa4a34ff
Sep 22 23:14:28.098 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: b54ba00b-56b7-4fec-8b06-0390fa4a34ff, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:28.098 INFO 23d5a1a6-b740-4ec5-ba77-4864ba7d7d72 WaitActive WaitActive WaitActive
Sep 22 23:14:28.098 INFO [2] downstairs client at 127.0.0.1:61465 has UUID d07ff0ba-0bf5-4b77-9ec5-a7938bb906f6
Sep 22 23:14:28.098 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: d07ff0ba-0bf5-4b77-9ec5-a7938bb906f6, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:28.098 INFO 23d5a1a6-b740-4ec5-ba77-4864ba7d7d72 WaitActive WaitActive WaitActive
Sep 22 23:14:28.098 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:28.098 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:28.099 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:28.099 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:28.099 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:28.099 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:28.099 INFO [0] 23d5a1a6-b740-4ec5-ba77-4864ba7d7d72 (b557ec33-3655-4c40-910a-6ea3a4aafaac) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:28.099 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:14:28.099 WARN [0] new RM replaced this: None
Sep 22 23:14:28.099 INFO [0] Starts reconcile loop
Sep 22 23:14:28.099 INFO [1] 23d5a1a6-b740-4ec5-ba77-4864ba7d7d72 (b557ec33-3655-4c40-910a-6ea3a4aafaac) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:28.099 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:14:28.099 WARN [1] new RM replaced this: None
Sep 22 23:14:28.099 INFO [1] Starts reconcile loop
Sep 22 23:14:28.099 INFO [2] 23d5a1a6-b740-4ec5-ba77-4864ba7d7d72 (b557ec33-3655-4c40-910a-6ea3a4aafaac) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:14:28.099 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:14:28.099 WARN [2] new RM replaced this: None
Sep 22 23:14:28.100 INFO [2] Starts reconcile loop
Sep 22 23:14:28.100 INFO [0] 127.0.0.1:35251 task reports connection:true
Sep 22 23:14:28.100 INFO 23d5a1a6-b740-4ec5-ba77-4864ba7d7d72 WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:14:28.100 INFO [0]R  flush_numbers: [0, 0]
Sep 22 23:14:28.100 INFO [0]R  generation: [0, 0]
Sep 22 23:14:28.100 INFO [0]R  dirty: [false, false]
Sep 22 23:14:28.100 INFO [1]R  flush_numbers: [0, 0]
Sep 22 23:14:28.100 INFO [1]R  generation: [0, 0]
Sep 22 23:14:28.100 INFO [1]R  dirty: [false, false]
Sep 22 23:14:28.100 INFO [2]R  flush_numbers: [0, 0]
Sep 22 23:14:28.100 INFO [2]R  generation: [0, 0]
Sep 22 23:14:28.100 INFO [2]R  dirty: [false, false]
Sep 22 23:14:28.100 INFO Max found gen is 1
Sep 22 23:14:28.100 INFO Generation requested: 1 >= found:1
Sep 22 23:14:28.100 INFO Next flush: 1
Sep 22 23:14:28.100 INFO All extents match
Sep 22 23:14:28.100 INFO No downstairs repair required
Sep 22 23:14:28.100 INFO No initial repair work was required
Sep 22 23:14:28.100 INFO Set Downstairs and Upstairs active
Sep 22 23:14:28.100 INFO 23d5a1a6-b740-4ec5-ba77-4864ba7d7d72 is now active with session: b557ec33-3655-4c40-910a-6ea3a4aafaac
Sep 22 23:14:28.100 INFO 23d5a1a6-b740-4ec5-ba77-4864ba7d7d72 Set Active after no repair
Sep 22 23:14:28.100 INFO Notify all downstairs, region set compare is done.
Sep 22 23:14:28.100 INFO Set check for repair
Sep 22 23:14:28.100 INFO [1] 127.0.0.1:33061 task reports connection:true
Sep 22 23:14:28.100 INFO 23d5a1a6-b740-4ec5-ba77-4864ba7d7d72 Active Active Active
Sep 22 23:14:28.100 INFO Set check for repair
Sep 22 23:14:28.100 INFO [2] 127.0.0.1:61465 task reports connection:true
Sep 22 23:14:28.100 INFO 23d5a1a6-b740-4ec5-ba77-4864ba7d7d72 Active Active Active
Sep 22 23:14:28.100 INFO Set check for repair
Sep 22 23:14:28.100 DEBG up_ds_listen was notified
Sep 22 23:14:28.100 INFO [0] received reconcile message
Sep 22 23:14:28.100 DEBG up_ds_listen process 1001
Sep 22 23:14:28.100 INFO [0] All repairs completed, exit
Sep 22 23:14:28.100 DEBG [A] ack job 1001:2, : downstairs
Sep 22 23:14:28.100 INFO [0] Starts cmd_loop
Sep 22 23:14:28.100 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:28.100 INFO [1] received reconcile message
Sep 22 23:14:28.100 INFO [1] All repairs completed, exit
Sep 22 23:14:28.100 INFO [1] Starts cmd_loop
Sep 22 23:14:28.100 INFO [2] received reconcile message
Sep 22 23:14:28.100 INFO [2] All repairs completed, exit
Sep 22 23:14:28.100 INFO [2] Starts cmd_loop
The guest has finished waiting for activation
Sep 22 23:14:28.101 DEBG IO Write 1000 has deps []
Sep 22 23:14:28.101 DEBG IO Write 1002 has deps [JobId(1001), JobId(1000)]
Sep 22 23:14:28.101 DEBG up_ds_listen was notified
Sep 22 23:14:28.101 DEBG up_ds_listen process 1002
Sep 22 23:14:28.101 DEBG [A] ack job 1002:3, : downstairs
Sep 22 23:14:28.101 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:28.102 DEBG up_ds_listen was notified
Sep 22 23:14:28.102 DEBG up_ds_listen process 1000
Sep 22 23:14:28.102 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:14:28.102 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:28.102 DEBG IO Read  1001 has deps [JobId(1000)]
Sep 22 23:14:28.102 DEBG up_ds_listen was notified
Sep 22 23:14:28.103 DEBG up_ds_listen process 1000
Sep 22 23:14:28.103 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:14:28.103 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:28.103 DEBG Write     :1002 deps:[JobId(1001), JobId(1000)] res:true
Sep 22 23:14:28.103 DEBG IO Write 1000 has deps []
Sep 22 23:14:28.103 DEBG Read      :1001 deps:[JobId(1000)] res:true
Sep 22 23:14:28.103 DEBG Write     :1002 deps:[JobId(1001), JobId(1000)] res:true
Sep 22 23:14:28.103 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:28.103 INFO Opened existing region file "/tmp/downstairs-WRx1oCsz/region.json"
Sep 22 23:14:28.103 INFO Database read version 1
Sep 22 23:14:28.103 INFO Database write version 1
Sep 22 23:14:28.103 DEBG Write     :1002 deps:[JobId(1001), JobId(1000)] res:true
Sep 22 23:14:28.103 DEBG Read      :1001 deps:[JobId(1000)] res:true
Sep 22 23:14:28.104 DEBG Read      :1001 deps:[JobId(1000)] res:true
Sep 22 23:14:28.105 DEBG up_ds_listen was notified
Sep 22 23:14:28.105 DEBG up_ds_listen process 1000
Sep 22 23:14:28.105 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:14:28.105 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:28.105 DEBG IO Read  1001 has deps [JobId(1000)]
Sep 22 23:14:28.106 INFO UUID: bbec14f8-e0c9-4fdd-8394-c7675df3174b
Sep 22 23:14:28.106 INFO Blocks per extent:512 Total Extents: 188
Sep 22 23:14:28.106 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:28.106 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:28.106 INFO Using address: 127.0.0.1:53298, task: main
Sep 22 23:14:28.106 DEBG Read      :1001 deps:[JobId(1000)] res:true
Sep 22 23:14:28.106 DEBG Read      :1001 deps:[JobId(1000)] res:true
Sep 22 23:14:28.106 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:28.106 DEBG Read      :1001 deps:[JobId(1000)] res:true
Sep 22 23:14:28.106 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:45028, task: repair
Sep 22 23:14:28.106 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:45028, task: repair
Sep 22 23:14:28.106 INFO listening, local_addr: 127.0.0.1:45028, task: repair
Sep 22 23:14:28.106 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:45028, task: repair
Sep 22 23:14:28.106 INFO Using repair address: 127.0.0.1:45028, task: main
Sep 22 23:14:28.106 INFO No SSL acceptor configured, task: main
Sep 22 23:14:28.106 DEBG IO Write 1001 has deps [JobId(1000)]
Sep 22 23:14:28.106 DEBG [0] Read AckReady 1001, : downstairs
Sep 22 23:14:28.106 DEBG up_ds_listen was notified
Sep 22 23:14:28.107 DEBG up_ds_listen process 1001
Sep 22 23:14:28.107 DEBG [A] ack job 1001:2, : downstairs
Sep 22 23:14:28.107 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:28.107 DEBG [2] Read already AckReady 1001, : downstairs
Sep 22 23:14:28.107 DEBG [1] Read already AckReady 1001, : downstairs
Sep 22 23:14:28.107 DEBG up_ds_listen was notified
Sep 22 23:14:28.107 DEBG up_ds_listen process 1001
Sep 22 23:14:28.107 DEBG [A] ack job 1001:2, : downstairs
Sep 22 23:14:28.107 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:28.107 DEBG IO Read  1001 has deps [JobId(1000)]
Sep 22 23:14:28.108 DEBG Read      :1001 deps:[JobId(1000)] res:true
Sep 22 23:14:28.108 DEBG Read      :1001 deps:[JobId(1000)] res:true
Sep 22 23:14:28.108 DEBG Read      :1001 deps:[JobId(1000)] res:true
Sep 22 23:14:28.108 DEBG [0] Read AckReady 1001, : downstairs
Sep 22 23:14:28.109 DEBG [0] Read AckReady 1001, : downstairs
Sep 22 23:14:28.109 DEBG [1] Read already AckReady 1001, : downstairs
Sep 22 23:14:28.109 DEBG [2] Read already AckReady 1001, : downstairs
Sep 22 23:14:28.109 DEBG up_ds_listen was notified
Sep 22 23:14:28.109 DEBG IO Flush 1001 has deps [JobId(1000)]
Sep 22 23:14:28.109 DEBG up_ds_listen process 1001
Sep 22 23:14:28.109 DEBG [A] ack job 1001:2, : downstairs
Sep 22 23:14:28.109 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:28.110 DEBG Write     :1001 deps:[JobId(1000)] res:true
Sep 22 23:14:28.110 DEBG Write     :1001 deps:[JobId(1000)] res:true
Sep 22 23:14:28.110 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:1 g:1
Sep 22 23:14:28.111 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:1 g:1
Sep 22 23:14:28.111 DEBG up_ds_listen was notified
Sep 22 23:14:28.111 DEBG up_ds_listen process 1001
Sep 22 23:14:28.111 DEBG [A] ack job 1001:2, : downstairs
Sep 22 23:14:28.111 DEBG [1] Read already AckReady 1001, : downstairs
Sep 22 23:14:28.111 DEBG Write     :1001 deps:[JobId(1000)] res:true
Sep 22 23:14:28.111 DEBG [rc] retire 1001 clears [JobId(1000), JobId(1001)], : downstairs
Sep 22 23:14:28.111 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:28.111 DEBG IO Read  1003 has deps [JobId(1002), JobId(1001), JobId(1000)]
Sep 22 23:14:28.112 DEBG IO Write 1002 has deps [JobId(1001), JobId(1000)]
Sep 22 23:14:28.112 DEBG Read      :1003 deps:[JobId(1002), JobId(1001), JobId(1000)] res:true
Sep 22 23:14:28.113 DEBG Read      :1003 deps:[JobId(1002), JobId(1001), JobId(1000)] res:true
Sep 22 23:14:28.113 DEBG Read      :1003 deps:[JobId(1002), JobId(1001), JobId(1000)] res:true
Sep 22 23:14:28.114 DEBG [2] Read already AckReady 1001, : downstairs
Sep 22 23:14:28.114 DEBG up_ds_listen was notified
Sep 22 23:14:28.114 DEBG up_ds_listen process 1001
Sep 22 23:14:28.114 DEBG [A] ack job 1001:2, : downstairs
Sep 22 23:14:28.114 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:28.114 DEBG IO Read  1001 has deps [JobId(1000)]
Sep 22 23:14:28.115 DEBG up_ds_listen was notified
Sep 22 23:14:28.115 DEBG up_ds_listen process 1002
Sep 22 23:14:28.115 DEBG [A] ack job 1002:3, : downstairs
Sep 22 23:14:28.115 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:28.115 DEBG Read      :1001 deps:[JobId(1000)] res:true
Sep 22 23:14:28.116 DEBG Read      :1001 deps:[JobId(1000)] res:true
Sep 22 23:14:28.116 DEBG Read      :1001 deps:[JobId(1000)] res:true
Sep 22 23:14:28.117 DEBG IO Write 1002 has deps [JobId(1001), JobId(1000)]
Sep 22 23:14:28.117 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:28.117 INFO Created new region file "/tmp/downstairs-SvxEiywd/region.json"
Sep 22 23:14:28.118 DEBG [0] Read AckReady 1003, : downstairs
Sep 22 23:14:28.120 DEBG up_ds_listen was notified
Sep 22 23:14:28.120 DEBG up_ds_listen process 1002
Sep 22 23:14:28.120 DEBG [A] ack job 1002:3, : downstairs
Sep 22 23:14:28.120 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:28.120 DEBG IO Read  1003 has deps [JobId(1002), JobId(1000)]
Sep 22 23:14:28.120 DEBG [1] Read already AckReady 1003, : downstairs
Sep 22 23:14:28.120 DEBG [1] Read AckReady 1001, : downstairs
Sep 22 23:14:28.121 DEBG Read      :1003 deps:[JobId(1002), JobId(1000)] res:true
Sep 22 23:14:28.121 DEBG Read      :1003 deps:[JobId(1002), JobId(1000)] res:true
Sep 22 23:14:28.122 DEBG Read      :1003 deps:[JobId(1002), JobId(1000)] res:true
Sep 22 23:14:28.123 DEBG [2] Read already AckReady 1003, : downstairs
Sep 22 23:14:28.123 DEBG up_ds_listen was notified
Sep 22 23:14:28.123 DEBG [0] Read already AckReady 1001, : downstairs
Sep 22 23:14:28.123 DEBG up_ds_listen process 1003
Sep 22 23:14:28.123 DEBG [A] ack job 1003:4, : downstairs
Sep 22 23:14:28.124 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:28.124 DEBG IO Read  1002 has deps [JobId(1001), JobId(1000)]
Sep 22 23:14:28.124 DEBG [0] Read AckReady 1003, : downstairs
Sep 22 23:14:28.125 DEBG Read      :1002 deps:[JobId(1001), JobId(1000)] res:true
Sep 22 23:14:28.125 DEBG Read      :1002 deps:[JobId(1001), JobId(1000)] res:true
Sep 22 23:14:28.126 DEBG [2] Read already AckReady 1003, : downstairs
Sep 22 23:14:28.126 DEBG [2] Read already AckReady 1001, : downstairs
Sep 22 23:14:28.126 DEBG Read      :1002 deps:[JobId(1001), JobId(1000)] res:true
Sep 22 23:14:28.126 DEBG up_ds_listen was notified
Sep 22 23:14:28.126 DEBG up_ds_listen process 1001
Sep 22 23:14:28.126 DEBG [A] ack job 1001:2, : downstairs
Sep 22 23:14:28.127 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:28.128 DEBG [1] Read already AckReady 1003, : downstairs
Sep 22 23:14:28.128 DEBG up_ds_listen was notified
Sep 22 23:14:28.128 DEBG up_ds_listen process 1003
Sep 22 23:14:28.128 DEBG [A] ack job 1003:4, : downstairs
Sep 22 23:14:28.128 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:28.128 DEBG IO Read  1003 has deps [JobId(1002), JobId(1000)]
Sep 22 23:14:28.129 DEBG Read      :1003 deps:[JobId(1002), JobId(1000)] res:true
Sep 22 23:14:28.129 DEBG Read      :1003 deps:[JobId(1002), JobId(1000)] res:true
Sep 22 23:14:28.129 DEBG Read      :1003 deps:[JobId(1002), JobId(1000)] res:true
Sep 22 23:14:28.130 DEBG IO Write 1002 has deps [JobId(1001), JobId(1000)]
Sep 22 23:14:28.130 DEBG [0] Read AckReady 1002, : downstairs
Sep 22 23:14:28.132 DEBG [0] Read AckReady 1003, : downstairs
Sep 22 23:14:28.133 DEBG [1] Read already AckReady 1002, : downstairs
Sep 22 23:14:28.133 DEBG up_ds_listen was notified
Sep 22 23:14:28.133 DEBG up_ds_listen process 1002
Sep 22 23:14:28.133 DEBG [A] ack job 1002:3, : downstairs
Sep 22 23:14:28.133 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:28.134 DEBG [1] Read already AckReady 1003, : downstairs
Sep 22 23:14:28.136 DEBG [2] Read already AckReady 1003, : downstairs
Sep 22 23:14:28.136 DEBG up_ds_listen was notified
Sep 22 23:14:28.136 DEBG up_ds_listen process 1003
Sep 22 23:14:28.136 DEBG [A] ack job 1003:4, : downstairs
Sep 22 23:14:28.136 DEBG [2] Read already AckReady 1002, : downstairs
Sep 22 23:14:28.136 DEBG up_ds_listen was notified
Sep 22 23:14:28.136 DEBG up_ds_listen process 1002
Sep 22 23:14:28.136 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:28.136 DEBG [A] ack job 1002:3, : downstairs
Sep 22 23:14:28.136 DEBG IO Write 1002 has deps [JobId(1001), JobId(1000)]
Sep 22 23:14:28.136 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:28.142 DEBG up_ds_listen was notified
Sep 22 23:14:28.142 DEBG up_ds_listen process 1002
Sep 22 23:14:28.142 DEBG [A] ack job 1002:3, : downstairs
Sep 22 23:14:28.142 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:28.142 DEBG IO Read  1003 has deps [JobId(1002), JobId(1000)]
test test::integration_test_volume_write_unwritten_subvols_3 ... ok
Sep 22 23:14:28.143 DEBG Read      :1003 deps:[JobId(1002), JobId(1000)] res:true
Sep 22 23:14:28.143 INFO current number of open files limit 65536 is already the maximum
test test::integration_test_volume_write_unwritten_subvols_sparse ... ok
Sep 22 23:14:28.143 INFO Created new region file "/tmp/downstairs-6IEHRjPM/region.json"
Sep 22 23:14:28.143 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:28.143 DEBG Read      :1003 deps:[JobId(1002), JobId(1000)] res:true
Sep 22 23:14:28.143 INFO Created new region file "/tmp/downstairs-2o76BUZT/region.json"
Sep 22 23:14:28.144 DEBG Read      :1003 deps:[JobId(1002), JobId(1000)] res:true
Sep 22 23:14:28.146 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:28.147 INFO Opened existing region file "/tmp/downstairs-2o76BUZT/region.json"
Sep 22 23:14:28.147 INFO Database read version 1
Sep 22 23:14:28.147 INFO Database write version 1
Sep 22 23:14:28.147 INFO UUID: 16035766-fcf1-4aff-8be5-4662118fbeba
Sep 22 23:14:28.147 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:28.147 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:28.148 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:28.148 INFO Using address: 127.0.0.1:64713, task: main
Sep 22 23:14:28.148 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:28.148 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:43597, task: repair
Sep 22 23:14:28.148 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:43597, task: repair
Sep 22 23:14:28.148 INFO listening, local_addr: 127.0.0.1:43597, task: repair
Sep 22 23:14:28.148 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:43597, task: repair
Sep 22 23:14:28.148 INFO Using repair address: 127.0.0.1:43597, task: main
Sep 22 23:14:28.148 INFO No SSL acceptor configured, task: main
Sep 22 23:14:28.148 DEBG [0] Read AckReady 1003, : downstairs
Sep 22 23:14:28.148 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:28.148 INFO Created new region file "/tmp/downstairs-etPhPbAg/region.json"
Sep 22 23:14:28.150 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:28.151 INFO Opened existing region file "/tmp/downstairs-etPhPbAg/region.json"
Sep 22 23:14:28.151 INFO Database read version 1
Sep 22 23:14:28.151 INFO Database write version 1
Sep 22 23:14:28.151 DEBG [1] Read already AckReady 1003, : downstairs
Sep 22 23:14:28.152 INFO UUID: c1ff2b45-bb0b-4e95-bfb6-6e2ba62da531
Sep 22 23:14:28.152 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:28.152 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:28.152 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:28.152 INFO Using address: 127.0.0.1:60775, task: main
Sep 22 23:14:28.152 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:28.152 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:42948, task: repair
Sep 22 23:14:28.152 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:42948, task: repair
Sep 22 23:14:28.152 INFO listening, local_addr: 127.0.0.1:42948, task: repair
Sep 22 23:14:28.152 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:42948, task: repair
Sep 22 23:14:28.152 INFO Using repair address: 127.0.0.1:42948, task: main
Sep 22 23:14:28.152 INFO No SSL acceptor configured, task: main
Sep 22 23:14:28.152 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:28.152 INFO Created new region file "/tmp/downstairs-19iRuZ8i/region.json"
Sep 22 23:14:28.154 DEBG [2] Read already AckReady 1003, : downstairs
Sep 22 23:14:28.154 DEBG up_ds_listen was notified
Sep 22 23:14:28.154 DEBG up_ds_listen process 1003
Sep 22 23:14:28.154 DEBG [A] ack job 1003:4, : downstairs
Sep 22 23:14:28.154 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:28.155 DEBG IO Read  1003 has deps [JobId(1002), JobId(1000)]
Sep 22 23:14:28.155 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:28.155 INFO Opened existing region file "/tmp/downstairs-19iRuZ8i/region.json"
Sep 22 23:14:28.155 INFO Database read version 1
Sep 22 23:14:28.155 INFO Database write version 1
Sep 22 23:14:28.155 DEBG Read      :1003 deps:[JobId(1002), JobId(1000)] res:true
Sep 22 23:14:28.156 INFO UUID: f77cd6ac-8353-429b-8c45-d7fd309de24d
Sep 22 23:14:28.156 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:28.156 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:28.156 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:28.156 INFO Using address: 127.0.0.1:35797, task: main
Sep 22 23:14:28.156 DEBG Read      :1003 deps:[JobId(1002), JobId(1000)] res:true
Sep 22 23:14:28.156 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:28.156 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:47430, task: repair
Sep 22 23:14:28.156 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:47430, task: repair
Sep 22 23:14:28.156 INFO listening, local_addr: 127.0.0.1:47430, task: repair
Sep 22 23:14:28.156 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:47430, task: repair
Sep 22 23:14:28.156 INFO Using repair address: 127.0.0.1:47430, task: main
Sep 22 23:14:28.156 INFO No SSL acceptor configured, task: main
Sep 22 23:14:28.156 DEBG Read      :1003 deps:[JobId(1002), JobId(1000)] res:true
note: configured to log to "/dev/stdout"
{"msg":"listening","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.158273173Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"local_addr":"127.0.0.1:48931","component":"dropshot"}
{"msg":"listen IP: 127.0.0.1:48931","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.158465059Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
Sep 22 23:14:28.161 DEBG [1] Read AckReady 1003, : downstairs
Sep 22 23:14:28.163 DEBG [0] Read already AckReady 1003, : downstairs
Sep 22 23:14:28.166 DEBG [2] Read already AckReady 1003, : downstairs
Sep 22 23:14:28.166 DEBG up_ds_listen was notified
Sep 22 23:14:28.166 DEBG up_ds_listen process 1003
Sep 22 23:14:28.166 DEBG [A] ack job 1003:4, : downstairs
Sep 22 23:14:28.166 DEBG up_ds_listen checked 1 jobs, back to waiting
test test::integration_test_volume_write_unwritten_subvols ... ok
Sep 22 23:14:28.176 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:28.176 INFO Created new region file "/tmp/downstairs-57IWH8PC/region.json"
Sep 22 23:14:28.194 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:28.194 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:28.194 INFO listening on 127.0.0.1:0, task: main
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.195110105Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"local_addr":"127.0.0.1:40823","component":"dropshot","remote_addr":"127.0.0.1:53528"}
{"msg":"no entry exists for volume 6fe510e3-7bac-480b-b94c-14525ec58a40, constructing...","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.196498826Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Upstairs starts","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.196726758Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Crucible Version: BuildInfo {\n    version: \"0.0.1\",\n    git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n    git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n    git_branch: \"main\",\n    rustc_semver: \"1.70.0\",\n    rustc_channel: \"stable\",\n    rustc_host_triple: \"x86_64-unknown-illumos\",\n    rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n    cargo_triple: \"x86_64-unknown-illumos\",\n    debug: true,\n    opt_level: 0,\n}","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.196801399Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.196834837Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Crucible stats registered with UUID: b85efbb3-3a5a-4a31-b421-0eab80343c65","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.196859676Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Crucible b85efbb3-3a5a-4a31-b421-0eab80343c65 has session id: fb76dd05-e142-43e1-91db-873cbbb0ea10","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.196884174Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] connecting to 127.0.0.1:56487","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.19707803Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"0","component":"datafile"}
{"msg":"[1] connecting to 127.0.0.1:49049","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.19723821Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"1","component":"datafile"}
{"msg":"[2] connecting to 127.0.0.1:57827","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.19730125Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"2","component":"datafile"}
{"msg":"up_listen starts","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.197430872Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile","task":"up_listen"}
{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.197462231Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Flush timeout: 0.5","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.197484612Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"volume 6fe510e3-7bac-480b-b94c-14525ec58a40 constructed ok","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.197720743Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
The guest has requested activation
{"msg":"b85efbb3-3a5a-4a31-b421-0eab80343c65 active request set","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.198162527Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:28.198 INFO accepted connection from 127.0.0.1:39551, task: main
Sep 22 23:14:28.198 INFO accepted connection from 127.0.0.1:59221, task: main
Sep 22 23:14:28.198 INFO accepted connection from 127.0.0.1:36069, task: main
{"msg":"[0] b85efbb3-3a5a-4a31-b421-0eab80343c65 looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.198517261Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"0","component":"datafile"}
{"msg":"[0] Proc runs for 127.0.0.1:56487 in state New","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.198548016Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] b85efbb3-3a5a-4a31-b421-0eab80343c65 looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.198647179Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"1","component":"datafile"}
{"msg":"[1] Proc runs for 127.0.0.1:49049 in state New","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.198677659Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] b85efbb3-3a5a-4a31-b421-0eab80343c65 looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.198719313Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"2","component":"datafile"}
{"msg":"[2] Proc runs for 127.0.0.1:57827 in state New","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.198743655Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:28.198 INFO Connection request from b85efbb3-3a5a-4a31-b421-0eab80343c65 with version 4, task: proc
Sep 22 23:14:28.198 INFO upstairs UpstairsConnection { upstairs_id: b85efbb3-3a5a-4a31-b421-0eab80343c65, session_id: 3edd5ed2-e7dd-4825-829f-91f58f1773fc, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:28.199 INFO Connection request from b85efbb3-3a5a-4a31-b421-0eab80343c65 with version 4, task: proc
Sep 22 23:14:28.199 INFO upstairs UpstairsConnection { upstairs_id: b85efbb3-3a5a-4a31-b421-0eab80343c65, session_id: 3edd5ed2-e7dd-4825-829f-91f58f1773fc, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:28.199 INFO Connection request from b85efbb3-3a5a-4a31-b421-0eab80343c65 with version 4, task: proc
Sep 22 23:14:28.199 INFO upstairs UpstairsConnection { upstairs_id: b85efbb3-3a5a-4a31-b421-0eab80343c65, session_id: 3edd5ed2-e7dd-4825-829f-91f58f1773fc, gen: 1 } connected, version 4, task: proc
{"msg":"[0] b85efbb3-3a5a-4a31-b421-0eab80343c65 (3edd5ed2-e7dd-4825-829f-91f58f1773fc) New New New ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.199388329Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.199422732Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] client is_active_req TRUE, promote! session 3edd5ed2-e7dd-4825-829f-91f58f1773fc","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.199447995Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] b85efbb3-3a5a-4a31-b421-0eab80343c65 (3edd5ed2-e7dd-4825-829f-91f58f1773fc) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.199479372Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.199511268Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] client is_active_req TRUE, promote! session 3edd5ed2-e7dd-4825-829f-91f58f1773fc","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.199535874Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] b85efbb3-3a5a-4a31-b421-0eab80343c65 (3edd5ed2-e7dd-4825-829f-91f58f1773fc) WaitActive WaitActive New ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.199567007Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.199589646Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] client is_active_req TRUE, promote! session 3edd5ed2-e7dd-4825-829f-91f58f1773fc","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.199612575Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:28.199 INFO UpstairsConnection { upstairs_id: b85efbb3-3a5a-4a31-b421-0eab80343c65, session_id: 3edd5ed2-e7dd-4825-829f-91f58f1773fc, gen: 1 } is now active (read-write)
Sep 22 23:14:28.199 INFO UpstairsConnection { upstairs_id: b85efbb3-3a5a-4a31-b421-0eab80343c65, session_id: 3edd5ed2-e7dd-4825-829f-91f58f1773fc, gen: 1 } is now active (read-write)
Sep 22 23:14:28.199 INFO UpstairsConnection { upstairs_id: b85efbb3-3a5a-4a31-b421-0eab80343c65, session_id: 3edd5ed2-e7dd-4825-829f-91f58f1773fc, gen: 1 } is now active (read-write)
{"msg":"[0] downstairs client at 127.0.0.1:56487 has UUID b7c099f2-5446-4f38-bad7-8d884c3a5016","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.200430609Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: b7c099f2-5446-4f38-bad7-8d884c3a5016, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.20046522Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"b85efbb3-3a5a-4a31-b421-0eab80343c65 WaitActive WaitActive WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.200489666Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] downstairs client at 127.0.0.1:49049 has UUID 2fe4896e-5871-495b-8a9e-a2c5b4173cd5","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.200531969Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 2fe4896e-5871-495b-8a9e-a2c5b4173cd5, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.200564516Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"b85efbb3-3a5a-4a31-b421-0eab80343c65 WaitActive WaitActive WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.200589367Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] downstairs client at 127.0.0.1:57827 has UUID adbabfb8-1e80-4905-a6ee-914c6aaac853","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.200624598Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: adbabfb8-1e80-4905-a6ee-914c6aaac853, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.200652691Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"b85efbb3-3a5a-4a31-b421-0eab80343c65 WaitActive WaitActive WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.200675802Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:28.200 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:28.201 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:28.201 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:28.201 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:28.202 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:28.202 INFO Downstairs has completed Negotiation, task: proc
{"msg":"[0] b85efbb3-3a5a-4a31-b421-0eab80343c65 (3edd5ed2-e7dd-4825-829f-91f58f1773fc) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.20246141Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.202515617Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:14:28.202552257Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.202586633Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] b85efbb3-3a5a-4a31-b421-0eab80343c65 (3edd5ed2-e7dd-4825-829f-91f58f1773fc) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.202625674Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.202661284Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:14:28.202695288Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.202732447Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] b85efbb3-3a5a-4a31-b421-0eab80343c65 (3edd5ed2-e7dd-4825-829f-91f58f1773fc) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.202777611Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.202828174Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
The guest has finished waiting for activation
{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:14:28.202868024Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.202905663Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] 127.0.0.1:56487 task reports connection:true","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.202942124Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"b85efbb3-3a5a-4a31-b421-0eab80343c65 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.202982054Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0]R  flush_numbers: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.20302215Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0]R  generation: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.203059605Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0]R  dirty: [false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.203095055Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1]R  flush_numbers: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.203129377Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1]R  generation: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.203165606Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1]R  dirty: [false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.203204311Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2]R  flush_numbers: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.20323945Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2]R  generation: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.203274376Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2]R  dirty: [false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.203308565Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Max found gen is 1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.203340851Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.203375035Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Next flush: 1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.203408508Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"All extents match","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.203442021Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"No downstairs repair required","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.203475669Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"No initial repair work was required","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.203515798Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.203550755Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"b85efbb3-3a5a-4a31-b421-0eab80343c65 is now active with session: 3edd5ed2-e7dd-4825-829f-91f58f1773fc","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.203587832Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"b85efbb3-3a5a-4a31-b421-0eab80343c65 Set Active after no repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.203623474Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.203658826Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Set check for repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.20369254Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] 127.0.0.1:49049 task reports connection:true","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.20372862Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"b85efbb3-3a5a-4a31-b421-0eab80343c65 Active Active Active","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.203773652Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Set check for repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.203813414Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] 127.0.0.1:57827 task reports connection:true","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.203855211Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"b85efbb3-3a5a-4a31-b421-0eab80343c65 Active Active Active","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.203895026Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Set check for repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.203934461Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] received reconcile message","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.203972279Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.204007679Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.204041413Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] received reconcile message","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.204076736Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.204112332Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.204146535Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] received reconcile message","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.204187055Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.204222314Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.204256287Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"volume 6fe510e3-7bac-480b-b94c-14525ec58a40 activated ok","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.204291151Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"volume 6fe510e3-7bac-480b-b94c-14525ec58a40 constructed and inserted ok","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.204326055Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.204358999Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/6fe510e3-7bac-480b-b94c-14525ec58a40","method":"POST","req_id":"28f2ad76-6d7c-4684-bffe-a57312bf354d","remote_addr":"127.0.0.1:53528","local_addr":"127.0.0.1:40823","component":"dropshot","latency_us":7859,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.205872776Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/6fe510e3-7bac-480b-b94c-14525ec58a40/bulk_write","method":"POST","req_id":"4cb6ab2d-f8d2-4eb6-ba1c-cc2664672daf","remote_addr":"127.0.0.1:53528","local_addr":"127.0.0.1:40823","component":"dropshot","latency_us":1506,"response_code":"204"}
Sep 22 23:14:28.206 DEBG Write     :1000 deps:[] res:true
Sep 22 23:14:28.207 DEBG Write     :1000 deps:[] res:true
Sep 22 23:14:28.208 DEBG Write     :1000 deps:[] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.210783775Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/6fe510e3-7bac-480b-b94c-14525ec58a40/bulk_write","method":"POST","req_id":"6b505b2f-ed30-40b2-9197-9bceb8c3a10a","remote_addr":"127.0.0.1:53528","local_addr":"127.0.0.1:40823","component":"dropshot","latency_us":1428,"response_code":"204"}
Sep 22 23:14:28.211 DEBG Write     :1001 deps:[] res:true
Sep 22 23:14:28.211 DEBG Write     :1001 deps:[] res:true
Sep 22 23:14:28.211 DEBG Write     :1001 deps:[] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.214157417Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/6fe510e3-7bac-480b-b94c-14525ec58a40/bulk_write","method":"POST","req_id":"5f819979-a240-430c-946a-e3d6e048b24b","remote_addr":"127.0.0.1:53528","local_addr":"127.0.0.1:40823","component":"dropshot","latency_us":1359,"response_code":"204"}
Sep 22 23:14:28.214 DEBG Write     :1002 deps:[] res:true
Sep 22 23:14:28.215 DEBG Write     :1002 deps:[] res:true
Sep 22 23:14:28.215 DEBG Write     :1002 deps:[] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.217608594Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/6fe510e3-7bac-480b-b94c-14525ec58a40/bulk_write","method":"POST","req_id":"45f4ff80-c5b9-40be-bb86-838fb49f9c78","remote_addr":"127.0.0.1:53528","local_addr":"127.0.0.1:40823","component":"dropshot","latency_us":1425,"response_code":"204"}
Sep 22 23:14:28.218 DEBG Write     :1003 deps:[] res:true
Sep 22 23:14:28.218 DEBG Write     :1003 deps:[] res:true
Sep 22 23:14:28.218 DEBG Write     :1003 deps:[] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.220362468Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/6fe510e3-7bac-480b-b94c-14525ec58a40/bulk_write","method":"POST","req_id":"c64b56e1-3c54-4f6b-86f2-3a16e9ed07cb","remote_addr":"127.0.0.1:53528","local_addr":"127.0.0.1:40823","component":"dropshot","latency_us":1033,"response_code":"204"}
Sep 22 23:14:28.220 DEBG Write     :1004 deps:[] res:true
Sep 22 23:14:28.221 DEBG Write     :1004 deps:[] res:true
Sep 22 23:14:28.221 DEBG Write     :1004 deps:[] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.223601754Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/6fe510e3-7bac-480b-b94c-14525ec58a40/bulk_write","method":"POST","req_id":"5b81c2ae-f65c-4c0b-b4d6-239be7a0b488","remote_addr":"127.0.0.1:53528","local_addr":"127.0.0.1:40823","component":"dropshot","latency_us":1218,"response_code":"204"}
Sep 22 23:14:28.224 DEBG Write     :1005 deps:[] res:true
Sep 22 23:14:28.225 DEBG Write     :1005 deps:[] res:true
Sep 22 23:14:28.225 DEBG Write     :1005 deps:[] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.228285027Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/6fe510e3-7bac-480b-b94c-14525ec58a40/bulk_write","method":"POST","req_id":"2c6a1296-0cd8-4fe0-b447-fd1566f38f7f","remote_addr":"127.0.0.1:53528","local_addr":"127.0.0.1:40823","component":"dropshot","latency_us":1643,"response_code":"204"}
Sep 22 23:14:28.228 DEBG Write     :1006 deps:[] res:true
Sep 22 23:14:28.229 DEBG Write     :1006 deps:[] res:true
Sep 22 23:14:28.229 DEBG Write     :1006 deps:[] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.23175651Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/6fe510e3-7bac-480b-b94c-14525ec58a40/bulk_write","method":"POST","req_id":"dbdb70f4-870e-4754-a60c-3ebfccf17d1d","remote_addr":"127.0.0.1:53528","local_addr":"127.0.0.1:40823","component":"dropshot","latency_us":1462,"response_code":"204"}
Sep 22 23:14:28.232 DEBG Write     :1007 deps:[] res:true
Sep 22 23:14:28.232 DEBG Write     :1007 deps:[] res:true
Sep 22 23:14:28.232 DEBG Write     :1007 deps:[] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.235174055Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/6fe510e3-7bac-480b-b94c-14525ec58a40/bulk_write","method":"POST","req_id":"d2c74d6c-f974-4944-9b53-05e4cac458e1","remote_addr":"127.0.0.1:53528","local_addr":"127.0.0.1:40823","component":"dropshot","latency_us":1396,"response_code":"204"}
Sep 22 23:14:28.235 DEBG Write     :1008 deps:[] res:true
Sep 22 23:14:28.236 DEBG Write     :1008 deps:[] res:true
Sep 22 23:14:28.236 DEBG Write     :1008 deps:[] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.238610078Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/6fe510e3-7bac-480b-b94c-14525ec58a40/bulk_write","method":"POST","req_id":"7bc63b22-9568-43af-a3e7-c5147dc5aed1","remote_addr":"127.0.0.1:53528","local_addr":"127.0.0.1:40823","component":"dropshot","latency_us":1410,"response_code":"204"}
Sep 22 23:14:28.239 DEBG Write     :1009 deps:[] res:true
Sep 22 23:14:28.239 DEBG Write     :1009 deps:[] res:true
Sep 22 23:14:28.239 DEBG Write     :1009 deps:[] res:true
Sep 22 23:14:28.241 DEBG Read      :1010 deps:[JobId(1000)] res:true
Sep 22 23:14:28.242 DEBG Read      :1010 deps:[JobId(1000)] res:true
Sep 22 23:14:28.242 DEBG Read      :1010 deps:[JobId(1000)] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.24368874Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/6fe510e3-7bac-480b-b94c-14525ec58a40/bulk_read","method":"POST","req_id":"cb5bd336-164d-4cab-b2a9-09aef8439cfa","remote_addr":"127.0.0.1:53528","local_addr":"127.0.0.1:40823","component":"dropshot","latency_us":3061,"response_code":"200"}
Sep 22 23:14:28.245 DEBG Read      :1011 deps:[JobId(1001)] res:true
Sep 22 23:14:28.245 DEBG Read      :1011 deps:[JobId(1001)] res:true
Sep 22 23:14:28.245 DEBG Read      :1011 deps:[JobId(1001)] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.246839359Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/6fe510e3-7bac-480b-b94c-14525ec58a40/bulk_read","method":"POST","req_id":"2835b3cd-bb7c-4df5-a892-dcef0bc0c99c","remote_addr":"127.0.0.1:53528","local_addr":"127.0.0.1:40823","component":"dropshot","latency_us":2488,"response_code":"200"}
Sep 22 23:14:28.248 DEBG Read      :1012 deps:[JobId(1002)] res:true
Sep 22 23:14:28.248 DEBG Read      :1012 deps:[JobId(1002)] res:true
Sep 22 23:14:28.248 DEBG Read      :1012 deps:[JobId(1002)] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.250227865Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/6fe510e3-7bac-480b-b94c-14525ec58a40/bulk_read","method":"POST","req_id":"036e364f-528a-4ac9-831f-5ab654aa43aa","remote_addr":"127.0.0.1:53528","local_addr":"127.0.0.1:40823","component":"dropshot","latency_us":2804,"response_code":"200"}
Sep 22 23:14:28.252 DEBG Read      :1013 deps:[JobId(1003)] res:true
Sep 22 23:14:28.252 DEBG Read      :1013 deps:[JobId(1003)] res:true
Sep 22 23:14:28.252 DEBG Read      :1013 deps:[JobId(1003)] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.254711515Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/6fe510e3-7bac-480b-b94c-14525ec58a40/bulk_read","method":"POST","req_id":"e051b346-5530-40ed-8149-212b79f50021","remote_addr":"127.0.0.1:53528","local_addr":"127.0.0.1:40823","component":"dropshot","latency_us":3781,"response_code":"200"}
Sep 22 23:14:28.256 DEBG Read      :1014 deps:[JobId(1004)] res:true
Sep 22 23:14:28.256 DEBG Read      :1014 deps:[JobId(1004)] res:true
Sep 22 23:14:28.257 DEBG Read      :1014 deps:[JobId(1004)] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.259149854Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/6fe510e3-7bac-480b-b94c-14525ec58a40/bulk_read","method":"POST","req_id":"8d900b84-f6c9-492c-ba5c-e966bc77df92","remote_addr":"127.0.0.1:53528","local_addr":"127.0.0.1:40823","component":"dropshot","latency_us":3646,"response_code":"200"}
Sep 22 23:14:28.259 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:28.259 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:28.259 INFO listening on 127.0.0.1:0, task: main
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.259824372Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"local_addr":"127.0.0.1:48931","component":"dropshot","remote_addr":"127.0.0.1:35367"}
{"msg":"no entry exists for volume 78899c23-7806-4736-ba33-b5f432ab026e, constructing...","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.260455931Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Upstairs starts","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.260667878Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Crucible Version: BuildInfo {\n    version: \"0.0.1\",\n    git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n    git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n    git_branch: \"main\",\n    rustc_semver: \"1.70.0\",\n    rustc_channel: \"stable\",\n    rustc_host_triple: \"x86_64-unknown-illumos\",\n    rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n    cargo_triple: \"x86_64-unknown-illumos\",\n    debug: true,\n    opt_level: 0,\n}","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.260740832Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.260774455Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Crucible stats registered with UUID: aa7cbd55-9585-444d-917b-34b9d1d5a423","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.260799602Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Crucible aa7cbd55-9585-444d-917b-34b9d1d5a423 has session id: 7a7229f8-e775-4acb-96eb-89fbc1a08db9","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.26082389Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] connecting to 127.0.0.1:64713","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.260989017Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"0","component":"datafile"}
Sep 22 23:14:28.261 DEBG Read      :1015 deps:[JobId(1005)] res:true
{"msg":"[1] connecting to 127.0.0.1:60775","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.261110521Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"1","component":"datafile"}
Sep 22 23:14:28.261 DEBG Read      :1015 deps:[JobId(1005)] res:true
{"msg":"[2] connecting to 127.0.0.1:35797","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.261300018Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"2","component":"datafile"}
{"msg":"up_listen starts","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.261379985Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile","task":"up_listen"}
{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.261424542Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Flush timeout: 0.5","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.261461264Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:28.261 DEBG Read      :1015 deps:[JobId(1005)] res:true
{"msg":"volume 78899c23-7806-4736-ba33-b5f432ab026e constructed ok","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.261668647Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
The guest has requested activation
{"msg":"aa7cbd55-9585-444d-917b-34b9d1d5a423 active request set","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.261970024Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:28.262 INFO accepted connection from 127.0.0.1:59929, task: main
Sep 22 23:14:28.262 INFO accepted connection from 127.0.0.1:32800, task: main
Sep 22 23:14:28.262 INFO accepted connection from 127.0.0.1:55758, task: main
{"msg":"[0] aa7cbd55-9585-444d-917b-34b9d1d5a423 looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.26248176Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"0","component":"datafile"}
{"msg":"[0] Proc runs for 127.0.0.1:64713 in state New","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.262526302Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] aa7cbd55-9585-444d-917b-34b9d1d5a423 looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.26256904Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"1","component":"datafile"}
{"msg":"[1] Proc runs for 127.0.0.1:60775 in state New","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.262615124Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] aa7cbd55-9585-444d-917b-34b9d1d5a423 looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.262653615Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"2","component":"datafile"}
{"msg":"[2] Proc runs for 127.0.0.1:35797 in state New","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.2626963Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:28.262 INFO Connection request from aa7cbd55-9585-444d-917b-34b9d1d5a423 with version 4, task: proc
Sep 22 23:14:28.262 INFO upstairs UpstairsConnection { upstairs_id: aa7cbd55-9585-444d-917b-34b9d1d5a423, session_id: ae2b86c4-327d-41e8-b04c-5803e0cf8d60, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:28.262 INFO Connection request from aa7cbd55-9585-444d-917b-34b9d1d5a423 with version 4, task: proc
Sep 22 23:14:28.262 INFO upstairs UpstairsConnection { upstairs_id: aa7cbd55-9585-444d-917b-34b9d1d5a423, session_id: ae2b86c4-327d-41e8-b04c-5803e0cf8d60, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:28.263 INFO Connection request from aa7cbd55-9585-444d-917b-34b9d1d5a423 with version 4, task: proc
Sep 22 23:14:28.263 INFO upstairs UpstairsConnection { upstairs_id: aa7cbd55-9585-444d-917b-34b9d1d5a423, session_id: ae2b86c4-327d-41e8-b04c-5803e0cf8d60, gen: 1 } connected, version 4, task: proc
{"msg":"[0] aa7cbd55-9585-444d-917b-34b9d1d5a423 (ae2b86c4-327d-41e8-b04c-5803e0cf8d60) New New New ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.263303731Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.26335275Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] client is_active_req TRUE, promote! session ae2b86c4-327d-41e8-b04c-5803e0cf8d60","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.263391342Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] aa7cbd55-9585-444d-917b-34b9d1d5a423 (ae2b86c4-327d-41e8-b04c-5803e0cf8d60) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.263418644Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.26344475Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] client is_active_req TRUE, promote! session ae2b86c4-327d-41e8-b04c-5803e0cf8d60","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.263472809Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] aa7cbd55-9585-444d-917b-34b9d1d5a423 (ae2b86c4-327d-41e8-b04c-5803e0cf8d60) WaitActive WaitActive New ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.263497778Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.263520181Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] client is_active_req TRUE, promote! session ae2b86c4-327d-41e8-b04c-5803e0cf8d60","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.263550559Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.263565864Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/6fe510e3-7bac-480b-b94c-14525ec58a40/bulk_read","method":"POST","req_id":"202bf347-4761-4970-a482-1249a249a5ba","remote_addr":"127.0.0.1:53528","local_addr":"127.0.0.1:40823","component":"dropshot","latency_us":3706,"response_code":"200"}
Sep 22 23:14:28.263 INFO UpstairsConnection { upstairs_id: aa7cbd55-9585-444d-917b-34b9d1d5a423, session_id: ae2b86c4-327d-41e8-b04c-5803e0cf8d60, gen: 1 } is now active (read-write)
Sep 22 23:14:28.263 INFO UpstairsConnection { upstairs_id: aa7cbd55-9585-444d-917b-34b9d1d5a423, session_id: ae2b86c4-327d-41e8-b04c-5803e0cf8d60, gen: 1 } is now active (read-write)
Sep 22 23:14:28.263 INFO UpstairsConnection { upstairs_id: aa7cbd55-9585-444d-917b-34b9d1d5a423, session_id: ae2b86c4-327d-41e8-b04c-5803e0cf8d60, gen: 1 } is now active (read-write)
{"msg":"[0] downstairs client at 127.0.0.1:64713 has UUID 16035766-fcf1-4aff-8be5-4662118fbeba","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.264450589Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 16035766-fcf1-4aff-8be5-4662118fbeba, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.26449189Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"aa7cbd55-9585-444d-917b-34b9d1d5a423 WaitActive WaitActive WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.264519532Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] downstairs client at 127.0.0.1:60775 has UUID c1ff2b45-bb0b-4e95-bfb6-6e2ba62da531","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.264628832Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: c1ff2b45-bb0b-4e95-bfb6-6e2ba62da531, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.264663615Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"aa7cbd55-9585-444d-917b-34b9d1d5a423 WaitActive WaitActive WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.264687571Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] downstairs client at 127.0.0.1:35797 has UUID f77cd6ac-8353-429b-8c45-d7fd309de24d","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.264724651Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: f77cd6ac-8353-429b-8c45-d7fd309de24d, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.264753032Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"aa7cbd55-9585-444d-917b-34b9d1d5a423 WaitActive WaitActive WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.264787492Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:28.265 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:28.265 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:28.265 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:28.265 DEBG Read      :1016 deps:[JobId(1006)] res:true
Sep 22 23:14:28.265 DEBG Read      :1016 deps:[JobId(1006)] res:true
Sep 22 23:14:28.265 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:28.265 DEBG Read      :1016 deps:[JobId(1006)] res:true
Sep 22 23:14:28.266 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:28.266 INFO Downstairs has completed Negotiation, task: proc
{"msg":"[0] aa7cbd55-9585-444d-917b-34b9d1d5a423 (ae2b86c4-327d-41e8-b04c-5803e0cf8d60) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.266531941Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.266562073Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:14:28.266584945Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.266606605Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] aa7cbd55-9585-444d-917b-34b9d1d5a423 (ae2b86c4-327d-41e8-b04c-5803e0cf8d60) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.266635252Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.266662381Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:14:28.266689194Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.266726514Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] aa7cbd55-9585-444d-917b-34b9d1d5a423 (ae2b86c4-327d-41e8-b04c-5803e0cf8d60) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.26675769Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.266780673Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:14:28.266802606Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.26682418Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] 127.0.0.1:64713 task reports connection:true","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.266846254Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"aa7cbd55-9585-444d-917b-34b9d1d5a423 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.266869026Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0]R  flush_numbers: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.266890953Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0]R  generation: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.266912775Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0]R  dirty: [false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.266934627Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1]R  flush_numbers: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.266961895Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1]R  generation: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.266984375Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1]R  dirty: [false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.267005866Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2]R  flush_numbers: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.267027133Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2]R  generation: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.267048425Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
The guest has finished waiting for activation
{"msg":"[2]R  dirty: [false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.267073013Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Max found gen is 1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.267109371Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.267140317Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Next flush: 1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.267162805Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"All extents match","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.267184152Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"No downstairs repair required","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.267213617Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"No initial repair work was required","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.267237655Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.267265656Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"aa7cbd55-9585-444d-917b-34b9d1d5a423 is now active with session: ae2b86c4-327d-41e8-b04c-5803e0cf8d60","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.26729652Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"aa7cbd55-9585-444d-917b-34b9d1d5a423 Set Active after no repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.26732004Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.267342276Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Set check for repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.267363792Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] 127.0.0.1:60775 task reports connection:true","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.26738584Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"aa7cbd55-9585-444d-917b-34b9d1d5a423 Active Active Active","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.26741803Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Set check for repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.267455542Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] 127.0.0.1:35797 task reports connection:true","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.267489674Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"aa7cbd55-9585-444d-917b-34b9d1d5a423 Active Active Active","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.267519909Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Set check for repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.267546198Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] received reconcile message","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.26758259Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.267619505Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.267654169Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] received reconcile message","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.26767807Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.267711928Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.267736186Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] received reconcile message","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.267757931Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.267793265Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.267788029Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/6fe510e3-7bac-480b-b94c-14525ec58a40/bulk_read","method":"POST","req_id":"e6f4af19-4ca5-49d6-8f8c-d6d166ba5df7","remote_addr":"127.0.0.1:53528","local_addr":"127.0.0.1:40823","component":"dropshot","latency_us":3481,"response_code":"200"}
{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.267827436Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"volume 78899c23-7806-4736-ba33-b5f432ab026e activated ok","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.267862356Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"volume 78899c23-7806-4736-ba33-b5f432ab026e constructed and inserted ok","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.267899531Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.267936379Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/78899c23-7806-4736-ba33-b5f432ab026e","method":"POST","req_id":"19c572f7-03f0-43d4-bd63-d56ceab01e33","remote_addr":"127.0.0.1:35367","local_addr":"127.0.0.1:48931","component":"dropshot","latency_us":7420,"response_code":"200"}
Sep 22 23:14:28.269 DEBG Read      :1017 deps:[JobId(1007)] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.26979144Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/78899c23-7806-4736-ba33-b5f432ab026e/bulk_write","method":"POST","req_id":"b4807276-cc2f-443d-88e4-ebee284d6639","remote_addr":"127.0.0.1:35367","local_addr":"127.0.0.1:48931","component":"dropshot","latency_us":1450,"response_code":"204"}
Sep 22 23:14:28.269 DEBG Read      :1017 deps:[JobId(1007)] res:true
Sep 22 23:14:28.270 DEBG Read      :1017 deps:[JobId(1007)] res:true
Sep 22 23:14:28.270 DEBG Write     :1000 deps:[] res:true
Sep 22 23:14:28.271 DEBG Write     :1000 deps:[] res:true
Sep 22 23:14:28.271 DEBG Write     :1000 deps:[] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.272063308Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/6fe510e3-7bac-480b-b94c-14525ec58a40/bulk_read","method":"POST","req_id":"fc4d52af-0e45-4704-9688-9261bf81c61d","remote_addr":"127.0.0.1:53528","local_addr":"127.0.0.1:40823","component":"dropshot","latency_us":3548,"response_code":"200"}
Sep 22 23:14:28.273 INFO UUID: 353b066e-6f4d-4802-9336-0f9c4c2ea130
Sep 22 23:14:28.273 INFO Blocks per extent:512 Total Extents: 188
Sep 22 23:14:28.273 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:28.273 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:28.273 INFO Using address: 127.0.0.1:42762, task: main
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.273976234Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/78899c23-7806-4736-ba33-b5f432ab026e/bulk_write","method":"POST","req_id":"de44c06f-8f20-42e9-9aec-536641baa80f","remote_addr":"127.0.0.1:35367","local_addr":"127.0.0.1:48931","component":"dropshot","latency_us":1397,"response_code":"204"}
Sep 22 23:14:28.274 DEBG Read      :1018 deps:[JobId(1008)] res:true
Sep 22 23:14:28.274 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:28.274 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:43667, task: repair
Sep 22 23:14:28.274 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:43667, task: repair
Sep 22 23:14:28.274 DEBG Read      :1018 deps:[JobId(1008)] res:true
Sep 22 23:14:28.274 INFO listening, local_addr: 127.0.0.1:43667, task: repair
Sep 22 23:14:28.274 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:43667, task: repair
Sep 22 23:14:28.274 DEBG Read      :1018 deps:[JobId(1008)] res:true
Sep 22 23:14:28.274 INFO Using repair address: 127.0.0.1:43667, task: main
Sep 22 23:14:28.274 INFO No SSL acceptor configured, task: main
Sep 22 23:14:28.274 DEBG Write     :1001 deps:[] res:true
Sep 22 23:14:28.274 DEBG Write     :1001 deps:[] res:true
Sep 22 23:14:28.275 DEBG Write     :1001 deps:[] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.27656753Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/6fe510e3-7bac-480b-b94c-14525ec58a40/bulk_read","method":"POST","req_id":"2ffb3177-418d-4c22-83a8-4a21e9901ec7","remote_addr":"127.0.0.1:53528","local_addr":"127.0.0.1:40823","component":"dropshot","latency_us":3788,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.277442166Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/78899c23-7806-4736-ba33-b5f432ab026e/bulk_write","method":"POST","req_id":"8c0f2aa4-9126-4c3f-93d1-2a01202558d6","remote_addr":"127.0.0.1:35367","local_addr":"127.0.0.1:48931","component":"dropshot","latency_us":1445,"response_code":"204"}
Sep 22 23:14:28.278 DEBG Write     :1002 deps:[] res:true
Sep 22 23:14:28.278 DEBG Write     :1002 deps:[] res:true
Sep 22 23:14:28.278 DEBG Read      :1019 deps:[JobId(1009)] res:true
Sep 22 23:14:28.278 DEBG Write     :1002 deps:[] res:true
Sep 22 23:14:28.278 DEBG Read      :1019 deps:[JobId(1009)] res:true
Sep 22 23:14:28.279 DEBG Read      :1019 deps:[JobId(1009)] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.280870502Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/78899c23-7806-4736-ba33-b5f432ab026e/bulk_write","method":"POST","req_id":"a8d9cb68-a0a0-4197-b8a4-37b3cb438873","remote_addr":"127.0.0.1:35367","local_addr":"127.0.0.1:48931","component":"dropshot","latency_us":1383,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.280992366Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/6fe510e3-7bac-480b-b94c-14525ec58a40/bulk_read","method":"POST","req_id":"a09ab62e-8daf-4a1a-b39a-e816bd60aa31","remote_addr":"127.0.0.1:53528","local_addr":"127.0.0.1:40823","component":"dropshot","latency_us":3657,"response_code":"200"}
Sep 22 23:14:28.281 DEBG Write     :1003 deps:[] res:true
Sep 22 23:14:28.282 DEBG Write     :1003 deps:[] res:true
Sep 22 23:14:28.282 DEBG Write     :1003 deps:[] res:true
Sep 22 23:14:28.283 DEBG Read      :1020 deps:[JobId(1009), JobId(1008), JobId(1007), JobId(1006), JobId(1005), JobId(1004), JobId(1003), JobId(1002), JobId(1001), JobId(1000)] res:true
Sep 22 23:14:28.283 DEBG Read      :1020 deps:[JobId(1009), JobId(1008), JobId(1007), JobId(1006), JobId(1005), JobId(1004), JobId(1003), JobId(1002), JobId(1001), JobId(1000)] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.284760071Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/78899c23-7806-4736-ba33-b5f432ab026e/bulk_write","method":"POST","req_id":"d8efcda3-7c48-4e3e-b200-b7b11e9cb39a","remote_addr":"127.0.0.1:35367","local_addr":"127.0.0.1:48931","component":"dropshot","latency_us":1304,"response_code":"204"}
Sep 22 23:14:28.284 DEBG Read      :1020 deps:[JobId(1009), JobId(1008), JobId(1007), JobId(1006), JobId(1005), JobId(1004), JobId(1003), JobId(1002), JobId(1001), JobId(1000)] res:true
Sep 22 23:14:28.285 DEBG Write     :1004 deps:[] res:true
Sep 22 23:14:28.285 DEBG Write     :1004 deps:[] res:true
Sep 22 23:14:28.286 DEBG Write     :1004 deps:[] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.288351068Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/78899c23-7806-4736-ba33-b5f432ab026e/bulk_write","method":"POST","req_id":"357853c6-a4d0-4093-acd2-4d1a56b42f8e","remote_addr":"127.0.0.1:35367","local_addr":"127.0.0.1:48931","component":"dropshot","latency_us":1437,"response_code":"204"}
Sep 22 23:14:28.289 DEBG Write     :1005 deps:[] res:true
Sep 22 23:14:28.289 DEBG Write     :1005 deps:[] res:true
Sep 22 23:14:28.290 DEBG Write     :1005 deps:[] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.292109395Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/78899c23-7806-4736-ba33-b5f432ab026e/bulk_write","method":"POST","req_id":"cd37d740-4ea4-43be-90cb-50f002e4d108","remote_addr":"127.0.0.1:35367","local_addr":"127.0.0.1:48931","component":"dropshot","latency_us":1093,"response_code":"204"}
Sep 22 23:14:28.292 DEBG Write     :1006 deps:[] res:true
Sep 22 23:14:28.292 DEBG Write     :1006 deps:[] res:true
Sep 22 23:14:28.293 DEBG Write     :1006 deps:[] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.294909615Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/78899c23-7806-4736-ba33-b5f432ab026e/bulk_write","method":"POST","req_id":"f8b62fce-60f5-4dc2-b374-de3d17bd5aa5","remote_addr":"127.0.0.1:35367","local_addr":"127.0.0.1:48931","component":"dropshot","latency_us":1099,"response_code":"204"}
Sep 22 23:14:28.295 DEBG Write     :1007 deps:[] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.295635Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/6fe510e3-7bac-480b-b94c-14525ec58a40/bulk_read","method":"POST","req_id":"d62a3f4f-b3fb-4434-bb77-20b0a7a8a18d","remote_addr":"127.0.0.1:53528","local_addr":"127.0.0.1:40823","component":"dropshot","latency_us":13946,"response_code":"200"}
Sep 22 23:14:28.295 DEBG Write     :1007 deps:[] res:true
Sep 22 23:14:28.296 DEBG Write     :1007 deps:[] res:true
{"msg":"detach removing entry for volume 6fe510e3-7bac-480b-b94c-14525ec58a40","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.296593745Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"detaching volume 6fe510e3-7bac-480b-b94c-14525ec58a40","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.296652051Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.297904988Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/78899c23-7806-4736-ba33-b5f432ab026e/bulk_write","method":"POST","req_id":"19414047-b377-4324-b1d4-9213ffd1493c","remote_addr":"127.0.0.1:35367","local_addr":"127.0.0.1:48931","component":"dropshot","latency_us":1041,"response_code":"204"}
Sep 22 23:14:28.298 DEBG Write     :1008 deps:[] res:true
Sep 22 23:14:28.298 DEBG Write     :1008 deps:[] res:true
Sep 22 23:14:28.298 DEBG Write     :1008 deps:[] res:true
Sep 22 23:14:28.299 DEBG Flush     :1021 extent_limit None deps:[JobId(1020), JobId(1019), JobId(1018), JobId(1017), JobId(1016), JobId(1015), JobId(1014), JobId(1013), JobId(1012), JobId(1011), JobId(1010), JobId(1009), JobId(1008), JobId(1007), JobId(1006), JobId(1005), JobId(1004), JobId(1003), JobId(1002), JobId(1001), JobId(1000)] res:true f:1 g:1
Sep 22 23:14:28.299 DEBG Flush     :1021 extent_limit None deps:[JobId(1020), JobId(1019), JobId(1018), JobId(1017), JobId(1016), JobId(1015), JobId(1014), JobId(1013), JobId(1012), JobId(1011), JobId(1010), JobId(1009), JobId(1008), JobId(1007), JobId(1006), JobId(1005), JobId(1004), JobId(1003), JobId(1002), JobId(1001), JobId(1000)] res:true f:1 g:1
Sep 22 23:14:28.299 DEBG Flush     :1021 extent_limit None deps:[JobId(1020), JobId(1019), JobId(1018), JobId(1017), JobId(1016), JobId(1015), JobId(1014), JobId(1013), JobId(1012), JobId(1011), JobId(1010), JobId(1009), JobId(1008), JobId(1007), JobId(1006), JobId(1005), JobId(1004), JobId(1003), JobId(1002), JobId(1001), JobId(1000)] res:true f:1 g:1
{"msg":"Request to deactivate this guest","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.300346754Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"b85efbb3-3a5a-4a31-b421-0eab80343c65 set deactivating.","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.300399532Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.300625758Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/6fe510e3-7bac-480b-b94c-14525ec58a40","method":"DELETE","req_id":"d3793d99-d440-407b-a757-f12e279e16cd","remote_addr":"127.0.0.1:53528","local_addr":"127.0.0.1:40823","component":"dropshot","latency_us":4163,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.300754553Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/78899c23-7806-4736-ba33-b5f432ab026e/bulk_write","method":"POST","req_id":"905ef482-6214-4e25-9c4c-6772b9478fd6","remote_addr":"127.0.0.1:35367","local_addr":"127.0.0.1:48931","component":"dropshot","latency_us":1128,"response_code":"204"}
Sep 22 23:14:28.301 DEBG Write     :1009 deps:[] res:true
Sep 22 23:14:28.301 DEBG Write     :1009 deps:[] res:true
Sep 22 23:14:28.301 DEBG Write     :1009 deps:[] res:true
{"msg":"detach removing entry for volume 78899c23-7806-4736-ba33-b5f432ab026e","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.302592137Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"detaching volume 78899c23-7806-4736-ba33-b5f432ab026e","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.302646436Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:28.306 DEBG Flush     :1010 extent_limit None deps:[JobId(1009), JobId(1008), JobId(1007), JobId(1006), JobId(1005), JobId(1004), JobId(1003), JobId(1002), JobId(1001), JobId(1000)] res:true f:1 g:1
Sep 22 23:14:28.306 DEBG Flush     :1010 extent_limit None deps:[JobId(1009), JobId(1008), JobId(1007), JobId(1006), JobId(1005), JobId(1004), JobId(1003), JobId(1002), JobId(1001), JobId(1000)] res:true f:1 g:1
Sep 22 23:14:28.306 DEBG Flush     :1010 extent_limit None deps:[JobId(1009), JobId(1008), JobId(1007), JobId(1006), JobId(1005), JobId(1004), JobId(1003), JobId(1002), JobId(1001), JobId(1000)] res:true f:1 g:1
{"msg":"Request to deactivate this guest","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.30723502Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:28.307 INFO listening on 127.0.0.1:0, task: main
{"msg":"aa7cbd55-9585-444d-917b-34b9d1d5a423 set deactivating.","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.307323496Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.307470709Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/78899c23-7806-4736-ba33-b5f432ab026e","method":"DELETE","req_id":"f9dc307e-8670-4496-8955-2f14a4cc4df3","remote_addr":"127.0.0.1:35367","local_addr":"127.0.0.1:48931","component":"dropshot","latency_us":4992,"response_code":"204"}
Sep 22 23:14:28.307 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:28.307 INFO Opened existing region file "/tmp/downstairs-SvxEiywd/region.json"
Sep 22 23:14:28.307 INFO Upstairs starts
Sep 22 23:14:28.307 INFO Database read version 1
Sep 22 23:14:28.307 INFO Database write version 1
Sep 22 23:14:28.307 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:14:28.307 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:14:28.307 INFO Crucible stats registered with UUID: aa7cbd55-9585-444d-917b-34b9d1d5a423
Sep 22 23:14:28.307 INFO Crucible aa7cbd55-9585-444d-917b-34b9d1d5a423 has session id: cc26b89c-1762-43aa-9f1e-89b721c949d3
Sep 22 23:14:28.307 INFO [0] connecting to 127.0.0.1:64713, looper: 0
Sep 22 23:14:28.308 INFO [1] connecting to 127.0.0.1:60775, looper: 1
Sep 22 23:14:28.308 INFO [2] connecting to 127.0.0.1:35797, looper: 2
Sep 22 23:14:28.308 INFO up_listen starts, task: up_listen
Sep 22 23:14:28.308 INFO Wait for all three downstairs to come online
Sep 22 23:14:28.308 INFO Flush timeout: 0.5
Sep 22 23:14:28.308 INFO [0] aa7cbd55-9585-444d-917b-34b9d1d5a423 looper connected, looper: 0
Sep 22 23:14:28.308 INFO [0] Proc runs for 127.0.0.1:64713 in state New
Sep 22 23:14:28.308 INFO [1] aa7cbd55-9585-444d-917b-34b9d1d5a423 looper connected, looper: 1
Sep 22 23:14:28.308 INFO [1] Proc runs for 127.0.0.1:60775 in state New
Sep 22 23:14:28.308 INFO [2] aa7cbd55-9585-444d-917b-34b9d1d5a423 looper connected, looper: 2
test test::test_pantry_bulk_read ... okSep 22 23:14:28.308 INFO [2] Proc runs for 127.0.0.1:35797 in state New

Sep 22 23:14:28.308 INFO accepted connection from 127.0.0.1:45873, task: main
Sep 22 23:14:28.308 INFO accepted connection from 127.0.0.1:33997, task: main
Sep 22 23:14:28.308 INFO accepted connection from 127.0.0.1:54269, task: main
Sep 22 23:14:28.309 INFO Connection request from aa7cbd55-9585-444d-917b-34b9d1d5a423 with version 4, task: proc
Sep 22 23:14:28.309 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:28.309 INFO upstairs UpstairsConnection { upstairs_id: aa7cbd55-9585-444d-917b-34b9d1d5a423, session_id: 76885fcd-caf3-4c24-a3a9-04dbfcef7b58, gen: 2 } connected, version 4, task: proc
Sep 22 23:14:28.309 INFO Connection request from aa7cbd55-9585-444d-917b-34b9d1d5a423 with version 4, task: proc
Sep 22 23:14:28.309 INFO upstairs UpstairsConnection { upstairs_id: aa7cbd55-9585-444d-917b-34b9d1d5a423, session_id: 76885fcd-caf3-4c24-a3a9-04dbfcef7b58, gen: 2 } connected, version 4, task: proc
Sep 22 23:14:28.309 INFO Created new region file "/tmp/downstairs-XcMs9S9H/region.json"
Sep 22 23:14:28.309 INFO Connection request from aa7cbd55-9585-444d-917b-34b9d1d5a423 with version 4, task: proc
Sep 22 23:14:28.309 INFO upstairs UpstairsConnection { upstairs_id: aa7cbd55-9585-444d-917b-34b9d1d5a423, session_id: 76885fcd-caf3-4c24-a3a9-04dbfcef7b58, gen: 2 } connected, version 4, task: proc
Sep 22 23:14:28.309 INFO [0] aa7cbd55-9585-444d-917b-34b9d1d5a423 (76885fcd-caf3-4c24-a3a9-04dbfcef7b58) New New New ds_transition to WaitActive
Sep 22 23:14:28.309 INFO [0] Transition from New to WaitActive
Sep 22 23:14:28.309 INFO [1] aa7cbd55-9585-444d-917b-34b9d1d5a423 (76885fcd-caf3-4c24-a3a9-04dbfcef7b58) WaitActive New New ds_transition to WaitActive
Sep 22 23:14:28.309 INFO [1] Transition from New to WaitActive
Sep 22 23:14:28.309 INFO [2] aa7cbd55-9585-444d-917b-34b9d1d5a423 (76885fcd-caf3-4c24-a3a9-04dbfcef7b58) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:14:28.309 INFO [2] Transition from New to WaitActive
The guest has requested activation
Sep 22 23:14:28.309 INFO aa7cbd55-9585-444d-917b-34b9d1d5a423 active request set
Sep 22 23:14:28.309 INFO [0] received activate with gen 2
Sep 22 23:14:28.309 INFO [0] client got ds_active_rx, promote! session 76885fcd-caf3-4c24-a3a9-04dbfcef7b58
Sep 22 23:14:28.309 INFO [1] received activate with gen 2
Sep 22 23:14:28.309 INFO [1] client got ds_active_rx, promote! session 76885fcd-caf3-4c24-a3a9-04dbfcef7b58
Sep 22 23:14:28.310 INFO [2] received activate with gen 2
Sep 22 23:14:28.310 INFO [2] client got ds_active_rx, promote! session 76885fcd-caf3-4c24-a3a9-04dbfcef7b58
Sep 22 23:14:28.310 WARN Attempting RW takeover from UpstairsConnection { upstairs_id: aa7cbd55-9585-444d-917b-34b9d1d5a423, session_id: ae2b86c4-327d-41e8-b04c-5803e0cf8d60, gen: 1 } to UpstairsConnection { upstairs_id: aa7cbd55-9585-444d-917b-34b9d1d5a423, session_id: 76885fcd-caf3-4c24-a3a9-04dbfcef7b58, gen: 2 }
Sep 22 23:14:28.310 WARN Signaling to UpstairsConnection { upstairs_id: aa7cbd55-9585-444d-917b-34b9d1d5a423, session_id: ae2b86c4-327d-41e8-b04c-5803e0cf8d60, gen: 1 } thread that UpstairsConnection { upstairs_id: aa7cbd55-9585-444d-917b-34b9d1d5a423, session_id: 76885fcd-caf3-4c24-a3a9-04dbfcef7b58, gen: 2 } is being promoted (read-write)
Sep 22 23:14:28.310 INFO UpstairsConnection { upstairs_id: aa7cbd55-9585-444d-917b-34b9d1d5a423, session_id: 76885fcd-caf3-4c24-a3a9-04dbfcef7b58, gen: 2 } is now active (read-write)
Sep 22 23:14:28.310 WARN Attempting RW takeover from UpstairsConnection { upstairs_id: aa7cbd55-9585-444d-917b-34b9d1d5a423, session_id: ae2b86c4-327d-41e8-b04c-5803e0cf8d60, gen: 1 } to UpstairsConnection { upstairs_id: aa7cbd55-9585-444d-917b-34b9d1d5a423, session_id: 76885fcd-caf3-4c24-a3a9-04dbfcef7b58, gen: 2 }
Sep 22 23:14:28.310 WARN Signaling to UpstairsConnection { upstairs_id: aa7cbd55-9585-444d-917b-34b9d1d5a423, session_id: ae2b86c4-327d-41e8-b04c-5803e0cf8d60, gen: 1 } thread that UpstairsConnection { upstairs_id: aa7cbd55-9585-444d-917b-34b9d1d5a423, session_id: 76885fcd-caf3-4c24-a3a9-04dbfcef7b58, gen: 2 } is being promoted (read-write)
Sep 22 23:14:28.310 INFO UpstairsConnection { upstairs_id: aa7cbd55-9585-444d-917b-34b9d1d5a423, session_id: 76885fcd-caf3-4c24-a3a9-04dbfcef7b58, gen: 2 } is now active (read-write)
Sep 22 23:14:28.310 WARN Attempting RW takeover from UpstairsConnection { upstairs_id: aa7cbd55-9585-444d-917b-34b9d1d5a423, session_id: ae2b86c4-327d-41e8-b04c-5803e0cf8d60, gen: 1 } to UpstairsConnection { upstairs_id: aa7cbd55-9585-444d-917b-34b9d1d5a423, session_id: 76885fcd-caf3-4c24-a3a9-04dbfcef7b58, gen: 2 }
Sep 22 23:14:28.310 WARN Signaling to UpstairsConnection { upstairs_id: aa7cbd55-9585-444d-917b-34b9d1d5a423, session_id: ae2b86c4-327d-41e8-b04c-5803e0cf8d60, gen: 1 } thread that UpstairsConnection { upstairs_id: aa7cbd55-9585-444d-917b-34b9d1d5a423, session_id: 76885fcd-caf3-4c24-a3a9-04dbfcef7b58, gen: 2 } is being promoted (read-write)
Sep 22 23:14:28.310 INFO UpstairsConnection { upstairs_id: aa7cbd55-9585-444d-917b-34b9d1d5a423, session_id: 76885fcd-caf3-4c24-a3a9-04dbfcef7b58, gen: 2 } is now active (read-write)
Sep 22 23:14:28.310 WARN Another upstairs UpstairsConnection { upstairs_id: aa7cbd55-9585-444d-917b-34b9d1d5a423, session_id: 76885fcd-caf3-4c24-a3a9-04dbfcef7b58, gen: 2 } promoted to active, shutting down connection for UpstairsConnection { upstairs_id: aa7cbd55-9585-444d-917b-34b9d1d5a423, session_id: ae2b86c4-327d-41e8-b04c-5803e0cf8d60, gen: 1 }, task: main
Sep 22 23:14:28.310 INFO connection (127.0.0.1:59929): all done
Sep 22 23:14:28.310 WARN Another upstairs UpstairsConnection { upstairs_id: aa7cbd55-9585-444d-917b-34b9d1d5a423, session_id: 76885fcd-caf3-4c24-a3a9-04dbfcef7b58, gen: 2 } promoted to active, shutting down connection for UpstairsConnection { upstairs_id: aa7cbd55-9585-444d-917b-34b9d1d5a423, session_id: ae2b86c4-327d-41e8-b04c-5803e0cf8d60, gen: 1 }, task: main
Sep 22 23:14:28.310 INFO connection (127.0.0.1:32800): all done
Sep 22 23:14:28.310 WARN Another upstairs UpstairsConnection { upstairs_id: aa7cbd55-9585-444d-917b-34b9d1d5a423, session_id: 76885fcd-caf3-4c24-a3a9-04dbfcef7b58, gen: 2 } promoted to active, shutting down connection for UpstairsConnection { upstairs_id: aa7cbd55-9585-444d-917b-34b9d1d5a423, session_id: ae2b86c4-327d-41e8-b04c-5803e0cf8d60, gen: 1 }, task: main
Sep 22 23:14:28.310 INFO connection (127.0.0.1:55758): all done
{"msg":"[0] aa7cbd55-9585-444d-917b-34b9d1d5a423 (ae2b86c4-327d-41e8-b04c-5803e0cf8d60) cmd_loop saw YouAreNoLongerActive aa7cbd55-9585-444d-917b-34b9d1d5a423 76885fcd-caf3-4c24-a3a9-04dbfcef7b58 2","v":0,"name":"crucible-pantry","level":50,"time":"2023-09-22T23:14:28.311270107Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] aa7cbd55-9585-444d-917b-34b9d1d5a423 (ae2b86c4-327d-41e8-b04c-5803e0cf8d60) Active Active Active ds_transition to Disabled","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.311355872Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] Transition from Active to Disabled","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.31139287Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"aa7cbd55-9585-444d-917b-34b9d1d5a423 set inactive, session ae2b86c4-327d-41e8-b04c-5803e0cf8d60","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.31142853Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"127.0.0.1:64713: proc: Generation number is too low: saw YouAreNoLongerActive with larger gen 2 than ours 1","v":0,"name":"crucible-pantry","level":50,"time":"2023-09-22T23:14:28.311473758Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"0","component":"datafile"}
{"msg":"[0] aa7cbd55-9585-444d-917b-34b9d1d5a423 Gone missing, transition from Disabled to Disconnected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.311519464Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] aa7cbd55-9585-444d-917b-34b9d1d5a423 connection to 127.0.0.1:64713 closed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.311560081Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"0","component":"datafile"}
{"msg":"[1] aa7cbd55-9585-444d-917b-34b9d1d5a423 (ae2b86c4-327d-41e8-b04c-5803e0cf8d60) cmd_loop saw YouAreNoLongerActive aa7cbd55-9585-444d-917b-34b9d1d5a423 76885fcd-caf3-4c24-a3a9-04dbfcef7b58 2","v":0,"name":"crucible-pantry","level":50,"time":"2023-09-22T23:14:28.31160638Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] aa7cbd55-9585-444d-917b-34b9d1d5a423 (ae2b86c4-327d-41e8-b04c-5803e0cf8d60) Disconnected Active Active ds_transition to Disabled","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.31164379Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] Transition from Active to Disabled","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.311667524Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"aa7cbd55-9585-444d-917b-34b9d1d5a423 set inactive, session ae2b86c4-327d-41e8-b04c-5803e0cf8d60","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.311690991Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"127.0.0.1:60775: proc: Generation number is too low: saw YouAreNoLongerActive with larger gen 2 than ours 1","v":0,"name":"crucible-pantry","level":50,"time":"2023-09-22T23:14:28.311730649Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"1","component":"datafile"}
{"msg":"[1] aa7cbd55-9585-444d-917b-34b9d1d5a423 Gone missing, transition from Disabled to Disconnected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.31175629Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] aa7cbd55-9585-444d-917b-34b9d1d5a423 connection to 127.0.0.1:60775 closed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.31177949Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"1","component":"datafile"}
{"msg":"[2] aa7cbd55-9585-444d-917b-34b9d1d5a423 (ae2b86c4-327d-41e8-b04c-5803e0cf8d60) cmd_loop saw YouAreNoLongerActive aa7cbd55-9585-444d-917b-34b9d1d5a423 76885fcd-caf3-4c24-a3a9-04dbfcef7b58 2","v":0,"name":"crucible-pantry","level":50,"time":"2023-09-22T23:14:28.311806778Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] aa7cbd55-9585-444d-917b-34b9d1d5a423 (ae2b86c4-327d-41e8-b04c-5803e0cf8d60) Disconnected Disconnected Active ds_transition to Disabled","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.311844706Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] Transition from Active to Disabled","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.311882455Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:28.311 INFO [0] downstairs client at 127.0.0.1:64713 has UUID 16035766-fcf1-4aff-8be5-4662118fbeba
Sep 22 23:14:28.311 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 16035766-fcf1-4aff-8be5-4662118fbeba, encrypted: true, database_read_version: 1, database_write_version: 1 }
{"msg":"aa7cbd55-9585-444d-917b-34b9d1d5a423 set inactive, session ae2b86c4-327d-41e8-b04c-5803e0cf8d60","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.311923973Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:28.311 INFO aa7cbd55-9585-444d-917b-34b9d1d5a423 WaitActive WaitActive WaitActive
{"msg":"127.0.0.1:35797: proc: Generation number is too low: saw YouAreNoLongerActive with larger gen 2 than ours 1","v":0,"name":"crucible-pantry","level":50,"time":"2023-09-22T23:14:28.31198014Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"2","component":"datafile"}
{"msg":"[2] aa7cbd55-9585-444d-917b-34b9d1d5a423 Gone missing, transition from Disabled to Disconnected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.312023692Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:28.312 INFO [1] downstairs client at 127.0.0.1:60775 has UUID c1ff2b45-bb0b-4e95-bfb6-6e2ba62da531
Sep 22 23:14:28.312 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: c1ff2b45-bb0b-4e95-bfb6-6e2ba62da531, encrypted: true, database_read_version: 1, database_write_version: 1 }
{"msg":"[2] aa7cbd55-9585-444d-917b-34b9d1d5a423 connection to 127.0.0.1:35797 closed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.312064523Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"2","component":"datafile"}
Sep 22 23:14:28.312 INFO aa7cbd55-9585-444d-917b-34b9d1d5a423 WaitActive WaitActive WaitActive
{"msg":"[0] pm_task rx.recv() is None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:14:28.312107729Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] 127.0.0.1:64713 task reports connection:false","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.312144973Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:28.312 INFO [2] downstairs client at 127.0.0.1:35797 has UUID f77cd6ac-8353-429b-8c45-d7fd309de24d
Sep 22 23:14:28.312 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: f77cd6ac-8353-429b-8c45-d7fd309de24d, encrypted: true, database_read_version: 1, database_write_version: 1 }
{"msg":"aa7cbd55-9585-444d-917b-34b9d1d5a423 Disconnected Disconnected Disconnected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.312187428Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:28.312 INFO aa7cbd55-9585-444d-917b-34b9d1d5a423 WaitActive WaitActive WaitActive
{"msg":"[0] 127.0.0.1:64713 task reports offline","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.312230747Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] 127.0.0.1:60775 task reports connection:false","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.31227211Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"aa7cbd55-9585-444d-917b-34b9d1d5a423 Disconnected Disconnected Disconnected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.312311136Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:28.312 INFO Current flush_numbers [0..12]: [1, 1]
{"msg":"[1] 127.0.0.1:60775 task reports offline","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.312346777Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] 127.0.0.1:35797 task reports connection:false","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.312396421Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"aa7cbd55-9585-444d-917b-34b9d1d5a423 Disconnected Disconnected Disconnected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.312439286Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] 127.0.0.1:35797 task reports offline","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.312481246Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] pm_task rx.recv() is None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:14:28.312519571Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] pm_task rx.recv() is None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:14:28.312557788Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:28.313 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:28.313 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:28.313 INFO Opened existing region file "/tmp/downstairs-XcMs9S9H/region.json"
Sep 22 23:14:28.313 INFO Database read version 1
Sep 22 23:14:28.313 INFO Database write version 1
Sep 22 23:14:28.313 INFO Current flush_numbers [0..12]: [1, 1]
Sep 22 23:14:28.313 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:28.313 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:28.313 INFO Current flush_numbers [0..12]: [1, 1]
Sep 22 23:14:28.313 INFO Created new region file "/tmp/downstairs-kApwRwyr/region.json"
Sep 22 23:14:28.314 INFO UUID: 46e0ee2f-0f49-4348-a954-78960194fdcd
Sep 22 23:14:28.314 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:28.314 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:28.314 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:28.314 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:28.314 INFO Using address: 127.0.0.1:60580, task: main
Sep 22 23:14:28.314 INFO [0] aa7cbd55-9585-444d-917b-34b9d1d5a423 (76885fcd-caf3-4c24-a3a9-04dbfcef7b58) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:28.314 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:14:28.314 WARN [0] new RM replaced this: None
Sep 22 23:14:28.314 INFO [0] Starts reconcile loop
Sep 22 23:14:28.314 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:28.314 INFO [1] aa7cbd55-9585-444d-917b-34b9d1d5a423 (76885fcd-caf3-4c24-a3a9-04dbfcef7b58) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:28.314 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:52587, task: repair
Sep 22 23:14:28.314 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:14:28.314 WARN [1] new RM replaced this: None
Sep 22 23:14:28.314 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:52587, task: repair
Sep 22 23:14:28.314 INFO [1] Starts reconcile loop
Sep 22 23:14:28.314 INFO listening, local_addr: 127.0.0.1:52587, task: repair
Sep 22 23:14:28.314 INFO [2] aa7cbd55-9585-444d-917b-34b9d1d5a423 (76885fcd-caf3-4c24-a3a9-04dbfcef7b58) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:14:28.314 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:14:28.314 WARN [2] new RM replaced this: None
Sep 22 23:14:28.314 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:52587, task: repair
Sep 22 23:14:28.315 INFO [2] Starts reconcile loop
Sep 22 23:14:28.315 INFO Using repair address: 127.0.0.1:52587, task: main
Sep 22 23:14:28.315 INFO No SSL acceptor configured, task: main
Sep 22 23:14:28.315 INFO [0] 127.0.0.1:64713 task reports connection:true
Sep 22 23:14:28.315 INFO aa7cbd55-9585-444d-917b-34b9d1d5a423 WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:14:28.315 INFO [0]R  flush_numbers: [1, 1]
Sep 22 23:14:28.315 INFO [0]R  generation: [1, 1]
Sep 22 23:14:28.315 INFO [0]R  dirty: [false, false]
Sep 22 23:14:28.315 INFO [1]R  flush_numbers: [1, 1]
Sep 22 23:14:28.315 INFO [1]R  generation: [1, 1]
Sep 22 23:14:28.315 INFO [1]R  dirty: [false, false]
Sep 22 23:14:28.315 INFO [2]R  flush_numbers: [1, 1]
Sep 22 23:14:28.315 INFO [2]R  generation: [1, 1]
Sep 22 23:14:28.315 INFO [2]R  dirty: [false, false]
Sep 22 23:14:28.315 INFO Max found gen is 2
Sep 22 23:14:28.315 INFO Generation requested: 2 >= found:2
Sep 22 23:14:28.315 INFO Next flush: 2
Sep 22 23:14:28.315 INFO All extents match
Sep 22 23:14:28.315 INFO No downstairs repair required
Sep 22 23:14:28.315 INFO No initial repair work was required
Sep 22 23:14:28.315 INFO Set Downstairs and Upstairs active
Sep 22 23:14:28.315 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:28.315 INFO aa7cbd55-9585-444d-917b-34b9d1d5a423 is now active with session: 76885fcd-caf3-4c24-a3a9-04dbfcef7b58
Sep 22 23:14:28.315 INFO aa7cbd55-9585-444d-917b-34b9d1d5a423 Set Active after no repair
Sep 22 23:14:28.315 INFO Notify all downstairs, region set compare is done.
Sep 22 23:14:28.315 INFO Created new region file "/tmp/downstairs-1NlQYu1c/region.json"
Sep 22 23:14:28.315 INFO Set check for repair
Sep 22 23:14:28.315 INFO [1] 127.0.0.1:60775 task reports connection:true
Sep 22 23:14:28.315 INFO aa7cbd55-9585-444d-917b-34b9d1d5a423 Active Active Active
Sep 22 23:14:28.315 INFO Set check for repair
Sep 22 23:14:28.315 INFO [2] 127.0.0.1:35797 task reports connection:true
Sep 22 23:14:28.315 INFO aa7cbd55-9585-444d-917b-34b9d1d5a423 Active Active Active
Sep 22 23:14:28.315 INFO Set check for repair
Sep 22 23:14:28.315 INFO [0] received reconcile message
Sep 22 23:14:28.315 INFO [0] All repairs completed, exit
Sep 22 23:14:28.315 INFO [0] Starts cmd_loop
Sep 22 23:14:28.315 INFO [1] received reconcile message
Sep 22 23:14:28.316 INFO [1] All repairs completed, exit
Sep 22 23:14:28.316 INFO [1] Starts cmd_loop
Sep 22 23:14:28.316 INFO [2] received reconcile message
Sep 22 23:14:28.316 INFO [2] All repairs completed, exit
Sep 22 23:14:28.316 INFO [2] Starts cmd_loop
The guest has finished waiting for activation
Sep 22 23:14:28.316 DEBG IO Read  1000 has deps []
Sep 22 23:14:28.317 DEBG Read      :1000 deps:[] res:true
Sep 22 23:14:28.318 DEBG Read      :1000 deps:[] res:true
Sep 22 23:14:28.319 DEBG Read      :1000 deps:[] res:true
Sep 22 23:14:28.319 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:28.319 INFO Opened existing region file "/tmp/downstairs-1NlQYu1c/region.json"
Sep 22 23:14:28.319 INFO Database read version 1
Sep 22 23:14:28.319 INFO Database write version 1
Sep 22 23:14:28.320 INFO UUID: 3b4e7e40-734f-4eaa-bb4c-32e1ad5c2936
Sep 22 23:14:28.320 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:28.321 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:28.321 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:28.321 INFO Using address: 127.0.0.1:59657, task: main
Sep 22 23:14:28.321 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:28.321 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:47357, task: repair
Sep 22 23:14:28.321 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:47357, task: repair
Sep 22 23:14:28.321 INFO listening, local_addr: 127.0.0.1:47357, task: repair
Sep 22 23:14:28.321 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:47357, task: repair
Sep 22 23:14:28.321 INFO Using repair address: 127.0.0.1:47357, task: main
Sep 22 23:14:28.321 INFO No SSL acceptor configured, task: main
Sep 22 23:14:28.321 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:28.321 INFO Created new region file "/tmp/downstairs-pDAr1mVj/region.json"
Sep 22 23:14:28.323 DEBG [0] Read AckReady 1000, : downstairs
Sep 22 23:14:28.325 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:28.325 INFO Opened existing region file "/tmp/downstairs-pDAr1mVj/region.json"
Sep 22 23:14:28.325 INFO Database read version 1
Sep 22 23:14:28.325 INFO Database write version 1
Sep 22 23:14:28.325 DEBG [1] Read already AckReady 1000, : downstairs
Sep 22 23:14:28.328 INFO UUID: 9706eaea-4b57-4d6f-860a-37f6ab1693f0
Sep 22 23:14:28.328 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:28.328 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:28.328 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:28.328 INFO Using address: 127.0.0.1:60352, task: main
Sep 22 23:14:28.328 DEBG [2] Read already AckReady 1000, : downstairs
Sep 22 23:14:28.328 DEBG up_ds_listen was notified
Sep 22 23:14:28.328 DEBG up_ds_listen process 1000
Sep 22 23:14:28.328 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:14:28.328 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:28.328 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:37633, task: repair
Sep 22 23:14:28.328 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:37633, task: repair
Sep 22 23:14:28.328 INFO listening, local_addr: 127.0.0.1:37633, task: repair
Sep 22 23:14:28.329 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:37633, task: repair
Sep 22 23:14:28.329 INFO Using repair address: 127.0.0.1:37633, task: main
Sep 22 23:14:28.329 INFO No SSL acceptor configured, task: main
Sep 22 23:14:28.329 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:28.329 INFO Upstairs starts
Sep 22 23:14:28.329 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:14:28.329 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:14:28.329 INFO Crucible stats registered with UUID: 4aa9f481-1957-4d0f-b88c-338224283496
Sep 22 23:14:28.329 INFO Crucible 4aa9f481-1957-4d0f-b88c-338224283496 has session id: 4055688a-4d6b-46d4-b6db-8d303b02a8c6
Sep 22 23:14:28.329 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:28.329 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:28.329 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:28.329 INFO [0] connecting to 127.0.0.1:60580, looper: 0
Sep 22 23:14:28.329 INFO [1] connecting to 127.0.0.1:59657, looper: 1
Sep 22 23:14:28.329 INFO [2] connecting to 127.0.0.1:60352, looper: 2
Sep 22 23:14:28.330 INFO up_listen starts, task: up_listen
Sep 22 23:14:28.330 INFO Wait for all three downstairs to come online
Sep 22 23:14:28.330 INFO Flush timeout: 0.5
Sep 22 23:14:28.330 INFO accepted connection from 127.0.0.1:57962, task: main
Sep 22 23:14:28.330 INFO accepted connection from 127.0.0.1:48077, task: main
Sep 22 23:14:28.330 INFO accepted connection from 127.0.0.1:51233, task: main
Sep 22 23:14:28.330 INFO [0] 4aa9f481-1957-4d0f-b88c-338224283496 looper connected, looper: 0
Sep 22 23:14:28.330 INFO [0] Proc runs for 127.0.0.1:60580 in state New
Sep 22 23:14:28.330 INFO [1] 4aa9f481-1957-4d0f-b88c-338224283496 looper connected, looper: 1
Sep 22 23:14:28.330 INFO [1] Proc runs for 127.0.0.1:59657 in state New
Sep 22 23:14:28.330 INFO [2] 4aa9f481-1957-4d0f-b88c-338224283496 looper connected, looper: 2
Sep 22 23:14:28.330 INFO [2] Proc runs for 127.0.0.1:60352 in state New
Sep 22 23:14:28.330 INFO Connection request from 4aa9f481-1957-4d0f-b88c-338224283496 with version 4, task: proc
Sep 22 23:14:28.330 INFO upstairs UpstairsConnection { upstairs_id: 4aa9f481-1957-4d0f-b88c-338224283496, session_id: 601739f4-d570-4d2f-845d-5eff25c7a668, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:28.331 INFO Connection request from 4aa9f481-1957-4d0f-b88c-338224283496 with version 4, task: proc
Sep 22 23:14:28.331 INFO upstairs UpstairsConnection { upstairs_id: 4aa9f481-1957-4d0f-b88c-338224283496, session_id: 601739f4-d570-4d2f-845d-5eff25c7a668, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:28.331 INFO Connection request from 4aa9f481-1957-4d0f-b88c-338224283496 with version 4, task: proc
Sep 22 23:14:28.331 INFO upstairs UpstairsConnection { upstairs_id: 4aa9f481-1957-4d0f-b88c-338224283496, session_id: 601739f4-d570-4d2f-845d-5eff25c7a668, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:28.331 INFO [0] 4aa9f481-1957-4d0f-b88c-338224283496 (601739f4-d570-4d2f-845d-5eff25c7a668) New New New ds_transition to WaitActive
Sep 22 23:14:28.331 INFO [0] Transition from New to WaitActive
Sep 22 23:14:28.331 INFO [1] 4aa9f481-1957-4d0f-b88c-338224283496 (601739f4-d570-4d2f-845d-5eff25c7a668) WaitActive New New ds_transition to WaitActive
Sep 22 23:14:28.331 INFO [1] Transition from New to WaitActive
Sep 22 23:14:28.331 INFO [2] 4aa9f481-1957-4d0f-b88c-338224283496 (601739f4-d570-4d2f-845d-5eff25c7a668) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:14:28.331 INFO [2] Transition from New to WaitActive
The guest has requested activation
Sep 22 23:14:28.331 INFO 4aa9f481-1957-4d0f-b88c-338224283496 active request set
Sep 22 23:14:28.331 INFO [0] received activate with gen 1
Sep 22 23:14:28.331 INFO [0] client got ds_active_rx, promote! session 601739f4-d570-4d2f-845d-5eff25c7a668
Sep 22 23:14:28.331 INFO [1] received activate with gen 1
Sep 22 23:14:28.331 INFO [1] client got ds_active_rx, promote! session 601739f4-d570-4d2f-845d-5eff25c7a668
Sep 22 23:14:28.331 INFO [2] received activate with gen 1
Sep 22 23:14:28.331 INFO [2] client got ds_active_rx, promote! session 601739f4-d570-4d2f-845d-5eff25c7a668
Sep 22 23:14:28.331 INFO UpstairsConnection { upstairs_id: 4aa9f481-1957-4d0f-b88c-338224283496, session_id: 601739f4-d570-4d2f-845d-5eff25c7a668, gen: 1 } is now active (read-write)
Sep 22 23:14:28.332 INFO UpstairsConnection { upstairs_id: 4aa9f481-1957-4d0f-b88c-338224283496, session_id: 601739f4-d570-4d2f-845d-5eff25c7a668, gen: 1 } is now active (read-write)
Sep 22 23:14:28.332 INFO UpstairsConnection { upstairs_id: 4aa9f481-1957-4d0f-b88c-338224283496, session_id: 601739f4-d570-4d2f-845d-5eff25c7a668, gen: 1 } is now active (read-write)
Sep 22 23:14:28.332 INFO [0] downstairs client at 127.0.0.1:60580 has UUID 46e0ee2f-0f49-4348-a954-78960194fdcd
Sep 22 23:14:28.332 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 46e0ee2f-0f49-4348-a954-78960194fdcd, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:28.332 INFO 4aa9f481-1957-4d0f-b88c-338224283496 WaitActive WaitActive WaitActive
Sep 22 23:14:28.332 INFO [1] downstairs client at 127.0.0.1:59657 has UUID 3b4e7e40-734f-4eaa-bb4c-32e1ad5c2936
Sep 22 23:14:28.332 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 3b4e7e40-734f-4eaa-bb4c-32e1ad5c2936, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:28.332 INFO 4aa9f481-1957-4d0f-b88c-338224283496 WaitActive WaitActive WaitActive
Sep 22 23:14:28.332 INFO [2] downstairs client at 127.0.0.1:60352 has UUID 9706eaea-4b57-4d6f-860a-37f6ab1693f0
Sep 22 23:14:28.332 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 9706eaea-4b57-4d6f-860a-37f6ab1693f0, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:28.332 INFO 4aa9f481-1957-4d0f-b88c-338224283496 WaitActive WaitActive WaitActive
Sep 22 23:14:28.333 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:28.333 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:28.334 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:28.335 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:28.335 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:28.335 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:28.335 INFO [0] 4aa9f481-1957-4d0f-b88c-338224283496 (601739f4-d570-4d2f-845d-5eff25c7a668) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:28.335 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:14:28.335 WARN [0] new RM replaced this: None
Sep 22 23:14:28.335 INFO [0] Starts reconcile loop
Sep 22 23:14:28.335 INFO [1] 4aa9f481-1957-4d0f-b88c-338224283496 (601739f4-d570-4d2f-845d-5eff25c7a668) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:28.336 INFO [1] Transition from WaitActive to WaitQuorum
test test::test_pantry_bulk_write ... Sep 22 23:14:28.336 WARN [1] new RM replaced this: None
ok
Sep 22 23:14:28.336 INFO [1] Starts reconcile loop
Sep 22 23:14:28.336 INFO [2] 4aa9f481-1957-4d0f-b88c-338224283496 (601739f4-d570-4d2f-845d-5eff25c7a668) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:14:28.336 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:14:28.336 WARN [2] new RM replaced this: None
Sep 22 23:14:28.336 INFO [2] Starts reconcile loop
Sep 22 23:14:28.336 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:28.336 INFO [0] 127.0.0.1:60580 task reports connection:true
Sep 22 23:14:28.336 INFO 4aa9f481-1957-4d0f-b88c-338224283496 WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:14:28.336 INFO Created new region file "/tmp/downstairs-cOOWrUaJ/region.json"
Sep 22 23:14:28.336 INFO [0]R  flush_numbers: [0, 0]
Sep 22 23:14:28.336 INFO [0]R  generation: [0, 0]
Sep 22 23:14:28.336 INFO [0]R  dirty: [false, false]
Sep 22 23:14:28.336 INFO [1]R  flush_numbers: [0, 0]
Sep 22 23:14:28.336 INFO [1]R  generation: [0, 0]
Sep 22 23:14:28.336 INFO [1]R  dirty: [false, false]
Sep 22 23:14:28.336 INFO [2]R  flush_numbers: [0, 0]
Sep 22 23:14:28.336 INFO [2]R  generation: [0, 0]
Sep 22 23:14:28.336 INFO [2]R  dirty: [false, false]
Sep 22 23:14:28.336 INFO Max found gen is 1
Sep 22 23:14:28.336 INFO Generation requested: 1 >= found:1
Sep 22 23:14:28.336 INFO Next flush: 1
Sep 22 23:14:28.336 INFO All extents match
Sep 22 23:14:28.336 INFO No downstairs repair required
Sep 22 23:14:28.336 INFO No initial repair work was required
Sep 22 23:14:28.336 INFO Set Downstairs and Upstairs active
Sep 22 23:14:28.336 INFO 4aa9f481-1957-4d0f-b88c-338224283496 is now active with session: 601739f4-d570-4d2f-845d-5eff25c7a668
Sep 22 23:14:28.336 INFO 4aa9f481-1957-4d0f-b88c-338224283496 Set Active after no repair
Sep 22 23:14:28.336 INFO Notify all downstairs, region set compare is done.
Sep 22 23:14:28.336 INFO Set check for repair
Sep 22 23:14:28.336 INFO [1] 127.0.0.1:59657 task reports connection:true
Sep 22 23:14:28.336 INFO 4aa9f481-1957-4d0f-b88c-338224283496 Active Active Active
Sep 22 23:14:28.336 INFO Set check for repair
Sep 22 23:14:28.336 INFO [2] 127.0.0.1:60352 task reports connection:true
Sep 22 23:14:28.336 INFO 4aa9f481-1957-4d0f-b88c-338224283496 Active Active Active
Sep 22 23:14:28.336 INFO Set check for repair
Sep 22 23:14:28.336 INFO [0] received reconcile message
Sep 22 23:14:28.336 INFO [0] All repairs completed, exit
Sep 22 23:14:28.336 INFO [0] Starts cmd_loop
Sep 22 23:14:28.336 INFO [1] received reconcile message
Sep 22 23:14:28.336 INFO [1] All repairs completed, exit
Sep 22 23:14:28.336 INFO [1] Starts cmd_loop
Sep 22 23:14:28.337 INFO [2] received reconcile message
Sep 22 23:14:28.337 INFO [2] All repairs completed, exit
Sep 22 23:14:28.337 INFO [2] Starts cmd_loop
The guest has finished waiting for activation
Sep 22 23:14:28.337 DEBG IO Read  1000 has deps []
Sep 22 23:14:28.338 DEBG Read      :1000 deps:[] res:true
Sep 22 23:14:28.339 DEBG Read      :1000 deps:[] res:true
Sep 22 23:14:28.339 DEBG Read      :1000 deps:[] res:true
Sep 22 23:14:28.340 DEBG [0] Read AckReady 1000, : downstairs
Sep 22 23:14:28.340 DEBG [1] Read already AckReady 1000, : downstairs
Sep 22 23:14:28.340 DEBG [2] Read already AckReady 1000, : downstairs
Sep 22 23:14:28.340 DEBG up_ds_listen was notified
Sep 22 23:14:28.340 DEBG up_ds_listen process 1000
Sep 22 23:14:28.340 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:14:28.341 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:28.341 INFO Request to deactivate this guest
Sep 22 23:14:28.341 INFO 4aa9f481-1957-4d0f-b88c-338224283496 set deactivating.
Sep 22 23:14:28.341 DEBG IO Flush 1001 has deps [JobId(1000)]
Sep 22 23:14:28.341 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:1 g:1
Sep 22 23:14:28.341 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:1 g:1
Sep 22 23:14:28.341 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:1 g:1
Sep 22 23:14:28.341 INFO [0] check deactivate YES
Sep 22 23:14:28.342 INFO [0] 4aa9f481-1957-4d0f-b88c-338224283496 (601739f4-d570-4d2f-845d-5eff25c7a668) Active Active Active ds_transition to Deactivated
Sep 22 23:14:28.342 INFO [0] Transition from Active to Deactivated
Sep 22 23:14:28.342 INFO [1] check deactivate YES
Sep 22 23:14:28.342 INFO [1] 4aa9f481-1957-4d0f-b88c-338224283496 (601739f4-d570-4d2f-845d-5eff25c7a668) Deactivated Active Active ds_transition to Deactivated
Sep 22 23:14:28.342 INFO [1] Transition from Active to Deactivated
Sep 22 23:14:28.342 DEBG [2] deactivate flush 1001 done, : downstairs
Sep 22 23:14:28.342 INFO [2] check deactivate YES
Sep 22 23:14:28.342 INFO [2] 4aa9f481-1957-4d0f-b88c-338224283496 (601739f4-d570-4d2f-845d-5eff25c7a668) Deactivated Deactivated Active ds_transition to Deactivated
Sep 22 23:14:28.342 INFO [2] Transition from Active to Deactivated
Sep 22 23:14:28.342 ERRO 127.0.0.1:60580: proc: [0] client work task ended, Ok(Err([0] exits after deactivation)), so we end too, looper: 0
Sep 22 23:14:28.342 INFO [0] 4aa9f481-1957-4d0f-b88c-338224283496 Gone missing, transition from Deactivated to New
Sep 22 23:14:28.342 INFO deactivate transition checking...
Sep 22 23:14:28.342 INFO deactivate_transition New Maybe 
Sep 22 23:14:28.342 INFO deactivate_transition Deactivated NO
Sep 22 23:14:28.342 INFO deactivate_transition Deactivated NO
Sep 22 23:14:28.342 INFO [0] 4aa9f481-1957-4d0f-b88c-338224283496 connection to 127.0.0.1:60580 closed, looper: 0
Sep 22 23:14:28.342 ERRO 127.0.0.1:59657: proc: [1] client work task ended, Ok(Err([1] exits after deactivation)), so we end too, looper: 1
Sep 22 23:14:28.342 INFO [1] 4aa9f481-1957-4d0f-b88c-338224283496 Gone missing, transition from Deactivated to New
Sep 22 23:14:28.342 INFO deactivate transition checking...
Sep 22 23:14:28.342 INFO deactivate_transition New Maybe 
Sep 22 23:14:28.342 INFO deactivate_transition New Maybe 
Sep 22 23:14:28.342 INFO deactivate_transition Deactivated NO
Sep 22 23:14:28.342 INFO [1] 4aa9f481-1957-4d0f-b88c-338224283496 connection to 127.0.0.1:59657 closed, looper: 1
Sep 22 23:14:28.342 DEBG up_ds_listen was notified
Sep 22 23:14:28.342 DEBG up_ds_listen process 1001
Sep 22 23:14:28.342 DEBG [A] ack job 1001:2, : downstairs
Sep 22 23:14:28.342 DEBG [rc] retire 1001 clears [JobId(1000), JobId(1001)], : downstairs
Sep 22 23:14:28.342 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:28.342 ERRO 127.0.0.1:60352: proc: [2] client work task ended, Ok(Err([2] exits after deactivation)), so we end too, looper: 2
Sep 22 23:14:28.342 INFO [2] 4aa9f481-1957-4d0f-b88c-338224283496 Gone missing, transition from Deactivated to New
Sep 22 23:14:28.342 INFO deactivate transition checking...
Sep 22 23:14:28.342 INFO deactivate_transition New Maybe 
Sep 22 23:14:28.342 INFO deactivate_transition New Maybe 
Sep 22 23:14:28.342 INFO deactivate_transition New Maybe 
Sep 22 23:14:28.342 INFO All DS in the proper state! -> INIT
Sep 22 23:14:28.342 INFO [2] 4aa9f481-1957-4d0f-b88c-338224283496 connection to 127.0.0.1:60352 closed, looper: 2
Sep 22 23:14:28.342 INFO [0] 127.0.0.1:60580 task reports connection:false
Sep 22 23:14:28.342 INFO 4aa9f481-1957-4d0f-b88c-338224283496 New New New
Sep 22 23:14:28.342 INFO [0] 127.0.0.1:60580 task reports offline
Sep 22 23:14:28.342 INFO [1] 127.0.0.1:59657 task reports connection:false
Sep 22 23:14:28.342 INFO 4aa9f481-1957-4d0f-b88c-338224283496 New New New
Sep 22 23:14:28.342 INFO [1] 127.0.0.1:59657 task reports offline
Sep 22 23:14:28.342 INFO [2] 127.0.0.1:60352 task reports connection:false
Sep 22 23:14:28.342 INFO 4aa9f481-1957-4d0f-b88c-338224283496 New New New
Sep 22 23:14:28.342 INFO [2] 127.0.0.1:60352 task reports offline
note: configured to log to "/dev/stdout"
{"msg":"listening","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.3451891Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"local_addr":"127.0.0.1:52411","component":"dropshot"}
{"msg":"listen IP: 127.0.0.1:52411","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.345392829Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
Sep 22 23:14:28.368 DEBG IO Flush 1001 has deps [JobId(1000)]
Sep 22 23:14:28.370 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:1 g:1
Sep 22 23:14:28.370 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:1 g:1
Sep 22 23:14:28.370 DEBG up_ds_listen was notified
Sep 22 23:14:28.370 DEBG up_ds_listen process 1001
Sep 22 23:14:28.370 DEBG [A] ack job 1001:2, : downstairs
Sep 22 23:14:28.371 DEBG [rc] retire 1001 clears [JobId(1000), JobId(1001)], : downstairs
Sep 22 23:14:28.371 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:28.391 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:28.391 INFO Opened existing region file "/tmp/downstairs-6IEHRjPM/region.json"
Sep 22 23:14:28.391 INFO Database read version 1
Sep 22 23:14:28.391 INFO Database write version 1
Sep 22 23:14:28.425 WARN upstairs UpstairsConnection { upstairs_id: 4aa9f481-1957-4d0f-b88c-338224283496, session_id: 601739f4-d570-4d2f-845d-5eff25c7a668, gen: 1 } disconnected, 0 jobs left, task: main
Sep 22 23:14:28.425 WARN upstairs UpstairsConnection { upstairs_id: 4aa9f481-1957-4d0f-b88c-338224283496, session_id: 601739f4-d570-4d2f-845d-5eff25c7a668, gen: 1 } was previously active, clearing, task: main
Sep 22 23:14:28.425 INFO connection (127.0.0.1:57962): all done
Sep 22 23:14:28.425 WARN upstairs UpstairsConnection { upstairs_id: 4aa9f481-1957-4d0f-b88c-338224283496, session_id: 601739f4-d570-4d2f-845d-5eff25c7a668, gen: 1 } disconnected, 0 jobs left, task: main
Sep 22 23:14:28.426 WARN upstairs UpstairsConnection { upstairs_id: 4aa9f481-1957-4d0f-b88c-338224283496, session_id: 601739f4-d570-4d2f-845d-5eff25c7a668, gen: 1 } was previously active, clearing, task: main
Sep 22 23:14:28.426 INFO connection (127.0.0.1:51233): all done
Sep 22 23:14:28.426 WARN upstairs UpstairsConnection { upstairs_id: 4aa9f481-1957-4d0f-b88c-338224283496, session_id: 601739f4-d570-4d2f-845d-5eff25c7a668, gen: 1 } disconnected, 0 jobs left, task: main
Sep 22 23:14:28.426 WARN upstairs UpstairsConnection { upstairs_id: 4aa9f481-1957-4d0f-b88c-338224283496, session_id: 601739f4-d570-4d2f-845d-5eff25c7a668, gen: 1 } was previously active, clearing, task: main
Sep 22 23:14:28.426 INFO connection (127.0.0.1:48077): all done
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.426759722Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"local_addr":"127.0.0.1:52411","component":"dropshot","remote_addr":"127.0.0.1:52249"}
{"msg":"no entry exists for volume 7d24406e-7e52-4892-b53c-b8c4894a13ca, constructing...","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.427402061Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Upstairs starts","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.427614019Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Crucible Version: BuildInfo {\n    version: \"0.0.1\",\n    git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n    git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n    git_branch: \"main\",\n    rustc_semver: \"1.70.0\",\n    rustc_channel: \"stable\",\n    rustc_host_triple: \"x86_64-unknown-illumos\",\n    rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n    cargo_triple: \"x86_64-unknown-illumos\",\n    debug: true,\n    opt_level: 0,\n}","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.427693375Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.427742754Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Crucible stats registered with UUID: 4aa9f481-1957-4d0f-b88c-338224283496","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.42778866Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Crucible 4aa9f481-1957-4d0f-b88c-338224283496 has session id: 3adadc24-c69c-4056-8ffa-454bbc56b96f","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.427838153Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] connecting to 127.0.0.1:60580","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.427883656Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"0","component":"datafile"}
{"msg":"[1] connecting to 127.0.0.1:59657","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.427984965Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"1","component":"datafile"}
{"msg":"[2] connecting to 127.0.0.1:60352","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.428027471Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"2","component":"datafile"}
{"msg":"up_listen starts","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.428201474Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile","task":"up_listen"}
{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.428241954Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Flush timeout: 0.5","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.42826658Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"volume 7d24406e-7e52-4892-b53c-b8c4894a13ca constructed ok","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.428451539Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
The guest has requested activation
{"msg":"4aa9f481-1957-4d0f-b88c-338224283496 active request set","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.42868495Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:28.428 INFO accepted connection from 127.0.0.1:63586, task: main
Sep 22 23:14:28.428 INFO accepted connection from 127.0.0.1:59585, task: main
Sep 22 23:14:28.428 INFO accepted connection from 127.0.0.1:35200, task: main
{"msg":"[0] 4aa9f481-1957-4d0f-b88c-338224283496 looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.429051869Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"0","component":"datafile"}
{"msg":"[0] Proc runs for 127.0.0.1:60580 in state New","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.429088925Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] 4aa9f481-1957-4d0f-b88c-338224283496 looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.429133339Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"1","component":"datafile"}
{"msg":"[1] Proc runs for 127.0.0.1:59657 in state New","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.429158542Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] 4aa9f481-1957-4d0f-b88c-338224283496 looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.429255586Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"2","component":"datafile"}
{"msg":"[2] Proc runs for 127.0.0.1:60352 in state New","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.429286937Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:28.429 INFO Connection request from 4aa9f481-1957-4d0f-b88c-338224283496 with version 4, task: proc
Sep 22 23:14:28.429 INFO upstairs UpstairsConnection { upstairs_id: 4aa9f481-1957-4d0f-b88c-338224283496, session_id: 7f994951-798d-4ca9-84d4-c9557433fc1b, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:28.429 INFO Connection request from 4aa9f481-1957-4d0f-b88c-338224283496 with version 4, task: proc
Sep 22 23:14:28.429 INFO upstairs UpstairsConnection { upstairs_id: 4aa9f481-1957-4d0f-b88c-338224283496, session_id: 7f994951-798d-4ca9-84d4-c9557433fc1b, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:28.429 INFO Connection request from 4aa9f481-1957-4d0f-b88c-338224283496 with version 4, task: proc
Sep 22 23:14:28.429 INFO upstairs UpstairsConnection { upstairs_id: 4aa9f481-1957-4d0f-b88c-338224283496, session_id: 7f994951-798d-4ca9-84d4-c9557433fc1b, gen: 1 } connected, version 4, task: proc
{"msg":"[0] 4aa9f481-1957-4d0f-b88c-338224283496 (7f994951-798d-4ca9-84d4-c9557433fc1b) New New New ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.429885989Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.429916258Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] client is_active_req TRUE, promote! session 7f994951-798d-4ca9-84d4-c9557433fc1b","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.429940956Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] 4aa9f481-1957-4d0f-b88c-338224283496 (7f994951-798d-4ca9-84d4-c9557433fc1b) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.429972209Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.430000827Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] client is_active_req TRUE, promote! session 7f994951-798d-4ca9-84d4-c9557433fc1b","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.430028322Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] 4aa9f481-1957-4d0f-b88c-338224283496 (7f994951-798d-4ca9-84d4-c9557433fc1b) WaitActive WaitActive New ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.430053636Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.43007606Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] client is_active_req TRUE, promote! session 7f994951-798d-4ca9-84d4-c9557433fc1b","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.430105348Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:28.430 INFO UpstairsConnection { upstairs_id: 4aa9f481-1957-4d0f-b88c-338224283496, session_id: 7f994951-798d-4ca9-84d4-c9557433fc1b, gen: 1 } is now active (read-write)
Sep 22 23:14:28.430 INFO UpstairsConnection { upstairs_id: 4aa9f481-1957-4d0f-b88c-338224283496, session_id: 7f994951-798d-4ca9-84d4-c9557433fc1b, gen: 1 } is now active (read-write)
Sep 22 23:14:28.430 INFO UpstairsConnection { upstairs_id: 4aa9f481-1957-4d0f-b88c-338224283496, session_id: 7f994951-798d-4ca9-84d4-c9557433fc1b, gen: 1 } is now active (read-write)
{"msg":"[0] downstairs client at 127.0.0.1:60580 has UUID 46e0ee2f-0f49-4348-a954-78960194fdcd","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.43090825Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 46e0ee2f-0f49-4348-a954-78960194fdcd, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.430943842Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"4aa9f481-1957-4d0f-b88c-338224283496 WaitActive WaitActive WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.430968452Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] downstairs client at 127.0.0.1:59657 has UUID 3b4e7e40-734f-4eaa-bb4c-32e1ad5c2936","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.431002878Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 3b4e7e40-734f-4eaa-bb4c-32e1ad5c2936, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.431030726Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"4aa9f481-1957-4d0f-b88c-338224283496 WaitActive WaitActive WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.431053925Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] downstairs client at 127.0.0.1:60352 has UUID 9706eaea-4b57-4d6f-860a-37f6ab1693f0","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.431160934Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 9706eaea-4b57-4d6f-860a-37f6ab1693f0, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.431195759Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"4aa9f481-1957-4d0f-b88c-338224283496 WaitActive WaitActive WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.431219565Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:28.431 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:28.431 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:28.431 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:28.431 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:28.432 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:28.432 INFO Downstairs has completed Negotiation, task: proc
{"msg":"[0] 4aa9f481-1957-4d0f-b88c-338224283496 (7f994951-798d-4ca9-84d4-c9557433fc1b) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.432451254Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.432488294Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:14:28.43251255Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.432542361Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] 4aa9f481-1957-4d0f-b88c-338224283496 (7f994951-798d-4ca9-84d4-c9557433fc1b) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.432568169Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.432590396Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:14:28.432612105Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.432633762Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] 4aa9f481-1957-4d0f-b88c-338224283496 (7f994951-798d-4ca9-84d4-c9557433fc1b) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.432658294Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.432680453Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:14:28.432702412Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.43272412Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] 127.0.0.1:60580 task reports connection:true","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.432746452Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"4aa9f481-1957-4d0f-b88c-338224283496 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.43276915Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0]R  flush_numbers: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.432790832Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0]R  generation: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.432812508Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0]R  dirty: [false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.432834158Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1]R  flush_numbers: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.432855759Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
The guest has finished waiting for activation
{"msg":"[1]R  generation: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.432882593Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1]R  dirty: [false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.432930673Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2]R  flush_numbers: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.432965028Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2]R  generation: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.43298882Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2]R  dirty: [false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.43301046Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Max found gen is 1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.433031855Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.433053696Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Next flush: 1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.43308012Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"All extents match","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.433105419Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"No downstairs repair required","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.433133952Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"No initial repair work was required","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.433157667Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.433179547Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"4aa9f481-1957-4d0f-b88c-338224283496 is now active with session: 7f994951-798d-4ca9-84d4-c9557433fc1b","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.433203311Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"4aa9f481-1957-4d0f-b88c-338224283496 Set Active after no repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.43322586Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.433247993Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Set check for repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.433269368Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] 127.0.0.1:59657 task reports connection:true","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.433296907Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"4aa9f481-1957-4d0f-b88c-338224283496 Active Active Active","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.433321947Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Set check for repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.433348431Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] 127.0.0.1:60352 task reports connection:true","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.433371737Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"4aa9f481-1957-4d0f-b88c-338224283496 Active Active Active","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.433394406Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Set check for repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.43341577Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] received reconcile message","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.433437652Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.433459608Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.433480954Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] received reconcile message","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.433502418Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.433524131Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.433545451Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] received reconcile message","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.433567069Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.433588618Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.433609939Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"volume 7d24406e-7e52-4892-b53c-b8c4894a13ca activated ok","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.433632428Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"volume 7d24406e-7e52-4892-b53c-b8c4894a13ca constructed and inserted ok","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.433655392Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.433676718Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/7d24406e-7e52-4892-b53c-b8c4894a13ca","method":"POST","req_id":"ccc9200f-cdfc-4eec-bdd1-d6849c77d682","remote_addr":"127.0.0.1:52249","local_addr":"127.0.0.1:52411","component":"dropshot","latency_us":6202,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.434512678Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/7d24406e-7e52-4892-b53c-b8c4894a13ca/import_from_url","method":"POST","req_id":"6f9c82fb-719d-49de-9513-7be45e883cf5","remote_addr":"127.0.0.1:52249","local_addr":"127.0.0.1:52411","component":"dropshot","latency_us":396,"response_code":"200"}
Sep 22 23:14:28.453 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:28.453 INFO Opened existing region file "/tmp/downstairs-57IWH8PC/region.json"
Sep 22 23:14:28.453 INFO Database read version 1
Sep 22 23:14:28.453 INFO Database write version 1
Sep 22 23:14:28.532 DEBG Write     :1000 deps:[] res:true
Sep 22 23:14:28.533 DEBG Write     :1000 deps:[] res:true
Sep 22 23:14:28.534 DEBG Write     :1000 deps:[] res:true
Sep 22 23:14:28.536 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:1 g:1
Sep 22 23:14:28.536 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:1 g:1
Sep 22 23:14:28.536 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:1 g:1
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.5383616Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/job/23606df9-c011-4c56-9be8-e905fe1ea6c9/ok","method":"GET","req_id":"b8b0a52f-d38c-4a65-94b8-cda1910c7ba1","remote_addr":"127.0.0.1:52249","local_addr":"127.0.0.1:52411","component":"dropshot","latency_us":13619,"response_code":"200"}
{"msg":"detach removing entry for volume 7d24406e-7e52-4892-b53c-b8c4894a13ca","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.539037004Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"detaching volume 7d24406e-7e52-4892-b53c-b8c4894a13ca","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.53908456Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:28.539 DEBG Flush     :1002 extent_limit None deps:[] res:true f:2 g:1
Sep 22 23:14:28.539 DEBG Flush     :1002 extent_limit None deps:[] res:true f:2 g:1
Sep 22 23:14:28.539 DEBG Flush     :1002 extent_limit None deps:[] res:true f:2 g:1
{"msg":"Request to deactivate this guest","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.540489043Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"4aa9f481-1957-4d0f-b88c-338224283496 set deactivating.","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.540547652Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.540738795Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/7d24406e-7e52-4892-b53c-b8c4894a13ca","method":"DELETE","req_id":"cc521b2c-c243-46d4-b701-ffef569983d0","remote_addr":"127.0.0.1:52249","local_addr":"127.0.0.1:52411","component":"dropshot","latency_us":1882,"response_code":"204"}
Sep 22 23:14:28.541 INFO Upstairs starts
Sep 22 23:14:28.541 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:14:28.541 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:14:28.541 INFO Crucible stats registered with UUID: 4aa9f481-1957-4d0f-b88c-338224283496
Sep 22 23:14:28.541 INFO Crucible 4aa9f481-1957-4d0f-b88c-338224283496 has session id: 5347d05f-d3e8-4591-83a5-f28549ba8ac1
Sep 22 23:14:28.541 INFO [0] connecting to 127.0.0.1:60580, looper: 0
Sep 22 23:14:28.541 INFO [1] connecting to 127.0.0.1:59657, looper: 1
Sep 22 23:14:28.541 INFO [2] connecting to 127.0.0.1:60352, looper: 2
Sep 22 23:14:28.541 INFO up_listen starts, task: up_listen
Sep 22 23:14:28.541 INFO Wait for all three downstairs to come online
Sep 22 23:14:28.541 INFO Flush timeout: 0.5
Sep 22 23:14:28.541 INFO accepted connection from 127.0.0.1:55344, task: main
Sep 22 23:14:28.542 INFO accepted connection from 127.0.0.1:57999, task: main
Sep 22 23:14:28.542 INFO accepted connection from 127.0.0.1:46059, task: main
Sep 22 23:14:28.542 INFO [0] 4aa9f481-1957-4d0f-b88c-338224283496 looper connected, looper: 0
Sep 22 23:14:28.542 INFO [0] Proc runs for 127.0.0.1:60580 in state New
Sep 22 23:14:28.542 INFO [1] 4aa9f481-1957-4d0f-b88c-338224283496 looper connected, looper: 1
Sep 22 23:14:28.542 INFO [1] Proc runs for 127.0.0.1:59657 in state New
Sep 22 23:14:28.542 INFO [2] 4aa9f481-1957-4d0f-b88c-338224283496 looper connected, looper: 2
Sep 22 23:14:28.542 INFO [2] Proc runs for 127.0.0.1:60352 in state New
Sep 22 23:14:28.542 INFO Connection request from 4aa9f481-1957-4d0f-b88c-338224283496 with version 4, task: proc
Sep 22 23:14:28.542 INFO upstairs UpstairsConnection { upstairs_id: 4aa9f481-1957-4d0f-b88c-338224283496, session_id: e1a9869f-30e8-4dcc-a2a4-de17a7736d6e, gen: 3 } connected, version 4, task: proc
Sep 22 23:14:28.542 INFO Connection request from 4aa9f481-1957-4d0f-b88c-338224283496 with version 4, task: proc
Sep 22 23:14:28.542 INFO upstairs UpstairsConnection { upstairs_id: 4aa9f481-1957-4d0f-b88c-338224283496, session_id: e1a9869f-30e8-4dcc-a2a4-de17a7736d6e, gen: 3 } connected, version 4, task: proc
Sep 22 23:14:28.543 INFO Connection request from 4aa9f481-1957-4d0f-b88c-338224283496 with version 4, task: proc
Sep 22 23:14:28.543 INFO upstairs UpstairsConnection { upstairs_id: 4aa9f481-1957-4d0f-b88c-338224283496, session_id: e1a9869f-30e8-4dcc-a2a4-de17a7736d6e, gen: 3 } connected, version 4, task: proc
Sep 22 23:14:28.543 INFO [0] 4aa9f481-1957-4d0f-b88c-338224283496 (e1a9869f-30e8-4dcc-a2a4-de17a7736d6e) New New New ds_transition to WaitActive
Sep 22 23:14:28.543 INFO [0] Transition from New to WaitActive
Sep 22 23:14:28.543 INFO [1] 4aa9f481-1957-4d0f-b88c-338224283496 (e1a9869f-30e8-4dcc-a2a4-de17a7736d6e) WaitActive New New ds_transition to WaitActive
Sep 22 23:14:28.543 INFO [1] Transition from New to WaitActive
Sep 22 23:14:28.543 INFO [2] 4aa9f481-1957-4d0f-b88c-338224283496 (e1a9869f-30e8-4dcc-a2a4-de17a7736d6e) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:14:28.543 INFO [2] Transition from New to WaitActive
The guest has requested activation
Sep 22 23:14:28.543 INFO 4aa9f481-1957-4d0f-b88c-338224283496 active request set
Sep 22 23:14:28.543 INFO [0] received activate with gen 3
Sep 22 23:14:28.543 INFO [0] client got ds_active_rx, promote! session e1a9869f-30e8-4dcc-a2a4-de17a7736d6e
Sep 22 23:14:28.543 INFO [1] received activate with gen 3
Sep 22 23:14:28.543 INFO [1] client got ds_active_rx, promote! session e1a9869f-30e8-4dcc-a2a4-de17a7736d6e
Sep 22 23:14:28.543 INFO [2] received activate with gen 3
Sep 22 23:14:28.543 INFO [2] client got ds_active_rx, promote! session e1a9869f-30e8-4dcc-a2a4-de17a7736d6e
Sep 22 23:14:28.544 WARN Attempting RW takeover from UpstairsConnection { upstairs_id: 4aa9f481-1957-4d0f-b88c-338224283496, session_id: 7f994951-798d-4ca9-84d4-c9557433fc1b, gen: 1 } to UpstairsConnection { upstairs_id: 4aa9f481-1957-4d0f-b88c-338224283496, session_id: e1a9869f-30e8-4dcc-a2a4-de17a7736d6e, gen: 3 }
Sep 22 23:14:28.544 WARN Signaling to UpstairsConnection { upstairs_id: 4aa9f481-1957-4d0f-b88c-338224283496, session_id: 7f994951-798d-4ca9-84d4-c9557433fc1b, gen: 1 } thread that UpstairsConnection { upstairs_id: 4aa9f481-1957-4d0f-b88c-338224283496, session_id: e1a9869f-30e8-4dcc-a2a4-de17a7736d6e, gen: 3 } is being promoted (read-write)
Sep 22 23:14:28.544 INFO UpstairsConnection { upstairs_id: 4aa9f481-1957-4d0f-b88c-338224283496, session_id: e1a9869f-30e8-4dcc-a2a4-de17a7736d6e, gen: 3 } is now active (read-write)
Sep 22 23:14:28.544 WARN Attempting RW takeover from UpstairsConnection { upstairs_id: 4aa9f481-1957-4d0f-b88c-338224283496, session_id: 7f994951-798d-4ca9-84d4-c9557433fc1b, gen: 1 } to UpstairsConnection { upstairs_id: 4aa9f481-1957-4d0f-b88c-338224283496, session_id: e1a9869f-30e8-4dcc-a2a4-de17a7736d6e, gen: 3 }
Sep 22 23:14:28.544 WARN Signaling to UpstairsConnection { upstairs_id: 4aa9f481-1957-4d0f-b88c-338224283496, session_id: 7f994951-798d-4ca9-84d4-c9557433fc1b, gen: 1 } thread that UpstairsConnection { upstairs_id: 4aa9f481-1957-4d0f-b88c-338224283496, session_id: e1a9869f-30e8-4dcc-a2a4-de17a7736d6e, gen: 3 } is being promoted (read-write)
Sep 22 23:14:28.544 INFO UpstairsConnection { upstairs_id: 4aa9f481-1957-4d0f-b88c-338224283496, session_id: e1a9869f-30e8-4dcc-a2a4-de17a7736d6e, gen: 3 } is now active (read-write)
Sep 22 23:14:28.544 WARN Attempting RW takeover from UpstairsConnection { upstairs_id: 4aa9f481-1957-4d0f-b88c-338224283496, session_id: 7f994951-798d-4ca9-84d4-c9557433fc1b, gen: 1 } to UpstairsConnection { upstairs_id: 4aa9f481-1957-4d0f-b88c-338224283496, session_id: e1a9869f-30e8-4dcc-a2a4-de17a7736d6e, gen: 3 }
Sep 22 23:14:28.544 WARN Signaling to UpstairsConnection { upstairs_id: 4aa9f481-1957-4d0f-b88c-338224283496, session_id: 7f994951-798d-4ca9-84d4-c9557433fc1b, gen: 1 } thread that UpstairsConnection { upstairs_id: 4aa9f481-1957-4d0f-b88c-338224283496, session_id: e1a9869f-30e8-4dcc-a2a4-de17a7736d6e, gen: 3 } is being promoted (read-write)
Sep 22 23:14:28.544 INFO UpstairsConnection { upstairs_id: 4aa9f481-1957-4d0f-b88c-338224283496, session_id: e1a9869f-30e8-4dcc-a2a4-de17a7736d6e, gen: 3 } is now active (read-write)
Sep 22 23:14:28.544 WARN Another upstairs UpstairsConnection { upstairs_id: 4aa9f481-1957-4d0f-b88c-338224283496, session_id: e1a9869f-30e8-4dcc-a2a4-de17a7736d6e, gen: 3 } promoted to active, shutting down connection for UpstairsConnection { upstairs_id: 4aa9f481-1957-4d0f-b88c-338224283496, session_id: 7f994951-798d-4ca9-84d4-c9557433fc1b, gen: 1 }, task: main
Sep 22 23:14:28.544 INFO connection (127.0.0.1:63586): all done
Sep 22 23:14:28.544 WARN Another upstairs UpstairsConnection { upstairs_id: 4aa9f481-1957-4d0f-b88c-338224283496, session_id: e1a9869f-30e8-4dcc-a2a4-de17a7736d6e, gen: 3 } promoted to active, shutting down connection for UpstairsConnection { upstairs_id: 4aa9f481-1957-4d0f-b88c-338224283496, session_id: 7f994951-798d-4ca9-84d4-c9557433fc1b, gen: 1 }, task: main
Sep 22 23:14:28.544 INFO connection (127.0.0.1:59585): all done
Sep 22 23:14:28.544 WARN Another upstairs UpstairsConnection { upstairs_id: 4aa9f481-1957-4d0f-b88c-338224283496, session_id: e1a9869f-30e8-4dcc-a2a4-de17a7736d6e, gen: 3 } promoted to active, shutting down connection for UpstairsConnection { upstairs_id: 4aa9f481-1957-4d0f-b88c-338224283496, session_id: 7f994951-798d-4ca9-84d4-c9557433fc1b, gen: 1 }, task: main
Sep 22 23:14:28.544 INFO connection (127.0.0.1:35200): all done
{"msg":"[0] 4aa9f481-1957-4d0f-b88c-338224283496 (7f994951-798d-4ca9-84d4-c9557433fc1b) cmd_loop saw YouAreNoLongerActive 4aa9f481-1957-4d0f-b88c-338224283496 e1a9869f-30e8-4dcc-a2a4-de17a7736d6e 3","v":0,"name":"crucible-pantry","level":50,"time":"2023-09-22T23:14:28.545321815Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] 4aa9f481-1957-4d0f-b88c-338224283496 (7f994951-798d-4ca9-84d4-c9557433fc1b) Active Active Active ds_transition to Disabled","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.545434658Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] Transition from Active to Disabled","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.54547886Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"4aa9f481-1957-4d0f-b88c-338224283496 set inactive, session 7f994951-798d-4ca9-84d4-c9557433fc1b","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.545520368Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"127.0.0.1:60580: proc: Generation number is too low: saw YouAreNoLongerActive with larger gen 3 than ours 1","v":0,"name":"crucible-pantry","level":50,"time":"2023-09-22T23:14:28.545559105Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"0","component":"datafile"}
{"msg":"[0] 4aa9f481-1957-4d0f-b88c-338224283496 Gone missing, transition from Disabled to Disconnected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.545604543Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] 4aa9f481-1957-4d0f-b88c-338224283496 connection to 127.0.0.1:60580 closed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.545647216Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"0","component":"datafile"}
{"msg":"[1] 4aa9f481-1957-4d0f-b88c-338224283496 (7f994951-798d-4ca9-84d4-c9557433fc1b) cmd_loop saw YouAreNoLongerActive 4aa9f481-1957-4d0f-b88c-338224283496 e1a9869f-30e8-4dcc-a2a4-de17a7736d6e 3","v":0,"name":"crucible-pantry","level":50,"time":"2023-09-22T23:14:28.545696241Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] 4aa9f481-1957-4d0f-b88c-338224283496 (7f994951-798d-4ca9-84d4-c9557433fc1b) Disconnected Active Active ds_transition to Disabled","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.545736664Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] Transition from Active to Disabled","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.545770806Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"4aa9f481-1957-4d0f-b88c-338224283496 set inactive, session 7f994951-798d-4ca9-84d4-c9557433fc1b","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.545802385Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"127.0.0.1:59657: proc: Generation number is too low: saw YouAreNoLongerActive with larger gen 3 than ours 1","v":0,"name":"crucible-pantry","level":50,"time":"2023-09-22T23:14:28.545828744Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"1","component":"datafile"}
{"msg":"[1] 4aa9f481-1957-4d0f-b88c-338224283496 Gone missing, transition from Disabled to Disconnected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.545853687Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] 4aa9f481-1957-4d0f-b88c-338224283496 connection to 127.0.0.1:59657 closed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.545876418Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"1","component":"datafile"}
{"msg":"[2] 4aa9f481-1957-4d0f-b88c-338224283496 (7f994951-798d-4ca9-84d4-c9557433fc1b) cmd_loop saw YouAreNoLongerActive 4aa9f481-1957-4d0f-b88c-338224283496 e1a9869f-30e8-4dcc-a2a4-de17a7736d6e 3","v":0,"name":"crucible-pantry","level":50,"time":"2023-09-22T23:14:28.54591542Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] 4aa9f481-1957-4d0f-b88c-338224283496 (7f994951-798d-4ca9-84d4-c9557433fc1b) Disconnected Disconnected Active ds_transition to Disabled","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.545957938Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] Transition from Active to Disabled","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.545992433Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"4aa9f481-1957-4d0f-b88c-338224283496 set inactive, session 7f994951-798d-4ca9-84d4-c9557433fc1b","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.546020157Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"127.0.0.1:60352: proc: Generation number is too low: saw YouAreNoLongerActive with larger gen 3 than ours 1","v":0,"name":"crucible-pantry","level":50,"time":"2023-09-22T23:14:28.546048761Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"2","component":"datafile"}
{"msg":"[2] 4aa9f481-1957-4d0f-b88c-338224283496 Gone missing, transition from Disabled to Disconnected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.546092297Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] 4aa9f481-1957-4d0f-b88c-338224283496 connection to 127.0.0.1:60352 closed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.546140766Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"2","component":"datafile"}
{"msg":"[0] pm_task rx.recv() is None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:14:28.546185305Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] 127.0.0.1:60580 task reports connection:false","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.546223838Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"4aa9f481-1957-4d0f-b88c-338224283496 Disconnected Disconnected Disconnected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.546262145Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] 127.0.0.1:60580 task reports offline","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.546303503Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] 127.0.0.1:59657 task reports connection:false","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.546344456Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:28.546 INFO [0] downstairs client at 127.0.0.1:60580 has UUID 46e0ee2f-0f49-4348-a954-78960194fdcd
{"msg":"4aa9f481-1957-4d0f-b88c-338224283496 Disconnected Disconnected Disconnected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.546388805Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:28.546 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 46e0ee2f-0f49-4348-a954-78960194fdcd, encrypted: true, database_read_version: 1, database_write_version: 1 }
{"msg":"[1] 127.0.0.1:59657 task reports offline","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.546428423Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:28.546 INFO 4aa9f481-1957-4d0f-b88c-338224283496 WaitActive WaitActive WaitActive
{"msg":"[2] 127.0.0.1:60352 task reports connection:false","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.546463447Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"4aa9f481-1957-4d0f-b88c-338224283496 Disconnected Disconnected Disconnected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.546505944Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:28.546 INFO [1] downstairs client at 127.0.0.1:59657 has UUID 3b4e7e40-734f-4eaa-bb4c-32e1ad5c2936
{"msg":"[2] 127.0.0.1:60352 task reports offline","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:28.546545051Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:28.546 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 3b4e7e40-734f-4eaa-bb4c-32e1ad5c2936, encrypted: true, database_read_version: 1, database_write_version: 1 }
{"msg":"[1] pm_task rx.recv() is None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:14:28.546585246Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:28.546 INFO 4aa9f481-1957-4d0f-b88c-338224283496 WaitActive WaitActive WaitActive
{"msg":"[2] pm_task rx.recv() is None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:14:28.546625893Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:28.546 INFO [2] downstairs client at 127.0.0.1:60352 has UUID 9706eaea-4b57-4d6f-860a-37f6ab1693f0
Sep 22 23:14:28.546 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 9706eaea-4b57-4d6f-860a-37f6ab1693f0, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:28.546 INFO 4aa9f481-1957-4d0f-b88c-338224283496 WaitActive WaitActive WaitActive
Sep 22 23:14:28.546 INFO Current flush_numbers [0..12]: [1, 1]
Sep 22 23:14:28.547 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:28.547 INFO Current flush_numbers [0..12]: [1, 1]
Sep 22 23:14:28.548 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:28.548 INFO Current flush_numbers [0..12]: [1, 1]
Sep 22 23:14:28.548 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:28.548 INFO [0] 4aa9f481-1957-4d0f-b88c-338224283496 (e1a9869f-30e8-4dcc-a2a4-de17a7736d6e) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:28.548 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:14:28.548 WARN [0] new RM replaced this: None
Sep 22 23:14:28.548 INFO [0] Starts reconcile loop
Sep 22 23:14:28.548 INFO [1] 4aa9f481-1957-4d0f-b88c-338224283496 (e1a9869f-30e8-4dcc-a2a4-de17a7736d6e) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:28.548 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:14:28.548 WARN [1] new RM replaced this: None
Sep 22 23:14:28.548 INFO [1] Starts reconcile loop
Sep 22 23:14:28.548 INFO [2] 4aa9f481-1957-4d0f-b88c-338224283496 (e1a9869f-30e8-4dcc-a2a4-de17a7736d6e) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:14:28.548 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:14:28.548 WARN [2] new RM replaced this: None
Sep 22 23:14:28.548 INFO [2] Starts reconcile loop
Sep 22 23:14:28.548 INFO [0] 127.0.0.1:60580 task reports connection:true
Sep 22 23:14:28.548 INFO 4aa9f481-1957-4d0f-b88c-338224283496 WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:14:28.548 INFO [0]R  flush_numbers: [1, 1]
Sep 22 23:14:28.548 INFO [0]R  generation: [1, 1]
Sep 22 23:14:28.548 INFO [0]R  dirty: [false, false]
Sep 22 23:14:28.549 INFO [1]R  flush_numbers: [1, 1]
Sep 22 23:14:28.549 INFO [1]R  generation: [1, 1]
Sep 22 23:14:28.549 INFO [1]R  dirty: [false, false]
Sep 22 23:14:28.549 INFO [2]R  flush_numbers: [1, 1]
Sep 22 23:14:28.549 INFO [2]R  generation: [1, 1]
Sep 22 23:14:28.549 INFO [2]R  dirty: [false, false]
Sep 22 23:14:28.549 INFO Max found gen is 2
Sep 22 23:14:28.549 INFO Generation requested: 3 >= found:2
Sep 22 23:14:28.549 INFO Next flush: 2
Sep 22 23:14:28.549 INFO All extents match
Sep 22 23:14:28.549 INFO No downstairs repair required
Sep 22 23:14:28.549 INFO No initial repair work was required
Sep 22 23:14:28.549 INFO Set Downstairs and Upstairs active
Sep 22 23:14:28.549 INFO 4aa9f481-1957-4d0f-b88c-338224283496 is now active with session: e1a9869f-30e8-4dcc-a2a4-de17a7736d6e
Sep 22 23:14:28.549 INFO 4aa9f481-1957-4d0f-b88c-338224283496 Set Active after no repair
Sep 22 23:14:28.549 INFO Notify all downstairs, region set compare is done.
Sep 22 23:14:28.549 INFO Set check for repair
Sep 22 23:14:28.549 INFO [1] 127.0.0.1:59657 task reports connection:true
Sep 22 23:14:28.549 INFO 4aa9f481-1957-4d0f-b88c-338224283496 Active Active Active
Sep 22 23:14:28.549 INFO Set check for repair
Sep 22 23:14:28.549 INFO [2] 127.0.0.1:60352 task reports connection:true
Sep 22 23:14:28.549 INFO 4aa9f481-1957-4d0f-b88c-338224283496 Active Active Active
Sep 22 23:14:28.549 INFO Set check for repair
Sep 22 23:14:28.549 INFO [0] received reconcile message
Sep 22 23:14:28.549 INFO [0] All repairs completed, exit
Sep 22 23:14:28.549 INFO [0] Starts cmd_loop
Sep 22 23:14:28.549 INFO [1] received reconcile message
Sep 22 23:14:28.549 INFO [1] All repairs completed, exit
Sep 22 23:14:28.549 INFO [1] Starts cmd_loop
Sep 22 23:14:28.549 INFO [2] received reconcile message
Sep 22 23:14:28.549 INFO [2] All repairs completed, exit
Sep 22 23:14:28.549 INFO [2] Starts cmd_loop
The guest has finished waiting for activation
Sep 22 23:14:28.550 DEBG IO Read  1000 has deps []
Sep 22 23:14:28.551 DEBG Read      :1000 deps:[] res:true
Sep 22 23:14:28.551 DEBG Read      :1000 deps:[] res:true
Sep 22 23:14:28.551 DEBG Read      :1000 deps:[] res:true
Sep 22 23:14:28.555 DEBG [0] Read AckReady 1000, : downstairs
Sep 22 23:14:28.558 DEBG [1] Read already AckReady 1000, : downstairs
Sep 22 23:14:28.560 DEBG [2] Read already AckReady 1000, : downstairs
Sep 22 23:14:28.560 DEBG up_ds_listen was notified
Sep 22 23:14:28.560 DEBG up_ds_listen process 1000
Sep 22 23:14:28.561 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:14:28.561 DEBG up_ds_listen checked 1 jobs, back to waiting
test test::test_pantry_import_from_local_server ... ok
Sep 22 23:14:28.572 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:28.572 INFO Created new region file "/tmp/downstairs-Xs2g0Vs2/region.json"
Sep 22 23:14:28.617 INFO Checking if live repair is needed
Sep 22 23:14:28.617 INFO No Live Repair required at this time
Sep 22 23:14:28.636 INFO accepted connection from 127.0.0.1:56661, task: main
Sep 22 23:14:28.636 INFO [0] a50fc3fb-3de2-4743-9c50-cc80cfba77db looper connected, looper: 0
Sep 22 23:14:28.636 INFO [0] Proc runs for 127.0.0.1:62845 in state Replaced
Sep 22 23:14:28.637 INFO Connection request from a50fc3fb-3de2-4743-9c50-cc80cfba77db with version 4, task: proc
Sep 22 23:14:28.637 INFO upstairs UpstairsConnection { upstairs_id: a50fc3fb-3de2-4743-9c50-cc80cfba77db, session_id: 2b698136-bbd2-4fcb-ab7c-5f19b2267eb8, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:28.637 INFO [0] upstairs guest_io_ready=TRUE, promote! session 2b698136-bbd2-4fcb-ab7c-5f19b2267eb8
Sep 22 23:14:28.637 INFO UpstairsConnection { upstairs_id: a50fc3fb-3de2-4743-9c50-cc80cfba77db, session_id: 2b698136-bbd2-4fcb-ab7c-5f19b2267eb8, gen: 1 } is now active (read-write)
Sep 22 23:14:28.637 INFO [0] downstairs client at 127.0.0.1:62845 has UUID ae376576-6754-4185-9b1d-d23a94866419
Sep 22 23:14:28.637 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: ae376576-6754-4185-9b1d-d23a94866419, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:28.637 WARN [0] replace downstairs uuid:122e249d-ed33-49c6-bb2e-b79a59ebf9a6 with ae376576-6754-4185-9b1d-d23a94866419
Sep 22 23:14:28.637 INFO a50fc3fb-3de2-4743-9c50-cc80cfba77db Replaced Active Active
Sep 22 23:14:28.637 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:28.638 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:28.638 INFO [0] a50fc3fb-3de2-4743-9c50-cc80cfba77db (2b698136-bbd2-4fcb-ab7c-5f19b2267eb8) Replaced Active Active ds_transition to LiveRepairReady
Sep 22 23:14:28.638 INFO [0] Transition from Replaced to LiveRepairReady
Sep 22 23:14:28.638 WARN [0] new RM replaced this: None
Sep 22 23:14:28.638 WARN [0] a50fc3fb-3de2-4743-9c50-cc80cfba77db Enter Ready for LiveRepair mode
Sep 22 23:14:28.638 INFO [0] Starts cmd_loop
Sep 22 23:14:28.638 INFO [0] 127.0.0.1:62845 task reports connection:true
Sep 22 23:14:28.638 INFO a50fc3fb-3de2-4743-9c50-cc80cfba77db LiveRepairReady Active Active
Sep 22 23:14:28.638 INFO Set check for repair
Sep 22 23:14:28.681 INFO UUID: d2a767c4-8e80-4f99-8f17-2410bd865538
Sep 22 23:14:28.681 INFO Blocks per extent:512 Total Extents: 188
Sep 22 23:14:28.681 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:28.681 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:28.681 INFO Using address: 127.0.0.1:59093, task: main
Sep 22 23:14:28.682 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:28.682 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:36653, task: repair
Sep 22 23:14:28.682 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:36653, task: repair
Sep 22 23:14:28.682 INFO listening, local_addr: 127.0.0.1:36653, task: repair
Sep 22 23:14:28.682 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:36653, task: repair
Sep 22 23:14:28.682 INFO Using repair address: 127.0.0.1:36653, task: main
Sep 22 23:14:28.682 INFO No SSL acceptor configured, task: main
Sep 22 23:14:28.688 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:28.688 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:28.688 INFO Opened existing region file "/tmp/downstairs-kApwRwyr/region.json"
Sep 22 23:14:28.688 INFO Database read version 1
Sep 22 23:14:28.688 INFO Database write version 1
Sep 22 23:14:28.711 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:28.711 INFO Opened existing region file "/tmp/downstairs-cOOWrUaJ/region.json"
Sep 22 23:14:28.711 INFO Database read version 1
Sep 22 23:14:28.711 INFO Database write version 1
Sep 22 23:14:28.756 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:28.756 INFO Created new region file "/tmp/downstairs-i41xVTeA/region.json"
Sep 22 23:14:28.756 INFO UUID: 10109090-7e8f-489a-9e4a-65ce6167d00b
Sep 22 23:14:28.756 INFO Blocks per extent:512 Total Extents: 188
Sep 22 23:14:28.756 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:28.756 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:28.756 INFO Using address: 127.0.0.1:62776, task: main
Sep 22 23:14:28.757 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:28.757 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:61515, task: repair
Sep 22 23:14:28.757 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:61515, task: repair
Sep 22 23:14:28.757 INFO listening, local_addr: 127.0.0.1:61515, task: repair
Sep 22 23:14:28.757 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:61515, task: repair
Sep 22 23:14:28.757 INFO Using repair address: 127.0.0.1:61515, task: main
Sep 22 23:14:28.757 INFO No SSL acceptor configured, task: main
Sep 22 23:14:28.804 INFO UUID: bec31c6c-c0ce-43db-8467-8d94b73527a2
Sep 22 23:14:28.805 INFO Blocks per extent:512 Total Extents: 188
Sep 22 23:14:28.805 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:28.805 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:28.805 INFO Using address: 127.0.0.1:62296, task: main
Sep 22 23:14:28.805 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:28.805 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:51430, task: repair
Sep 22 23:14:28.805 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:51430, task: repair
Sep 22 23:14:28.805 INFO listening, local_addr: 127.0.0.1:51430, task: repair
Sep 22 23:14:28.805 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:51430, task: repair
Sep 22 23:14:28.805 INFO Using repair address: 127.0.0.1:51430, task: main
Sep 22 23:14:28.805 INFO No SSL acceptor configured, task: main
Sep 22 23:14:28.821 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:28.821 INFO Created new region file "/tmp/downstairs-OiIt4Fiz/region.json"
Sep 22 23:14:28.857 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:28.857 INFO Opened existing region file "/tmp/downstairs-Xs2g0Vs2/region.json"
Sep 22 23:14:28.857 INFO Database read version 1
Sep 22 23:14:28.857 INFO Database write version 1
Sep 22 23:14:28.866 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:28.866 INFO Created new region file "/tmp/downstairs-H2YCEjAN/region.json"
Sep 22 23:14:28.874 INFO Checking if live repair is needed
Sep 22 23:14:28.874 INFO No Live Repair required at this time
Sep 22 23:14:28.890 INFO accepted connection from 127.0.0.1:35498, task: main
Sep 22 23:14:28.890 INFO [0] e94af85e-3796-4fab-91a1-f12add9c3020 looper connected, looper: 0
Sep 22 23:14:28.890 INFO [0] Proc runs for 127.0.0.1:56689 in state Replaced
Sep 22 23:14:28.890 INFO Connection request from e94af85e-3796-4fab-91a1-f12add9c3020 with version 4, task: proc
Sep 22 23:14:28.890 INFO upstairs UpstairsConnection { upstairs_id: e94af85e-3796-4fab-91a1-f12add9c3020, session_id: a86ae319-11ad-4b2d-a80b-330efc92ebef, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:28.890 INFO [0] upstairs guest_io_ready=TRUE, promote! session a86ae319-11ad-4b2d-a80b-330efc92ebef
Sep 22 23:14:28.890 INFO UpstairsConnection { upstairs_id: e94af85e-3796-4fab-91a1-f12add9c3020, session_id: a86ae319-11ad-4b2d-a80b-330efc92ebef, gen: 1 } is now active (read-write)
Sep 22 23:14:28.891 INFO [0] downstairs client at 127.0.0.1:56689 has UUID b7c0d6d3-333f-4423-864b-ebd0befef183
Sep 22 23:14:28.891 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: b7c0d6d3-333f-4423-864b-ebd0befef183, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:28.891 WARN [0] replace downstairs uuid:30596aa6-8959-4f8c-98b8-ac91fdc30fc1 with b7c0d6d3-333f-4423-864b-ebd0befef183
Sep 22 23:14:28.891 INFO e94af85e-3796-4fab-91a1-f12add9c3020 Replaced Active Active
Sep 22 23:14:28.891 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:28.891 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:28.891 INFO [0] e94af85e-3796-4fab-91a1-f12add9c3020 (a86ae319-11ad-4b2d-a80b-330efc92ebef) Replaced Active Active ds_transition to LiveRepairReady
Sep 22 23:14:28.891 INFO [0] Transition from Replaced to LiveRepairReady
Sep 22 23:14:28.891 WARN [0] new RM replaced this: None
Sep 22 23:14:28.891 WARN [0] e94af85e-3796-4fab-91a1-f12add9c3020 Enter Ready for LiveRepair mode
Sep 22 23:14:28.891 INFO [0] Starts cmd_loop
Sep 22 23:14:28.892 INFO [0] 127.0.0.1:56689 task reports connection:true
Sep 22 23:14:28.892 INFO e94af85e-3796-4fab-91a1-f12add9c3020 LiveRepairReady Active Active
Sep 22 23:14:28.892 INFO Set check for repair
Sep 22 23:14:28.989 INFO UUID: df366b3b-ce8f-4a81-b993-9efbc4225b81
Sep 22 23:14:28.989 INFO Blocks per extent:512 Total Extents: 188
Sep 22 23:14:28.989 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:28.989 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:28.989 INFO Using address: 127.0.0.1:64149, task: main
Sep 22 23:14:28.989 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:28.989 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:14:28.989 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:14:28.989 INFO listening, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:14:28.990 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:14:28.990 INFO Using repair address: 127.0.0.1:46213, task: main
Sep 22 23:14:28.990 INFO No SSL acceptor configured, task: main
Sep 22 23:14:29.015 INFO UUID: b38dafac-2da7-4558-aae1-959c564193b2
Sep 22 23:14:29.016 INFO Blocks per extent:512 Total Extents: 188
Sep 22 23:14:29.016 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:29.016 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:29.016 INFO Using address: 127.0.0.1:38920, task: main
Sep 22 23:14:29.016 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:29.016 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:49456, task: repair
Sep 22 23:14:29.016 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:49456, task: repair
Sep 22 23:14:29.016 INFO listening, local_addr: 127.0.0.1:49456, task: repair
Sep 22 23:14:29.016 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:49456, task: repair
Sep 22 23:14:29.016 INFO Using repair address: 127.0.0.1:49456, task: main
Sep 22 23:14:29.016 INFO No SSL acceptor configured, task: main
Sep 22 23:14:29.058 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:29.059 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:29.059 INFO Opened existing region file "/tmp/downstairs-i41xVTeA/region.json"
Sep 22 23:14:29.059 INFO Database read version 1
Sep 22 23:14:29.059 INFO Database write version 1
Sep 22 23:14:29.062 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:29.062 INFO Created new region file "/tmp/downstairs-gGpp9x1g/region.json"
Sep 22 23:14:29.076 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:29.076 INFO Created new region file "/tmp/downstairs-uiRa834G/region.json"
Sep 22 23:14:29.128 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:29.129 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:29.129 INFO Opened existing region file "/tmp/downstairs-OiIt4Fiz/region.json"
Sep 22 23:14:29.129 INFO Database read version 1
Sep 22 23:14:29.129 INFO Database write version 1
Sep 22 23:14:29.207 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:29.207 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:29.207 INFO Opened existing region file "/tmp/downstairs-H2YCEjAN/region.json"
Sep 22 23:14:29.207 INFO Database read version 1
Sep 22 23:14:29.207 INFO Database write version 1
Sep 22 23:14:29.240 INFO UUID: ed5caa6d-3818-4a41-afcf-7deb29720fa0
Sep 22 23:14:29.240 INFO Blocks per extent:512 Total Extents: 188
Sep 22 23:14:29.240 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:29.240 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:29.240 INFO Using address: 127.0.0.1:46850, task: main
Sep 22 23:14:29.240 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:29.240 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:45771, task: repair
Sep 22 23:14:29.240 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:45771, task: repair
Sep 22 23:14:29.241 INFO listening, local_addr: 127.0.0.1:45771, task: repair
Sep 22 23:14:29.241 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:45771, task: repair
Sep 22 23:14:29.241 INFO Using repair address: 127.0.0.1:45771, task: main
Sep 22 23:14:29.241 INFO No SSL acceptor configured, task: main
Sep 22 23:14:29.330 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:29.331 INFO Created new region file "/tmp/downstairs-XicHDLp1/region.json"
Sep 22 23:14:29.416 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:29.417 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:29.417 INFO Opened existing region file "/tmp/downstairs-gGpp9x1g/region.json"
Sep 22 23:14:29.417 INFO Database read version 1
Sep 22 23:14:29.417 INFO Database write version 1
Sep 22 23:14:29.452 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:29.453 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:29.453 INFO Opened existing region file "/tmp/downstairs-uiRa834G/region.json"
Sep 22 23:14:29.453 INFO Database read version 1
Sep 22 23:14:29.453 INFO Database write version 1
Sep 22 23:14:29.462 INFO UUID: a0cbf067-d00b-4cf2-ba16-d1cbb9ac9209
Sep 22 23:14:29.462 INFO Blocks per extent:512 Total Extents: 188
Sep 22 23:14:29.462 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:29.462 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:29.462 INFO Using address: 127.0.0.1:46586, task: main
Sep 22 23:14:29.462 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:29.462 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:56696, task: repair
Sep 22 23:14:29.462 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:56696, task: repair
Sep 22 23:14:29.462 INFO listening, local_addr: 127.0.0.1:56696, task: repair
Sep 22 23:14:29.463 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:56696, task: repair
Sep 22 23:14:29.463 INFO Using repair address: 127.0.0.1:56696, task: main
Sep 22 23:14:29.463 INFO No SSL acceptor configured, task: main
Sep 22 23:14:29.517 INFO UUID: 02d6cbbb-c0cd-42b6-b416-45900a2ec4e0
Sep 22 23:14:29.517 INFO Blocks per extent:512 Total Extents: 188
Sep 22 23:14:29.518 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:29.518 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:29.518 INFO Using address: 127.0.0.1:60728, task: main
Sep 22 23:14:29.518 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:29.518 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:59468, task: repair
Sep 22 23:14:29.518 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:59468, task: repair
Sep 22 23:14:29.518 INFO listening, local_addr: 127.0.0.1:59468, task: repair
Sep 22 23:14:29.518 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:59468, task: repair
Sep 22 23:14:29.518 INFO Using repair address: 127.0.0.1:59468, task: main
Sep 22 23:14:29.518 INFO No SSL acceptor configured, task: main
Sep 22 23:14:29.548 INFO Upstairs starts
Sep 22 23:14:29.548 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:14:29.548 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:14:29.548 INFO Crucible stats registered with UUID: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6
Sep 22 23:14:29.548 INFO Crucible 4cf66ffa-a12c-4555-b256-f1d6116cf5e6 has session id: 6e6def71-e8d5-48fa-90f4-5751ac48d9ee
Sep 22 23:14:29.549 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:29.549 INFO [0] connecting to 127.0.0.1:53298, looper: 0
Sep 22 23:14:29.549 INFO [1] connecting to 127.0.0.1:59093, looper: 1
Sep 22 23:14:29.549 INFO [2] connecting to 127.0.0.1:46586, looper: 2
Sep 22 23:14:29.549 INFO up_listen starts, task: up_listen
Sep 22 23:14:29.549 INFO Wait for all three downstairs to come online
Sep 22 23:14:29.549 INFO Flush timeout: 0.5
Sep 22 23:14:29.549 INFO accepted connection from 127.0.0.1:43836, task: main
Sep 22 23:14:29.550 INFO [0] 4cf66ffa-a12c-4555-b256-f1d6116cf5e6 looper connected, looper: 0
Sep 22 23:14:29.550 INFO [0] Proc runs for 127.0.0.1:53298 in state New
Sep 22 23:14:29.550 INFO [2] 4cf66ffa-a12c-4555-b256-f1d6116cf5e6 looper connected, looper: 2
Sep 22 23:14:29.550 INFO [2] Proc runs for 127.0.0.1:46586 in state New
Sep 22 23:14:29.550 INFO accepted connection from 127.0.0.1:46522, task: main
Sep 22 23:14:29.550 INFO [1] 4cf66ffa-a12c-4555-b256-f1d6116cf5e6 looper connected, looper: 1
Sep 22 23:14:29.550 INFO [1] Proc runs for 127.0.0.1:59093 in state New
Sep 22 23:14:29.550 INFO accepted connection from 127.0.0.1:49426, task: main
Sep 22 23:14:29.550 INFO Connection request from 4cf66ffa-a12c-4555-b256-f1d6116cf5e6 with version 4, task: proc
Sep 22 23:14:29.550 INFO upstairs UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:29.550 INFO Connection request from 4cf66ffa-a12c-4555-b256-f1d6116cf5e6 with version 4, task: proc
Sep 22 23:14:29.550 INFO upstairs UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:29.551 INFO Connection request from 4cf66ffa-a12c-4555-b256-f1d6116cf5e6 with version 4, task: proc
Sep 22 23:14:29.551 INFO upstairs UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } connected, version 4, task: proc
The guest has requested activation
Sep 22 23:14:29.551 INFO [0] 4cf66ffa-a12c-4555-b256-f1d6116cf5e6 (61abb1c6-aab0-4918-867b-523054aca8f6) New New New ds_transition to WaitActive
Sep 22 23:14:29.551 INFO [0] Transition from New to WaitActive
Sep 22 23:14:29.551 INFO [2] 4cf66ffa-a12c-4555-b256-f1d6116cf5e6 (61abb1c6-aab0-4918-867b-523054aca8f6) WaitActive New New ds_transition to WaitActive
Sep 22 23:14:29.551 INFO [2] Transition from New to WaitActive
Sep 22 23:14:29.551 INFO [1] 4cf66ffa-a12c-4555-b256-f1d6116cf5e6 (61abb1c6-aab0-4918-867b-523054aca8f6) WaitActive New WaitActive ds_transition to WaitActive
Sep 22 23:14:29.551 INFO [1] Transition from New to WaitActive
Sep 22 23:14:29.551 INFO 4cf66ffa-a12c-4555-b256-f1d6116cf5e6 active request set
Sep 22 23:14:29.551 INFO [0] received activate with gen 1
Sep 22 23:14:29.551 INFO [0] client got ds_active_rx, promote! session 61abb1c6-aab0-4918-867b-523054aca8f6
Sep 22 23:14:29.551 INFO [1] received activate with gen 1
Sep 22 23:14:29.551 INFO [1] client got ds_active_rx, promote! session 61abb1c6-aab0-4918-867b-523054aca8f6
Sep 22 23:14:29.551 INFO [2] received activate with gen 1
Sep 22 23:14:29.551 INFO [2] client got ds_active_rx, promote! session 61abb1c6-aab0-4918-867b-523054aca8f6
Sep 22 23:14:29.552 INFO UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } is now active (read-write)
Sep 22 23:14:29.552 INFO UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } is now active (read-write)
Sep 22 23:14:29.552 INFO UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } is now active (read-write)
Sep 22 23:14:29.553 INFO [0] downstairs client at 127.0.0.1:53298 has UUID bbec14f8-e0c9-4fdd-8394-c7675df3174b
Sep 22 23:14:29.553 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 512, shift: 9 }, extent_count: 188, uuid: bbec14f8-e0c9-4fdd-8394-c7675df3174b, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:29.553 INFO 4cf66ffa-a12c-4555-b256-f1d6116cf5e6 WaitActive WaitActive WaitActive
Sep 22 23:14:29.553 INFO [2] downstairs client at 127.0.0.1:46586 has UUID a0cbf067-d00b-4cf2-ba16-d1cbb9ac9209
Sep 22 23:14:29.553 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 512, shift: 9 }, extent_count: 188, uuid: a0cbf067-d00b-4cf2-ba16-d1cbb9ac9209, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:29.553 INFO 4cf66ffa-a12c-4555-b256-f1d6116cf5e6 WaitActive WaitActive WaitActive
Sep 22 23:14:29.553 INFO [1] downstairs client at 127.0.0.1:59093 has UUID d2a767c4-8e80-4f99-8f17-2410bd865538
Sep 22 23:14:29.553 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 512, shift: 9 }, extent_count: 188, uuid: d2a767c4-8e80-4f99-8f17-2410bd865538, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:29.553 INFO 4cf66ffa-a12c-4555-b256-f1d6116cf5e6 WaitActive WaitActive WaitActive
Sep 22 23:14:29.559 INFO UUID: ede072e9-d3f2-4884-bb86-eea53898f9d9
Sep 22 23:14:29.559 INFO Blocks per extent:512 Total Extents: 188
Sep 22 23:14:29.559 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:29.559 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:29.559 INFO Using address: 127.0.0.1:62299, task: main
Sep 22 23:14:29.559 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:29.559 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:63792, task: repair
Sep 22 23:14:29.559 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:63792, task: repair
Sep 22 23:14:29.559 INFO listening, local_addr: 127.0.0.1:63792, task: repair
Sep 22 23:14:29.560 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:63792, task: repair
Sep 22 23:14:29.560 INFO Using repair address: 127.0.0.1:63792, task: main
Sep 22 23:14:29.560 INFO No SSL acceptor configured, task: main
Sep 22 23:14:29.573 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:29.573 INFO Created new region file "/tmp/downstairs-ybEDH4hM/region.json"
Sep 22 23:14:29.605 INFO Current flush_numbers [0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:14:29.610 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:29.610 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:29.610 INFO Opened existing region file "/tmp/downstairs-XicHDLp1/region.json"
Sep 22 23:14:29.610 INFO Database read version 1
Sep 22 23:14:29.610 INFO Database write version 1
Sep 22 23:14:29.612 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:29.612 INFO Created new region file "/tmp/downstairs-GUu8sV6m/region.json"
Sep 22 23:14:29.614 INFO Current flush_numbers [0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:14:29.624 INFO Current flush_numbers [0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:14:29.636 WARN a50fc3fb-3de2-4743-9c50-cc80cfba77db request to replace downstairs 127.0.0.1:42970 with 127.0.0.1:62845
Sep 22 23:14:29.636 INFO a50fc3fb-3de2-4743-9c50-cc80cfba77db found new target: 127.0.0.1:62845 at 0
Sep 22 23:14:29.636 INFO Waiting for replacement to finish
Sep 22 23:14:29.639 INFO Checking if live repair is needed
Sep 22 23:14:29.639 INFO [0] a50fc3fb-3de2-4743-9c50-cc80cfba77db (2b698136-bbd2-4fcb-ab7c-5f19b2267eb8) LiveRepairReady Active Active ds_transition to LiveRepair
Sep 22 23:14:29.639 INFO [0] Transition from LiveRepairReady to LiveRepair
Sep 22 23:14:29.639 INFO Live Repair started
Sep 22 23:14:29.640 WARN Live Repair main task begins., task: repair
Sep 22 23:14:29.640 INFO Start Live Repair of extents 0 to 2, task: repair
Sep 22 23:14:29.640 INFO Start extent 0 repair, task: repair
Sep 22 23:14:29.640 DEBG RE:0 Repair extent begins
Sep 22 23:14:29.640 DEBG Create new job ids for 0, : downstairs
Sep 22 23:14:29.640 INFO RE:0 repair extent with ids 1002,1003,1004,1005 deps:[]
Sep 22 23:14:29.640 DEBG Enqueue repair job 1005, : downstairs
Sep 22 23:14:29.640 DEBG Enqueue repair job 1002, : downstairs
Sep 22 23:14:29.640 INFO RE:0 close id:1002 queued, notify DS
Sep 22 23:14:29.640 INFO RE:0 Wait for result from close command 1002:3
Sep 22 23:14:29.640 DEBG [0] 1002 Remove check skipped:{JobId(1001)} from deps:[], : downstairs
Sep 22 23:14:29.640 DEBG [0] 1002 Remove check < min repaired:1002 from deps:[], : downstairs
Sep 22 23:14:29.640 INFO [0] 1002 final dependency list [], : downstairs
Sep 22 23:14:29.640 DEBG [0] 1005 Remove check skipped:{JobId(1001)} from deps:[JobId(1002), JobId(1003), JobId(1004)], : downstairs
Sep 22 23:14:29.640 DEBG [0] 1005 Remove check < min repaired:1002 from deps:[JobId(1002), JobId(1003), JobId(1004)], : downstairs
Sep 22 23:14:29.640 INFO [0] 1005 final dependency list [JobId(1002), JobId(1003), JobId(1004)], : downstairs
Sep 22 23:14:29.641 DEBG Flush just extent 0 with f:2 and g:1
Sep 22 23:14:29.642 DEBG FlushClose:1002 extent 0 deps:[] res:true f:2 g:1
Sep 22 23:14:29.642 DEBG Flush just extent 0 with f:2 and g:1
Sep 22 23:14:29.643 DEBG FlushClose:1002 extent 0 deps:[] res:true f:2 g:1
Sep 22 23:14:29.645 DEBG JustClose :1002 extent 0 deps:[] res:true
Sep 22 23:14:29.645 DEBG [1] ELC got g:1 f:1 d:false
Sep 22 23:14:29.645 DEBG [2] ELC got g:1 f:1 d:false
Sep 22 23:14:29.645 DEBG [0] ELC got g:0 f:0 d:false
Sep 22 23:14:29.645 DEBG [0] ExtentFlushClose 1002 AckReady, : downstairs
Sep 22 23:14:29.645 DEBG up_ds_listen was notified
Sep 22 23:14:29.645 DEBG up_ds_listen process 1002
Sep 22 23:14:29.645 DEBG [A] ack job 1002:3, : downstairs
Sep 22 23:14:29.645 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:29.645 DEBG Extent 0 id:1002 Done
Sep 22 23:14:29.645 DEBG Get repair info for 2 source, : downstairs
Sep 22 23:14:29.645 DEBG Get repair info for 0 bad, : downstairs
Sep 22 23:14:29.645 INFO Repair for extent 0 s:2 d:[ClientId(0)], : downstairs
Sep 22 23:14:29.645 DEBG Enqueue repair job 1003, : downstairs
Sep 22 23:14:29.645 INFO RE:0 Wait for result from repair command 1003:4
Sep 22 23:14:29.645 DEBG [0] 1003 Remove check skipped:{JobId(1001)} from deps:[JobId(1002)], : downstairs
Sep 22 23:14:29.645 DEBG [0] 1003 Remove check < min repaired:1002 from deps:[JobId(1002)], : downstairs
Sep 22 23:14:29.645 INFO [0] 1003 final dependency list [JobId(1002)], : downstairs
Sep 22 23:14:29.646 DEBG Received NoOP 1003
Sep 22 23:14:29.646 DEBG Received NoOP 1003
Sep 22 23:14:29.646 DEBG Received ExtentLiveRepair 1003
Sep 22 23:14:29.646 DEBG Work of: LiveNoOp 1003
Sep 22 23:14:29.646 DEBG LiveNoOp  :1003 deps:[JobId(1002)] res:true
Sep 22 23:14:29.646 WARN 1005 job ELiveReopen for connection UpstairsConnection { upstairs_id: a50fc3fb-3de2-4743-9c50-cc80cfba77db, session_id: 2b698136-bbd2-4fcb-ab7c-5f19b2267eb8, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:14:29.646 DEBG Work of: LiveNoOp 1003
Sep 22 23:14:29.646 DEBG LiveNoOp  :1003 deps:[JobId(1002)] res:true
Sep 22 23:14:29.646 WARN 1005 job ELiveReopen for connection UpstairsConnection { upstairs_id: a50fc3fb-3de2-4743-9c50-cc80cfba77db, session_id: 2b698136-bbd2-4fcb-ab7c-5f19b2267eb8, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:14:29.646 DEBG ExtentLiveRepair: extent 0 sra:127.0.0.1:44564
Sep 22 23:14:29.646 INFO Created copy dir "/tmp/downstairs-zdyp8uYh/00/000/000.copy"
Sep 22 23:14:29.692 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:29.696 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:29.698 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:29.699 INFO [0] 4cf66ffa-a12c-4555-b256-f1d6116cf5e6 (61abb1c6-aab0-4918-867b-523054aca8f6) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:29.699 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:14:29.699 WARN [0] new RM replaced this: None
Sep 22 23:14:29.699 INFO [0] Starts reconcile loop
Sep 22 23:14:29.699 INFO [2] 4cf66ffa-a12c-4555-b256-f1d6116cf5e6 (61abb1c6-aab0-4918-867b-523054aca8f6) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:29.699 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:14:29.699 WARN [2] new RM replaced this: None
Sep 22 23:14:29.699 INFO [2] Starts reconcile loop
Sep 22 23:14:29.700 INFO [1] 4cf66ffa-a12c-4555-b256-f1d6116cf5e6 (61abb1c6-aab0-4918-867b-523054aca8f6) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum
Sep 22 23:14:29.700 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:14:29.700 WARN [1] new RM replaced this: None
Sep 22 23:14:29.700 INFO [1] Starts reconcile loop
Sep 22 23:14:29.700 INFO [0] 127.0.0.1:53298 task reports connection:true
Sep 22 23:14:29.700 INFO 4cf66ffa-a12c-4555-b256-f1d6116cf5e6 WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:14:29.700 INFO [0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:14:29.700 INFO [0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:14:29.700 INFO [0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]
Sep 22 23:14:29.700 INFO [1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:14:29.700 INFO [1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:14:29.700 INFO [1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]
Sep 22 23:14:29.700 INFO [2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:14:29.700 INFO [2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:14:29.700 INFO [2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]
Sep 22 23:14:29.700 INFO Max found gen is 1
Sep 22 23:14:29.700 INFO Generation requested: 1 >= found:1
Sep 22 23:14:29.700 INFO Next flush: 1
Sep 22 23:14:29.700 INFO All extents match
Sep 22 23:14:29.700 INFO No downstairs repair required
Sep 22 23:14:29.700 INFO No initial repair work was required
Sep 22 23:14:29.700 INFO Set Downstairs and Upstairs active
Sep 22 23:14:29.700 INFO 4cf66ffa-a12c-4555-b256-f1d6116cf5e6 is now active with session: 61abb1c6-aab0-4918-867b-523054aca8f6
Sep 22 23:14:29.700 INFO 4cf66ffa-a12c-4555-b256-f1d6116cf5e6 Set Active after no repair
Sep 22 23:14:29.700 INFO Notify all downstairs, region set compare is done.
Sep 22 23:14:29.700 INFO Set check for repair
Sep 22 23:14:29.700 INFO [2] 127.0.0.1:46586 task reports connection:true
Sep 22 23:14:29.700 INFO 4cf66ffa-a12c-4555-b256-f1d6116cf5e6 Active Active Active
Sep 22 23:14:29.700 INFO Set check for repair
Sep 22 23:14:29.700 INFO [1] 127.0.0.1:59093 task reports connection:true
Sep 22 23:14:29.701 INFO 4cf66ffa-a12c-4555-b256-f1d6116cf5e6 Active Active Active
Sep 22 23:14:29.701 INFO Set check for repair
Sep 22 23:14:29.701 INFO [0] received reconcile message
Sep 22 23:14:29.701 INFO [0] All repairs completed, exit
Sep 22 23:14:29.701 INFO [0] Starts cmd_loop
Sep 22 23:14:29.701 INFO [1] received reconcile message
Sep 22 23:14:29.701 INFO [1] All repairs completed, exit
Sep 22 23:14:29.701 INFO [1] Starts cmd_loop
Sep 22 23:14:29.701 INFO [2] received reconcile message
Sep 22 23:14:29.701 INFO [2] All repairs completed, exit
Sep 22 23:14:29.701 INFO [2] Starts cmd_loop
The guest has finished waiting for activation
Sep 22 23:14:29.709 INFO UUID: 1361b8e1-8898-44ce-892b-e72b28f2a9df
Sep 22 23:14:29.709 INFO Blocks per extent:512 Total Extents: 188
Sep 22 23:14:29.709 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:29.709 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:29.709 INFO Using address: 127.0.0.1:58182, task: main
Sep 22 23:14:29.710 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:29.710 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:45179, task: repair
Sep 22 23:14:29.710 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:45179, task: repair
Sep 22 23:14:29.710 INFO listening, local_addr: 127.0.0.1:45179, task: repair
Sep 22 23:14:29.710 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:45179, task: repair
Sep 22 23:14:29.710 INFO Using repair address: 127.0.0.1:45179, task: main
Sep 22 23:14:29.710 INFO No SSL acceptor configured, task: main
Sep 22 23:14:29.723 INFO UUID: 14399b3b-6e6e-4995-afe0-74c1441f0833
Sep 22 23:14:29.723 INFO Blocks per extent:512 Total Extents: 188
Sep 22 23:14:29.723 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:29.723 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:29.723 INFO Using address: 127.0.0.1:44435, task: main
Sep 22 23:14:29.723 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:29.723 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:36093, task: repair
Sep 22 23:14:29.723 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:36093, task: repair
Sep 22 23:14:29.724 INFO listening, local_addr: 127.0.0.1:36093, task: repair
Sep 22 23:14:29.724 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:36093, task: repair
Sep 22 23:14:29.724 INFO Using repair address: 127.0.0.1:36093, task: main
Sep 22 23:14:29.724 INFO No SSL acceptor configured, task: main
Sep 22 23:14:29.742 INFO accepted connection, remote_addr: 127.0.0.1:49176, local_addr: 127.0.0.1:44564, task: repair
Sep 22 23:14:29.743 TRCE incoming request, uri: /extent/0/files, method: GET, req_id: 88e31efa-976c-4991-8564-1f82328c288c, remote_addr: 127.0.0.1:49176, local_addr: 127.0.0.1:44564, task: repair
Sep 22 23:14:29.743 INFO request completed, latency_us: 396, response_code: 200, uri: /extent/0/files, method: GET, req_id: 88e31efa-976c-4991-8564-1f82328c288c, remote_addr: 127.0.0.1:49176, local_addr: 127.0.0.1:44564, task: repair
Sep 22 23:14:29.743 INFO eid:0 Found repair files: ["000", "000.db"]
Sep 22 23:14:29.744 TRCE incoming request, uri: /newextent/0/data, method: GET, req_id: 5f899b13-0308-44ae-b02b-57bb9a351324, remote_addr: 127.0.0.1:49176, local_addr: 127.0.0.1:44564, task: repair
Sep 22 23:14:29.744 INFO request completed, latency_us: 615, response_code: 200, uri: /newextent/0/data, method: GET, req_id: 5f899b13-0308-44ae-b02b-57bb9a351324, remote_addr: 127.0.0.1:49176, local_addr: 127.0.0.1:44564, task: repair
Sep 22 23:14:29.746 TRCE incoming request, uri: /newextent/0/db, method: GET, req_id: f8869a74-2fd7-4e08-bf49-7420fd09c6da, remote_addr: 127.0.0.1:49176, local_addr: 127.0.0.1:44564, task: repair
Sep 22 23:14:29.746 INFO request completed, latency_us: 370, response_code: 200, uri: /newextent/0/db, method: GET, req_id: f8869a74-2fd7-4e08-bf49-7420fd09c6da, remote_addr: 127.0.0.1:49176, local_addr: 127.0.0.1:44564, task: repair
Sep 22 23:14:29.747 INFO Repair files downloaded, move directory "/tmp/downstairs-zdyp8uYh/00/000/000.copy" to "/tmp/downstairs-zdyp8uYh/00/000/000.replace"
Sep 22 23:14:29.747 DEBG fsync completed for: "/tmp/downstairs-zdyp8uYh/00/000"
Sep 22 23:14:29.748 INFO Copy files from "/tmp/downstairs-zdyp8uYh/00/000/000.replace" in "/tmp/downstairs-zdyp8uYh/00/000"
Sep 22 23:14:29.748 DEBG fsync completed for: "/tmp/downstairs-zdyp8uYh/00/000/000"
Sep 22 23:14:29.748 DEBG fsync completed for: "/tmp/downstairs-zdyp8uYh/00/000/000.db"
Sep 22 23:14:29.748 DEBG fsync completed for: "/tmp/downstairs-zdyp8uYh/00/000"
Sep 22 23:14:29.749 INFO Move directory  "/tmp/downstairs-zdyp8uYh/00/000/000.replace" to "/tmp/downstairs-zdyp8uYh/00/000/000.completed"
Sep 22 23:14:29.749 DEBG fsync completed for: "/tmp/downstairs-zdyp8uYh/00/000"
Sep 22 23:14:29.749 DEBG fsync completed for: "/tmp/downstairs-zdyp8uYh/00/000"
Sep 22 23:14:29.749 DEBG LiveRepair:1003 extent 0 deps:[JobId(1002)] res:true
Sep 22 23:14:29.749 WARN 1005 job ELiveReopen for connection UpstairsConnection { upstairs_id: a50fc3fb-3de2-4743-9c50-cc80cfba77db, session_id: 2b698136-bbd2-4fcb-ab7c-5f19b2267eb8, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:14:29.749 DEBG [0] ExtentLiveRepair AckReady 1003, : downstairs
Sep 22 23:14:29.749 DEBG up_ds_listen was notified
Sep 22 23:14:29.749 DEBG up_ds_listen process 1003
Sep 22 23:14:29.749 DEBG [A] ack job 1003:4, : downstairs
Sep 22 23:14:29.749 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:29.749 DEBG Extent 0 id:1003 Done
Sep 22 23:14:29.750 DEBG Enqueue repair job 1004, : downstairs
Sep 22 23:14:29.750 INFO RE:0 Wait for result from NoOp command 1004:5
Sep 22 23:14:29.750 DEBG [0] 1004 Remove check skipped:{JobId(1001)} from deps:[JobId(1002), JobId(1003)], : downstairs
Sep 22 23:14:29.750 DEBG [0] 1004 Remove check < min repaired:1002 from deps:[JobId(1002), JobId(1003)], : downstairs
Sep 22 23:14:29.750 INFO [0] 1004 final dependency list [JobId(1002), JobId(1003)], : downstairs
Sep 22 23:14:29.750 DEBG Received NoOP 1004
Sep 22 23:14:29.750 DEBG Received NoOP 1004
Sep 22 23:14:29.750 DEBG Received NoOP 1004
Sep 22 23:14:29.750 DEBG Work of: LiveNoOp 1004
Sep 22 23:14:29.750 DEBG LiveNoOp  :1004 deps:[JobId(1002), JobId(1003)] res:true
Sep 22 23:14:29.753 DEBG LiveReopen:1005 extent 0 deps:[JobId(1002), JobId(1003), JobId(1004)] res:true
Sep 22 23:14:29.753 DEBG Work of: LiveNoOp 1004
Sep 22 23:14:29.753 DEBG LiveNoOp  :1004 deps:[JobId(1002), JobId(1003)] res:true
Sep 22 23:14:29.754 DEBG LiveReopen:1005 extent 0 deps:[JobId(1002), JobId(1003), JobId(1004)] res:true
Sep 22 23:14:29.755 DEBG Work of: LiveNoOp 1004
Sep 22 23:14:29.755 DEBG LiveNoOp  :1004 deps:[JobId(1002), JobId(1003)] res:true
Sep 22 23:14:29.756 DEBG LiveReopen:1005 extent 0 deps:[JobId(1002), JobId(1003), JobId(1004)] res:true
Sep 22 23:14:29.756 DEBG [0] ExtentLiveNoOp AckReady 1004, : downstairs
Sep 22 23:14:29.756 DEBG [0] ExtentLiveReopen AckReady 1005, : downstairs
Sep 22 23:14:29.756 DEBG up_ds_listen was notified
Sep 22 23:14:29.756 DEBG up_ds_listen process 1004
Sep 22 23:14:29.756 DEBG [A] ack job 1004:5, : downstairs
Sep 22 23:14:29.756 DEBG up_ds_listen process 1005
Sep 22 23:14:29.756 DEBG [A] ack job 1005:6, : downstairs
Sep 22 23:14:29.756 DEBG up_ds_listen checked 2 jobs, back to waiting
Sep 22 23:14:29.756 DEBG up_ds_listen was notified
Sep 22 23:14:29.756 DEBG up_ds_listen checked 0 jobs, back to waiting
Sep 22 23:14:29.756 DEBG Extent 0 id:1004 Done
Sep 22 23:14:29.756 INFO RE:0 Wait for result from reopen command 1005:6
Sep 22 23:14:29.756 DEBG Extent 0 id:1005 Done
Sep 22 23:14:29.756 INFO Start extent 1 repair, task: repair
Sep 22 23:14:29.756 DEBG RE:1 Repair extent begins
Sep 22 23:14:29.756 DEBG Create new job ids for 1, : downstairs
Sep 22 23:14:29.756 INFO RE:1 repair extent with ids 1006,1007,1008,1009 deps:[]
Sep 22 23:14:29.756 DEBG Enqueue repair job 1009, : downstairs
Sep 22 23:14:29.756 DEBG Enqueue repair job 1006, : downstairs
Sep 22 23:14:29.756 INFO RE:1 close id:1006 queued, notify DS
Sep 22 23:14:29.756 INFO RE:1 Wait for result from close command 1006:7
Sep 22 23:14:29.756 DEBG [0] 1006 Remove check skipped:{JobId(1001)} from deps:[], : downstairs
Sep 22 23:14:29.757 DEBG [0] 1006 Remove check < min repaired:1002 from deps:[], : downstairs
Sep 22 23:14:29.757 INFO [0] 1006 final dependency list [], : downstairs
Sep 22 23:14:29.757 DEBG [0] 1009 Remove check skipped:{JobId(1001)} from deps:[JobId(1006), JobId(1007), JobId(1008)], : downstairs
Sep 22 23:14:29.757 DEBG [0] 1009 Remove check < min repaired:1002 from deps:[JobId(1006), JobId(1007), JobId(1008)], : downstairs
Sep 22 23:14:29.757 INFO [0] 1009 final dependency list [JobId(1006), JobId(1007), JobId(1008)], : downstairs
Sep 22 23:14:29.757 DEBG Flush just extent 1 with f:3 and g:1
Sep 22 23:14:29.758 DEBG FlushClose:1006 extent 1 deps:[] res:true f:3 g:1
Sep 22 23:14:29.758 DEBG Flush just extent 1 with f:3 and g:1
Sep 22 23:14:29.759 DEBG FlushClose:1006 extent 1 deps:[] res:true f:3 g:1
Sep 22 23:14:29.760 DEBG JustClose :1006 extent 1 deps:[] res:true
Sep 22 23:14:29.761 DEBG [1] ELC got g:1 f:1 d:false
Sep 22 23:14:29.761 DEBG [2] ELC got g:1 f:1 d:false
Sep 22 23:14:29.761 DEBG [0] ELC got g:0 f:0 d:false
Sep 22 23:14:29.761 DEBG [0] ExtentFlushClose 1006 AckReady, : downstairs
Sep 22 23:14:29.761 DEBG up_ds_listen was notified
Sep 22 23:14:29.761 DEBG up_ds_listen process 1006
Sep 22 23:14:29.761 DEBG [A] ack job 1006:7, : downstairs
Sep 22 23:14:29.761 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:29.761 DEBG Extent 1 id:1006 Done
Sep 22 23:14:29.761 DEBG Get repair info for 2 source, : downstairs
Sep 22 23:14:29.761 DEBG Get repair info for 0 bad, : downstairs
Sep 22 23:14:29.761 INFO Repair for extent 1 s:2 d:[ClientId(0)], : downstairs
Sep 22 23:14:29.761 DEBG Enqueue repair job 1007, : downstairs
Sep 22 23:14:29.761 INFO RE:1 Wait for result from repair command 1007:8
Sep 22 23:14:29.761 DEBG [0] 1007 Remove check skipped:{JobId(1001)} from deps:[JobId(1006)], : downstairs
Sep 22 23:14:29.761 DEBG [0] 1007 Remove check < min repaired:1002 from deps:[JobId(1006)], : downstairs
Sep 22 23:14:29.761 INFO [0] 1007 final dependency list [JobId(1006)], : downstairs
Sep 22 23:14:29.762 DEBG Received NoOP 1007
Sep 22 23:14:29.762 DEBG Received NoOP 1007
Sep 22 23:14:29.762 DEBG Received ExtentLiveRepair 1007
Sep 22 23:14:29.762 DEBG Work of: LiveNoOp 1007
Sep 22 23:14:29.762 DEBG LiveNoOp  :1007 deps:[JobId(1006)] res:true
Sep 22 23:14:29.762 WARN 1009 job ELiveReopen for connection UpstairsConnection { upstairs_id: a50fc3fb-3de2-4743-9c50-cc80cfba77db, session_id: 2b698136-bbd2-4fcb-ab7c-5f19b2267eb8, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:14:29.762 DEBG Work of: LiveNoOp 1007
Sep 22 23:14:29.762 DEBG LiveNoOp  :1007 deps:[JobId(1006)] res:true
Sep 22 23:14:29.762 WARN 1009 job ELiveReopen for connection UpstairsConnection { upstairs_id: a50fc3fb-3de2-4743-9c50-cc80cfba77db, session_id: 2b698136-bbd2-4fcb-ab7c-5f19b2267eb8, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:14:29.762 DEBG ExtentLiveRepair: extent 1 sra:127.0.0.1:44564
Sep 22 23:14:29.762 INFO Created copy dir "/tmp/downstairs-zdyp8uYh/00/000/001.copy"
Sep 22 23:14:29.783 INFO Upstairs starts
Sep 22 23:14:29.783 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:14:29.783 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:14:29.783 INFO Crucible stats registered with UUID: 6e5c0f70-fd56-4280-9d20-71288e488216
Sep 22 23:14:29.783 INFO Crucible 6e5c0f70-fd56-4280-9d20-71288e488216 has session id: 5375fd1f-f6a1-4405-99f6-d882ff308715
Sep 22 23:14:29.784 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:29.784 INFO [0] connecting to 127.0.0.1:42762, looper: 0
Sep 22 23:14:29.784 INFO [1] connecting to 127.0.0.1:64149, looper: 1
Sep 22 23:14:29.784 INFO [2] connecting to 127.0.0.1:58182, looper: 2
Sep 22 23:14:29.784 INFO up_listen starts, task: up_listen
Sep 22 23:14:29.784 INFO Wait for all three downstairs to come online
Sep 22 23:14:29.784 INFO Flush timeout: 0.5
Sep 22 23:14:29.784 INFO accepted connection from 127.0.0.1:49772, task: main
Sep 22 23:14:29.784 INFO accepted connection from 127.0.0.1:48323, task: main
Sep 22 23:14:29.784 INFO [0] 6e5c0f70-fd56-4280-9d20-71288e488216 looper connected, looper: 0
Sep 22 23:14:29.784 INFO [0] Proc runs for 127.0.0.1:42762 in state New
Sep 22 23:14:29.785 INFO [1] 6e5c0f70-fd56-4280-9d20-71288e488216 looper connected, looper: 1
Sep 22 23:14:29.785 INFO [1] Proc runs for 127.0.0.1:64149 in state New
Sep 22 23:14:29.785 INFO [2] 6e5c0f70-fd56-4280-9d20-71288e488216 looper connected, looper: 2
Sep 22 23:14:29.785 INFO [2] Proc runs for 127.0.0.1:58182 in state New
Sep 22 23:14:29.785 INFO accepted connection from 127.0.0.1:59274, task: main
Sep 22 23:14:29.785 INFO Connection request from 6e5c0f70-fd56-4280-9d20-71288e488216 with version 4, task: proc
Sep 22 23:14:29.785 INFO upstairs UpstairsConnection { upstairs_id: 6e5c0f70-fd56-4280-9d20-71288e488216, session_id: f334c95e-b851-4a8a-a731-3fb69e42e934, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:29.785 INFO Connection request from 6e5c0f70-fd56-4280-9d20-71288e488216 with version 4, task: proc
Sep 22 23:14:29.785 INFO upstairs UpstairsConnection { upstairs_id: 6e5c0f70-fd56-4280-9d20-71288e488216, session_id: f334c95e-b851-4a8a-a731-3fb69e42e934, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:29.785 INFO Connection request from 6e5c0f70-fd56-4280-9d20-71288e488216 with version 4, task: proc
Sep 22 23:14:29.785 INFO upstairs UpstairsConnection { upstairs_id: 6e5c0f70-fd56-4280-9d20-71288e488216, session_id: f334c95e-b851-4a8a-a731-3fb69e42e934, gen: 1 } connected, version 4, task: proc
The guest has requested activation
Sep 22 23:14:29.785 INFO [0] 6e5c0f70-fd56-4280-9d20-71288e488216 (f334c95e-b851-4a8a-a731-3fb69e42e934) New New New ds_transition to WaitActive
Sep 22 23:14:29.785 INFO [0] Transition from New to WaitActive
Sep 22 23:14:29.786 INFO [1] 6e5c0f70-fd56-4280-9d20-71288e488216 (f334c95e-b851-4a8a-a731-3fb69e42e934) WaitActive New New ds_transition to WaitActive
Sep 22 23:14:29.786 INFO [1] Transition from New to WaitActive
Sep 22 23:14:29.786 INFO [2] 6e5c0f70-fd56-4280-9d20-71288e488216 (f334c95e-b851-4a8a-a731-3fb69e42e934) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:14:29.786 INFO [2] Transition from New to WaitActive
Sep 22 23:14:29.786 INFO 6e5c0f70-fd56-4280-9d20-71288e488216 active request set
Sep 22 23:14:29.786 INFO [0] received activate with gen 1
Sep 22 23:14:29.786 INFO [0] client got ds_active_rx, promote! session f334c95e-b851-4a8a-a731-3fb69e42e934
Sep 22 23:14:29.786 INFO [1] received activate with gen 1
Sep 22 23:14:29.786 INFO [1] client got ds_active_rx, promote! session f334c95e-b851-4a8a-a731-3fb69e42e934
Sep 22 23:14:29.786 INFO [2] received activate with gen 1
Sep 22 23:14:29.786 INFO [2] client got ds_active_rx, promote! session f334c95e-b851-4a8a-a731-3fb69e42e934
Sep 22 23:14:29.786 INFO UpstairsConnection { upstairs_id: 6e5c0f70-fd56-4280-9d20-71288e488216, session_id: f334c95e-b851-4a8a-a731-3fb69e42e934, gen: 1 } is now active (read-write)
Sep 22 23:14:29.787 INFO UpstairsConnection { upstairs_id: 6e5c0f70-fd56-4280-9d20-71288e488216, session_id: f334c95e-b851-4a8a-a731-3fb69e42e934, gen: 1 } is now active (read-write)
Sep 22 23:14:29.787 INFO UpstairsConnection { upstairs_id: 6e5c0f70-fd56-4280-9d20-71288e488216, session_id: f334c95e-b851-4a8a-a731-3fb69e42e934, gen: 1 } is now active (read-write)
Sep 22 23:14:29.787 INFO [0] downstairs client at 127.0.0.1:42762 has UUID 353b066e-6f4d-4802-9336-0f9c4c2ea130
Sep 22 23:14:29.787 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 512, shift: 9 }, extent_count: 188, uuid: 353b066e-6f4d-4802-9336-0f9c4c2ea130, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:29.787 INFO 6e5c0f70-fd56-4280-9d20-71288e488216 WaitActive WaitActive WaitActive
Sep 22 23:14:29.787 INFO [1] downstairs client at 127.0.0.1:64149 has UUID df366b3b-ce8f-4a81-b993-9efbc4225b81
Sep 22 23:14:29.787 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 512, shift: 9 }, extent_count: 188, uuid: df366b3b-ce8f-4a81-b993-9efbc4225b81, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:29.787 INFO 6e5c0f70-fd56-4280-9d20-71288e488216 WaitActive WaitActive WaitActive
Sep 22 23:14:29.787 INFO [2] downstairs client at 127.0.0.1:58182 has UUID 1361b8e1-8898-44ce-892b-e72b28f2a9df
Sep 22 23:14:29.787 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 512, shift: 9 }, extent_count: 188, uuid: 1361b8e1-8898-44ce-892b-e72b28f2a9df, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:29.787 INFO 6e5c0f70-fd56-4280-9d20-71288e488216 WaitActive WaitActive WaitActive
Sep 22 23:14:29.798 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:29.798 INFO Created new region file "/tmp/downstairs-fjJjawz1/region.json"
Sep 22 23:14:29.835 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:29.835 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:29.835 INFO Opened existing region file "/tmp/downstairs-ybEDH4hM/region.json"
Sep 22 23:14:29.835 INFO Database read version 1
Sep 22 23:14:29.835 INFO Database write version 1
Sep 22 23:14:29.842 INFO accepted connection, remote_addr: 127.0.0.1:65045, local_addr: 127.0.0.1:44564, task: repair
Sep 22 23:14:29.843 TRCE incoming request, uri: /extent/1/files, method: GET, req_id: 20b49939-e2a8-4c0d-99f2-91d72757c7d7, remote_addr: 127.0.0.1:65045, local_addr: 127.0.0.1:44564, task: repair
Sep 22 23:14:29.843 INFO request completed, latency_us: 233, response_code: 200, uri: /extent/1/files, method: GET, req_id: 20b49939-e2a8-4c0d-99f2-91d72757c7d7, remote_addr: 127.0.0.1:65045, local_addr: 127.0.0.1:44564, task: repair
Sep 22 23:14:29.843 INFO eid:1 Found repair files: ["001", "001.db"]
Sep 22 23:14:29.844 TRCE incoming request, uri: /newextent/1/data, method: GET, req_id: 9d9eb051-b271-4896-81c3-281dff2a10de, remote_addr: 127.0.0.1:65045, local_addr: 127.0.0.1:44564, task: repair
Sep 22 23:14:29.844 INFO request completed, latency_us: 380, response_code: 200, uri: /newextent/1/data, method: GET, req_id: 9d9eb051-b271-4896-81c3-281dff2a10de, remote_addr: 127.0.0.1:65045, local_addr: 127.0.0.1:44564, task: repair
Sep 22 23:14:29.845 TRCE incoming request, uri: /newextent/1/db, method: GET, req_id: 3953507f-608f-44c8-a979-2bf7485083a7, remote_addr: 127.0.0.1:65045, local_addr: 127.0.0.1:44564, task: repair
Sep 22 23:14:29.845 INFO request completed, latency_us: 335, response_code: 200, uri: /newextent/1/db, method: GET, req_id: 3953507f-608f-44c8-a979-2bf7485083a7, remote_addr: 127.0.0.1:65045, local_addr: 127.0.0.1:44564, task: repair
Sep 22 23:14:29.846 INFO Repair files downloaded, move directory "/tmp/downstairs-zdyp8uYh/00/000/001.copy" to "/tmp/downstairs-zdyp8uYh/00/000/001.replace"
Sep 22 23:14:29.846 DEBG fsync completed for: "/tmp/downstairs-zdyp8uYh/00/000"
Sep 22 23:14:29.847 INFO Copy files from "/tmp/downstairs-zdyp8uYh/00/000/001.replace" in "/tmp/downstairs-zdyp8uYh/00/000"
Sep 22 23:14:29.847 DEBG fsync completed for: "/tmp/downstairs-zdyp8uYh/00/000/001"
Sep 22 23:14:29.847 DEBG fsync completed for: "/tmp/downstairs-zdyp8uYh/00/000/001.db"
Sep 22 23:14:29.847 DEBG fsync completed for: "/tmp/downstairs-zdyp8uYh/00/000"
Sep 22 23:14:29.847 INFO Move directory  "/tmp/downstairs-zdyp8uYh/00/000/001.replace" to "/tmp/downstairs-zdyp8uYh/00/000/001.completed"
Sep 22 23:14:29.847 DEBG fsync completed for: "/tmp/downstairs-zdyp8uYh/00/000"
Sep 22 23:14:29.847 DEBG fsync completed for: "/tmp/downstairs-zdyp8uYh/00/000"
Sep 22 23:14:29.847 DEBG LiveRepair:1007 extent 1 deps:[JobId(1006)] res:true
Sep 22 23:14:29.847 WARN 1009 job ELiveReopen for connection UpstairsConnection { upstairs_id: a50fc3fb-3de2-4743-9c50-cc80cfba77db, session_id: 2b698136-bbd2-4fcb-ab7c-5f19b2267eb8, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:14:29.848 DEBG [0] ExtentLiveRepair AckReady 1007, : downstairs
Sep 22 23:14:29.848 DEBG up_ds_listen was notified
Sep 22 23:14:29.848 DEBG up_ds_listen process 1007
Sep 22 23:14:29.848 DEBG [A] ack job 1007:8, : downstairs
Sep 22 23:14:29.848 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:29.848 DEBG Extent 1 id:1007 Done
Sep 22 23:14:29.848 DEBG Enqueue repair job 1008, : downstairs
Sep 22 23:14:29.848 INFO RE:1 Wait for result from NoOp command 1008:9
Sep 22 23:14:29.848 DEBG [0] 1008 Remove check skipped:{JobId(1001)} from deps:[JobId(1006), JobId(1007)], : downstairs
Sep 22 23:14:29.848 DEBG [0] 1008 Remove check < min repaired:1002 from deps:[JobId(1006), JobId(1007)], : downstairs
Sep 22 23:14:29.848 INFO [0] 1008 final dependency list [JobId(1006), JobId(1007)], : downstairs
Sep 22 23:14:29.848 DEBG Received NoOP 1008
Sep 22 23:14:29.848 DEBG Received NoOP 1008
Sep 22 23:14:29.848 DEBG Received NoOP 1008
Sep 22 23:14:29.849 DEBG Work of: LiveNoOp 1008
Sep 22 23:14:29.849 DEBG LiveNoOp  :1008 deps:[JobId(1006), JobId(1007)] res:true
Sep 22 23:14:29.850 DEBG LiveReopen:1009 extent 1 deps:[JobId(1006), JobId(1007), JobId(1008)] res:true
Sep 22 23:14:29.850 DEBG Work of: LiveNoOp 1008
Sep 22 23:14:29.850 DEBG LiveNoOp  :1008 deps:[JobId(1006), JobId(1007)] res:true
Sep 22 23:14:29.852 DEBG LiveReopen:1009 extent 1 deps:[JobId(1006), JobId(1007), JobId(1008)] res:true
Sep 22 23:14:29.852 DEBG Work of: LiveNoOp 1008
Sep 22 23:14:29.852 DEBG LiveNoOp  :1008 deps:[JobId(1006), JobId(1007)] res:true
Sep 22 23:14:29.853 DEBG LiveReopen:1009 extent 1 deps:[JobId(1006), JobId(1007), JobId(1008)] res:true
Sep 22 23:14:29.854 DEBG [0] ExtentLiveNoOp AckReady 1008, : downstairs
Sep 22 23:14:29.854 DEBG [0] ExtentLiveReopen AckReady 1009, : downstairs
Sep 22 23:14:29.854 DEBG up_ds_listen was notified
Sep 22 23:14:29.854 DEBG up_ds_listen process 1008
Sep 22 23:14:29.854 DEBG [A] ack job 1008:9, : downstairs
Sep 22 23:14:29.854 DEBG up_ds_listen process 1009
Sep 22 23:14:29.854 DEBG [A] ack job 1009:10, : downstairs
Sep 22 23:14:29.854 DEBG up_ds_listen checked 2 jobs, back to waiting
Sep 22 23:14:29.854 DEBG up_ds_listen was notified
Sep 22 23:14:29.854 DEBG up_ds_listen checked 0 jobs, back to waiting
Sep 22 23:14:29.854 DEBG Extent 1 id:1008 Done
Sep 22 23:14:29.854 INFO RE:1 Wait for result from reopen command 1009:10
Sep 22 23:14:29.854 DEBG Extent 1 id:1009 Done
Sep 22 23:14:29.854 DEBG IO Flush 1010 has deps [JobId(1009), JobId(1008), JobId(1007), JobId(1006), JobId(1005), JobId(1004), JobId(1003), JobId(1002)]
Sep 22 23:14:29.854 INFO LiveRepair final flush submitted
Sep 22 23:14:29.854 DEBG [0] 1010 Remove check skipped:{JobId(1001)} from deps:[JobId(1009), JobId(1008), JobId(1007), JobId(1006), JobId(1005), JobId(1004), JobId(1003), JobId(1002)], : downstairs
Sep 22 23:14:29.854 DEBG [0] 1010 Remove check < min repaired:1002 from deps:[JobId(1009), JobId(1008), JobId(1007), JobId(1006), JobId(1005), JobId(1004), JobId(1003), JobId(1002)], : downstairs
Sep 22 23:14:29.854 INFO [0] 1010 final dependency list [JobId(1009), JobId(1008), JobId(1007), JobId(1006), JobId(1005), JobId(1004), JobId(1003), JobId(1002)], : downstairs
Sep 22 23:14:29.854 DEBG Flush     :1010 extent_limit None deps:[JobId(1009), JobId(1008), JobId(1007), JobId(1006), JobId(1005), JobId(1004), JobId(1003), JobId(1002)] res:true f:4 g:1
Sep 22 23:14:29.855 DEBG Flush     :1010 extent_limit None deps:[JobId(1009), JobId(1008), JobId(1007), JobId(1006), JobId(1005), JobId(1004), JobId(1003), JobId(1002)] res:true f:4 g:1
Sep 22 23:14:29.855 DEBG Flush     :1010 extent_limit Some(1) deps:[JobId(1009), JobId(1008), JobId(1007), JobId(1006), JobId(1005), JobId(1004), JobId(1003), JobId(1002)] res:true f:4 g:1
Sep 22 23:14:29.855 DEBG up_ds_listen was notified
Sep 22 23:14:29.855 DEBG up_ds_listen process 1010
Sep 22 23:14:29.855 DEBG [A] ack job 1010:11, : downstairs
Sep 22 23:14:29.855 DEBG [rc] retire 1010 clears [JobId(1002), JobId(1003), JobId(1004), JobId(1005), JobId(1006), JobId(1007), JobId(1008), JobId(1009), JobId(1010)], : downstairs
Sep 22 23:14:29.855 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:29.855 INFO LiveRepair final flush completed
Sep 22 23:14:29.855 INFO [0] a50fc3fb-3de2-4743-9c50-cc80cfba77db (2b698136-bbd2-4fcb-ab7c-5f19b2267eb8) LiveRepair Active Active ds_transition to Active
Sep 22 23:14:29.855 INFO [0] Transition from LiveRepair to Active
Sep 22 23:14:29.855 WARN Live Repair returns Ok(())
Sep 22 23:14:29.873 INFO Current flush_numbers [0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:14:29.876 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:29.876 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:29.876 INFO Opened existing region file "/tmp/downstairs-GUu8sV6m/region.json"
Sep 22 23:14:29.876 INFO Database read version 1
Sep 22 23:14:29.876 INFO Database write version 1
Sep 22 23:14:29.878 INFO Current flush_numbers [0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:14:29.883 INFO Current flush_numbers [0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:14:29.889 WARN e94af85e-3796-4fab-91a1-f12add9c3020 request to replace downstairs 127.0.0.1:52905 with 127.0.0.1:56689
Sep 22 23:14:29.889 INFO e94af85e-3796-4fab-91a1-f12add9c3020 found new target: 127.0.0.1:56689 at 0
Waiting for replacement to finish
Sep 22 23:14:29.893 INFO Checking if live repair is needed
Sep 22 23:14:29.893 INFO [0] e94af85e-3796-4fab-91a1-f12add9c3020 (a86ae319-11ad-4b2d-a80b-330efc92ebef) LiveRepairReady Active Active ds_transition to LiveRepair
Sep 22 23:14:29.893 INFO [0] Transition from LiveRepairReady to LiveRepair
Sep 22 23:14:29.893 INFO Live Repair started
Sep 22 23:14:29.893 WARN Live Repair main task begins., task: repair
Sep 22 23:14:29.893 INFO Start Live Repair of extents 0 to 2, task: repair
Sep 22 23:14:29.893 INFO Start extent 0 repair, task: repair
Sep 22 23:14:29.893 DEBG RE:0 Repair extent begins
Sep 22 23:14:29.893 DEBG Create new job ids for 0, : downstairs
Sep 22 23:14:29.893 INFO RE:0 repair extent with ids 1002,1003,1004,1005 deps:[]
Sep 22 23:14:29.893 DEBG Enqueue repair job 1005, : downstairs
Sep 22 23:14:29.893 DEBG Enqueue repair job 1002, : downstairs
Sep 22 23:14:29.893 INFO RE:0 close id:1002 queued, notify DS
Sep 22 23:14:29.893 INFO RE:0 Wait for result from close command 1002:3
Sep 22 23:14:29.893 DEBG [0] 1002 Remove check skipped:{JobId(1001)} from deps:[], : downstairs
Sep 22 23:14:29.893 DEBG [0] 1002 Remove check < min repaired:1002 from deps:[], : downstairs
Sep 22 23:14:29.893 INFO [0] 1002 final dependency list [], : downstairs
Sep 22 23:14:29.894 DEBG [0] 1005 Remove check skipped:{JobId(1001)} from deps:[JobId(1002), JobId(1003), JobId(1004)], : downstairs
Sep 22 23:14:29.894 DEBG [0] 1005 Remove check < min repaired:1002 from deps:[JobId(1002), JobId(1003), JobId(1004)], : downstairs
Sep 22 23:14:29.894 INFO [0] 1005 final dependency list [JobId(1002), JobId(1003), JobId(1004)], : downstairs
Sep 22 23:14:29.894 DEBG Flush just extent 0 with f:2 and g:1
Sep 22 23:14:29.895 DEBG FlushClose:1002 extent 0 deps:[] res:true f:2 g:1
Sep 22 23:14:29.895 DEBG Flush just extent 0 with f:2 and g:1
Sep 22 23:14:29.896 DEBG FlushClose:1002 extent 0 deps:[] res:true f:2 g:1
Sep 22 23:14:29.897 DEBG JustClose :1002 extent 0 deps:[] res:true
Sep 22 23:14:29.897 DEBG [1] ELC got g:1 f:1 d:false
Sep 22 23:14:29.897 DEBG [2] ELC got g:1 f:1 d:false
Sep 22 23:14:29.897 DEBG [0] ELC got g:0 f:0 d:false
Sep 22 23:14:29.897 DEBG [0] ExtentFlushClose 1002 AckReady, : downstairs
Sep 22 23:14:29.897 DEBG up_ds_listen was notified
Sep 22 23:14:29.897 DEBG up_ds_listen process 1002
Sep 22 23:14:29.897 DEBG [A] ack job 1002:3, : downstairs
Sep 22 23:14:29.897 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:29.897 DEBG Extent 0 id:1002 Done
Sep 22 23:14:29.897 DEBG Get repair info for 2 source, : downstairs
Sep 22 23:14:29.897 DEBG Get repair info for 0 bad, : downstairs
Sep 22 23:14:29.898 INFO Repair for extent 0 s:2 d:[ClientId(0)], : downstairs
Sep 22 23:14:29.898 DEBG Enqueue repair job 1003, : downstairs
Sep 22 23:14:29.898 INFO RE:0 Wait for result from repair command 1003:4
Sep 22 23:14:29.898 DEBG [0] 1003 Remove check skipped:{JobId(1001)} from deps:[JobId(1002)], : downstairs
Sep 22 23:14:29.898 DEBG [0] 1003 Remove check < min repaired:1002 from deps:[JobId(1002)], : downstairs
Sep 22 23:14:29.898 INFO [0] 1003 final dependency list [JobId(1002)], : downstairs
Sep 22 23:14:29.898 DEBG Received NoOP 1003
Sep 22 23:14:29.898 DEBG Received NoOP 1003
Sep 22 23:14:29.898 DEBG Received ExtentLiveRepair 1003
Sep 22 23:14:29.898 DEBG Work of: LiveNoOp 1003
Sep 22 23:14:29.898 DEBG LiveNoOp  :1003 deps:[JobId(1002)] res:true
Sep 22 23:14:29.898 WARN 1005 job ELiveReopen for connection UpstairsConnection { upstairs_id: e94af85e-3796-4fab-91a1-f12add9c3020, session_id: a86ae319-11ad-4b2d-a80b-330efc92ebef, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:14:29.898 DEBG Work of: LiveNoOp 1003
Sep 22 23:14:29.898 DEBG LiveNoOp  :1003 deps:[JobId(1002)] res:true
Sep 22 23:14:29.898 WARN 1005 job ELiveReopen for connection UpstairsConnection { upstairs_id: e94af85e-3796-4fab-91a1-f12add9c3020, session_id: a86ae319-11ad-4b2d-a80b-330efc92ebef, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:14:29.898 DEBG ExtentLiveRepair: extent 0 sra:127.0.0.1:54607
Sep 22 23:14:29.898 INFO Created copy dir "/tmp/downstairs-08law0EE/00/000/000.copy"
Sep 22 23:14:29.928 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:29.930 INFO UUID: 2b517ebe-5910-495c-8a3d-f94697b298ea
Sep 22 23:14:29.930 INFO Blocks per extent:512 Total Extents: 188
Sep 22 23:14:29.930 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:29.930 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:29.930 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:29.930 INFO Using address: 127.0.0.1:50214, task: main
Sep 22 23:14:29.931 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:29.931 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:55632, task: repair
Sep 22 23:14:29.931 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:55632, task: repair
Sep 22 23:14:29.931 INFO listening, local_addr: 127.0.0.1:55632, task: repair
Sep 22 23:14:29.931 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:55632, task: repair
Sep 22 23:14:29.932 INFO Using repair address: 127.0.0.1:55632, task: main
Sep 22 23:14:29.932 INFO No SSL acceptor configured, task: main
Sep 22 23:14:29.932 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:29.933 INFO [0] 6e5c0f70-fd56-4280-9d20-71288e488216 (f334c95e-b851-4a8a-a731-3fb69e42e934) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:29.933 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:14:29.933 WARN [0] new RM replaced this: None
Sep 22 23:14:29.933 INFO [0] Starts reconcile loop
Sep 22 23:14:29.933 INFO [1] 6e5c0f70-fd56-4280-9d20-71288e488216 (f334c95e-b851-4a8a-a731-3fb69e42e934) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:29.933 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:14:29.933 WARN [1] new RM replaced this: None
Sep 22 23:14:29.933 INFO [1] Starts reconcile loop
Sep 22 23:14:29.933 INFO [2] 6e5c0f70-fd56-4280-9d20-71288e488216 (f334c95e-b851-4a8a-a731-3fb69e42e934) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:14:29.933 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:14:29.933 WARN [2] new RM replaced this: None
Sep 22 23:14:29.934 INFO [2] Starts reconcile loop
Sep 22 23:14:29.934 INFO [0] 127.0.0.1:42762 task reports connection:true
Sep 22 23:14:29.934 INFO 6e5c0f70-fd56-4280-9d20-71288e488216 WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:14:29.934 INFO [0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:14:29.934 INFO [0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:14:29.934 INFO [0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]
Sep 22 23:14:29.934 INFO [1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:14:29.934 INFO [1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:14:29.934 INFO [1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]
Sep 22 23:14:29.934 INFO [2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:14:29.934 INFO [2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:14:29.934 INFO [2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]
Sep 22 23:14:29.934 INFO Max found gen is 1
Sep 22 23:14:29.934 INFO Generation requested: 1 >= found:1
Sep 22 23:14:29.934 INFO Next flush: 1
Sep 22 23:14:29.934 INFO All extents match
Sep 22 23:14:29.934 INFO No downstairs repair required
Sep 22 23:14:29.934 INFO No initial repair work was required
Sep 22 23:14:29.934 INFO Set Downstairs and Upstairs active
Sep 22 23:14:29.934 INFO 6e5c0f70-fd56-4280-9d20-71288e488216 is now active with session: f334c95e-b851-4a8a-a731-3fb69e42e934
Sep 22 23:14:29.934 INFO 6e5c0f70-fd56-4280-9d20-71288e488216 Set Active after no repair
Sep 22 23:14:29.934 INFO Notify all downstairs, region set compare is done.
Sep 22 23:14:29.934 INFO Set check for repair
Sep 22 23:14:29.934 INFO [1] 127.0.0.1:64149 task reports connection:true
Sep 22 23:14:29.934 INFO 6e5c0f70-fd56-4280-9d20-71288e488216 Active Active Active
Sep 22 23:14:29.934 INFO Set check for repair
Sep 22 23:14:29.934 INFO [2] 127.0.0.1:58182 task reports connection:true
Sep 22 23:14:29.934 INFO 6e5c0f70-fd56-4280-9d20-71288e488216 Active Active Active
Sep 22 23:14:29.934 INFO Set check for repair
Sep 22 23:14:29.934 INFO [0] received reconcile message
Sep 22 23:14:29.934 INFO [0] All repairs completed, exit
Sep 22 23:14:29.935 INFO [0] Starts cmd_loop
Sep 22 23:14:29.935 INFO [1] received reconcile message
Sep 22 23:14:29.935 INFO [1] All repairs completed, exit
Sep 22 23:14:29.935 INFO [1] Starts cmd_loop
Sep 22 23:14:29.935 INFO [2] received reconcile message
Sep 22 23:14:29.935 INFO [2] All repairs completed, exit
Sep 22 23:14:29.935 INFO [2] Starts cmd_loop
The guest has finished waiting for activation
Sep 22 23:14:29.972 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:29.973 INFO Created new region file "/tmp/downstairs-1HhZJsN0/region.json"
Sep 22 23:14:29.987 INFO accepted connection, remote_addr: 127.0.0.1:40413, local_addr: 127.0.0.1:54607, task: repair
Sep 22 23:14:29.987 TRCE incoming request, uri: /extent/0/files, method: GET, req_id: ec26dd66-293b-4bdf-9cf0-321bcfe2059a, remote_addr: 127.0.0.1:40413, local_addr: 127.0.0.1:54607, task: repair
Sep 22 23:14:29.987 INFO request completed, latency_us: 243, response_code: 200, uri: /extent/0/files, method: GET, req_id: ec26dd66-293b-4bdf-9cf0-321bcfe2059a, remote_addr: 127.0.0.1:40413, local_addr: 127.0.0.1:54607, task: repair
Sep 22 23:14:29.988 INFO eid:0 Found repair files: ["000", "000.db"]
Sep 22 23:14:29.988 TRCE incoming request, uri: /newextent/0/data, method: GET, req_id: ba561005-233a-402f-8ab5-b4db21257bd6, remote_addr: 127.0.0.1:40413, local_addr: 127.0.0.1:54607, task: repair
Sep 22 23:14:29.988 INFO request completed, latency_us: 443, response_code: 200, uri: /newextent/0/data, method: GET, req_id: ba561005-233a-402f-8ab5-b4db21257bd6, remote_addr: 127.0.0.1:40413, local_addr: 127.0.0.1:54607, task: repair
Sep 22 23:14:29.989 TRCE incoming request, uri: /newextent/0/db, method: GET, req_id: c37d79fe-c802-4dc0-ae49-a16967640313, remote_addr: 127.0.0.1:40413, local_addr: 127.0.0.1:54607, task: repair
Sep 22 23:14:29.990 INFO request completed, latency_us: 367, response_code: 200, uri: /newextent/0/db, method: GET, req_id: c37d79fe-c802-4dc0-ae49-a16967640313, remote_addr: 127.0.0.1:40413, local_addr: 127.0.0.1:54607, task: repair
Sep 22 23:14:29.991 INFO Repair files downloaded, move directory "/tmp/downstairs-08law0EE/00/000/000.copy" to "/tmp/downstairs-08law0EE/00/000/000.replace"
Sep 22 23:14:29.991 DEBG fsync completed for: "/tmp/downstairs-08law0EE/00/000"
Sep 22 23:14:29.992 INFO Copy files from "/tmp/downstairs-08law0EE/00/000/000.replace" in "/tmp/downstairs-08law0EE/00/000"
Sep 22 23:14:29.992 DEBG fsync completed for: "/tmp/downstairs-08law0EE/00/000/000"
Sep 22 23:14:29.992 DEBG fsync completed for: "/tmp/downstairs-08law0EE/00/000/000.db"
Sep 22 23:14:29.992 DEBG fsync completed for: "/tmp/downstairs-08law0EE/00/000"
Sep 22 23:14:29.992 INFO Move directory  "/tmp/downstairs-08law0EE/00/000/000.replace" to "/tmp/downstairs-08law0EE/00/000/000.completed"
Sep 22 23:14:29.992 DEBG fsync completed for: "/tmp/downstairs-08law0EE/00/000"
Sep 22 23:14:29.992 DEBG fsync completed for: "/tmp/downstairs-08law0EE/00/000"
Sep 22 23:14:29.992 DEBG LiveRepair:1003 extent 0 deps:[JobId(1002)] res:true
Sep 22 23:14:29.992 WARN 1005 job ELiveReopen for connection UpstairsConnection { upstairs_id: e94af85e-3796-4fab-91a1-f12add9c3020, session_id: a86ae319-11ad-4b2d-a80b-330efc92ebef, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:14:29.993 DEBG [0] ExtentLiveRepair AckReady 1003, : downstairs
Sep 22 23:14:29.993 DEBG up_ds_listen was notified
Sep 22 23:14:29.993 DEBG up_ds_listen process 1003
Sep 22 23:14:29.993 DEBG [A] ack job 1003:4, : downstairs
Sep 22 23:14:29.993 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:29.993 DEBG Extent 0 id:1003 Done
Sep 22 23:14:29.993 DEBG Enqueue repair job 1004, : downstairs
Sep 22 23:14:29.993 INFO RE:0 Wait for result from NoOp command 1004:5
Sep 22 23:14:29.993 DEBG [0] 1004 Remove check skipped:{JobId(1001)} from deps:[JobId(1002), JobId(1003)], : downstairs
Sep 22 23:14:29.993 DEBG [0] 1004 Remove check < min repaired:1002 from deps:[JobId(1002), JobId(1003)], : downstairs
Sep 22 23:14:29.993 INFO [0] 1004 final dependency list [JobId(1002), JobId(1003)], : downstairs
Sep 22 23:14:29.993 DEBG Received NoOP 1004
Sep 22 23:14:29.993 DEBG Received NoOP 1004
Sep 22 23:14:29.994 DEBG Received NoOP 1004
Sep 22 23:14:29.994 DEBG Work of: LiveNoOp 1004
Sep 22 23:14:29.994 DEBG LiveNoOp  :1004 deps:[JobId(1002), JobId(1003)] res:true
Sep 22 23:14:29.995 DEBG LiveReopen:1005 extent 0 deps:[JobId(1002), JobId(1003), JobId(1004)] res:true
Sep 22 23:14:29.995 DEBG Work of: LiveNoOp 1004
Sep 22 23:14:29.995 DEBG LiveNoOp  :1004 deps:[JobId(1002), JobId(1003)] res:true
Sep 22 23:14:29.997 DEBG LiveReopen:1005 extent 0 deps:[JobId(1002), JobId(1003), JobId(1004)] res:true
Sep 22 23:14:29.997 DEBG Work of: LiveNoOp 1004
Sep 22 23:14:29.997 DEBG LiveNoOp  :1004 deps:[JobId(1002), JobId(1003)] res:true
Sep 22 23:14:30.001 DEBG LiveReopen:1005 extent 0 deps:[JobId(1002), JobId(1003), JobId(1004)] res:true
Sep 22 23:14:30.002 DEBG [0] ExtentLiveNoOp AckReady 1004, : downstairs
Sep 22 23:14:30.002 DEBG [0] ExtentLiveReopen AckReady 1005, : downstairs
Sep 22 23:14:30.002 DEBG up_ds_listen was notified
Sep 22 23:14:30.002 DEBG up_ds_listen process 1004
Sep 22 23:14:30.002 DEBG [A] ack job 1004:5, : downstairs
Sep 22 23:14:30.002 DEBG up_ds_listen process 1005
Sep 22 23:14:30.002 DEBG [A] ack job 1005:6, : downstairs
Sep 22 23:14:30.002 DEBG up_ds_listen checked 2 jobs, back to waiting
Sep 22 23:14:30.002 DEBG up_ds_listen was notified
Sep 22 23:14:30.002 DEBG up_ds_listen checked 0 jobs, back to waiting
Sep 22 23:14:30.002 DEBG Extent 0 id:1004 Done
Sep 22 23:14:30.002 INFO RE:0 Wait for result from reopen command 1005:6
Sep 22 23:14:30.002 DEBG Extent 0 id:1005 Done
Sep 22 23:14:30.002 INFO Start extent 1 repair, task: repair
Sep 22 23:14:30.002 DEBG RE:1 Repair extent begins
Sep 22 23:14:30.002 DEBG Create new job ids for 1, : downstairs
Sep 22 23:14:30.002 INFO RE:1 repair extent with ids 1006,1007,1008,1009 deps:[]
Sep 22 23:14:30.002 DEBG Enqueue repair job 1009, : downstairs
Sep 22 23:14:30.002 DEBG Enqueue repair job 1006, : downstairs
Sep 22 23:14:30.002 INFO RE:1 close id:1006 queued, notify DS
Sep 22 23:14:30.002 INFO RE:1 Wait for result from close command 1006:7
Sep 22 23:14:30.002 DEBG [0] 1006 Remove check skipped:{JobId(1001)} from deps:[], : downstairs
Sep 22 23:14:30.003 DEBG [0] 1006 Remove check < min repaired:1002 from deps:[], : downstairs
Sep 22 23:14:30.003 INFO [0] 1006 final dependency list [], : downstairs
Sep 22 23:14:30.003 DEBG [0] 1009 Remove check skipped:{JobId(1001)} from deps:[JobId(1006), JobId(1007), JobId(1008)], : downstairs
Sep 22 23:14:30.003 DEBG [0] 1009 Remove check < min repaired:1002 from deps:[JobId(1006), JobId(1007), JobId(1008)], : downstairs
Sep 22 23:14:30.003 INFO [0] 1009 final dependency list [JobId(1006), JobId(1007), JobId(1008)], : downstairs
Sep 22 23:14:30.003 DEBG Flush just extent 1 with f:3 and g:1
Sep 22 23:14:30.004 DEBG FlushClose:1006 extent 1 deps:[] res:true f:3 g:1
Sep 22 23:14:30.004 DEBG Flush just extent 1 with f:3 and g:1
Sep 22 23:14:30.006 DEBG FlushClose:1006 extent 1 deps:[] res:true f:3 g:1
Sep 22 23:14:30.007 DEBG JustClose :1006 extent 1 deps:[] res:true
Sep 22 23:14:30.007 DEBG [1] ELC got g:1 f:1 d:false
Sep 22 23:14:30.007 DEBG [2] ELC got g:1 f:1 d:false
Sep 22 23:14:30.007 DEBG [0] ELC got g:0 f:0 d:false
Sep 22 23:14:30.007 DEBG [0] ExtentFlushClose 1006 AckReady, : downstairs
Sep 22 23:14:30.007 DEBG up_ds_listen was notified
Sep 22 23:14:30.007 DEBG up_ds_listen process 1006
Sep 22 23:14:30.007 DEBG [A] ack job 1006:7, : downstairs
Sep 22 23:14:30.007 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:30.007 DEBG Extent 1 id:1006 Done
Sep 22 23:14:30.007 DEBG Get repair info for 2 source, : downstairs
Sep 22 23:14:30.007 DEBG Get repair info for 0 bad, : downstairs
Sep 22 23:14:30.007 INFO Repair for extent 1 s:2 d:[ClientId(0)], : downstairs
Sep 22 23:14:30.007 DEBG Enqueue repair job 1007, : downstairs
Sep 22 23:14:30.007 INFO RE:1 Wait for result from repair command 1007:8
Sep 22 23:14:30.008 DEBG [0] 1007 Remove check skipped:{JobId(1001)} from deps:[JobId(1006)], : downstairs
Sep 22 23:14:30.008 DEBG [0] 1007 Remove check < min repaired:1002 from deps:[JobId(1006)], : downstairs
Sep 22 23:14:30.008 INFO [0] 1007 final dependency list [JobId(1006)], : downstairs
Sep 22 23:14:30.008 DEBG Received NoOP 1007
Sep 22 23:14:30.008 DEBG Received NoOP 1007
Sep 22 23:14:30.008 DEBG Received ExtentLiveRepair 1007
Sep 22 23:14:30.008 DEBG Work of: LiveNoOp 1007
Sep 22 23:14:30.008 DEBG LiveNoOp  :1007 deps:[JobId(1006)] res:true
Sep 22 23:14:30.008 WARN 1009 job ELiveReopen for connection UpstairsConnection { upstairs_id: e94af85e-3796-4fab-91a1-f12add9c3020, session_id: a86ae319-11ad-4b2d-a80b-330efc92ebef, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:14:30.008 DEBG Work of: LiveNoOp 1007
Sep 22 23:14:30.008 DEBG LiveNoOp  :1007 deps:[JobId(1006)] res:true
Sep 22 23:14:30.008 WARN 1009 job ELiveReopen for connection UpstairsConnection { upstairs_id: e94af85e-3796-4fab-91a1-f12add9c3020, session_id: a86ae319-11ad-4b2d-a80b-330efc92ebef, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:14:30.008 DEBG ExtentLiveRepair: extent 1 sra:127.0.0.1:54607
Sep 22 23:14:30.009 INFO Created copy dir "/tmp/downstairs-08law0EE/00/000/001.copy"
Sep 22 23:14:30.094 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:30.094 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:30.094 INFO Opened existing region file "/tmp/downstairs-fjJjawz1/region.json"
Sep 22 23:14:30.094 INFO Database read version 1
Sep 22 23:14:30.094 INFO Database write version 1
Sep 22 23:14:30.120 INFO accepted connection, remote_addr: 127.0.0.1:65506, local_addr: 127.0.0.1:54607, task: repair
Sep 22 23:14:30.120 TRCE incoming request, uri: /extent/1/files, method: GET, req_id: c0d11ec1-ae70-4353-a172-0ff1da32ca7e, remote_addr: 127.0.0.1:65506, local_addr: 127.0.0.1:54607, task: repair
Sep 22 23:14:30.121 INFO request completed, latency_us: 318, response_code: 200, uri: /extent/1/files, method: GET, req_id: c0d11ec1-ae70-4353-a172-0ff1da32ca7e, remote_addr: 127.0.0.1:65506, local_addr: 127.0.0.1:54607, task: repair
Sep 22 23:14:30.121 INFO eid:1 Found repair files: ["001", "001.db"]
Sep 22 23:14:30.121 TRCE incoming request, uri: /newextent/1/data, method: GET, req_id: 99f42dac-fa1b-4fdb-8823-813c7b7cd69f, remote_addr: 127.0.0.1:65506, local_addr: 127.0.0.1:54607, task: repair
Sep 22 23:14:30.122 INFO request completed, latency_us: 459, response_code: 200, uri: /newextent/1/data, method: GET, req_id: 99f42dac-fa1b-4fdb-8823-813c7b7cd69f, remote_addr: 127.0.0.1:65506, local_addr: 127.0.0.1:54607, task: repair
Sep 22 23:14:30.123 TRCE incoming request, uri: /newextent/1/db, method: GET, req_id: de091ddb-886a-40f5-8b4f-79405fac48ac, remote_addr: 127.0.0.1:65506, local_addr: 127.0.0.1:54607, task: repair
Sep 22 23:14:30.123 INFO request completed, latency_us: 381, response_code: 200, uri: /newextent/1/db, method: GET, req_id: de091ddb-886a-40f5-8b4f-79405fac48ac, remote_addr: 127.0.0.1:65506, local_addr: 127.0.0.1:54607, task: repair
Sep 22 23:14:30.124 INFO Repair files downloaded, move directory "/tmp/downstairs-08law0EE/00/000/001.copy" to "/tmp/downstairs-08law0EE/00/000/001.replace"
Sep 22 23:14:30.124 DEBG fsync completed for: "/tmp/downstairs-08law0EE/00/000"
Sep 22 23:14:30.125 INFO UUID: 65afeef9-2fd5-4a00-9b98-8d6c60e46268
Sep 22 23:14:30.125 INFO Blocks per extent:512 Total Extents: 188
Sep 22 23:14:30.125 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:30.125 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:30.125 INFO Using address: 127.0.0.1:62859, task: main
Sep 22 23:14:30.125 INFO Copy files from "/tmp/downstairs-08law0EE/00/000/001.replace" in "/tmp/downstairs-08law0EE/00/000"
Sep 22 23:14:30.125 DEBG fsync completed for: "/tmp/downstairs-08law0EE/00/000/001"
Sep 22 23:14:30.126 DEBG fsync completed for: "/tmp/downstairs-08law0EE/00/000/001.db"
Sep 22 23:14:30.126 DEBG fsync completed for: "/tmp/downstairs-08law0EE/00/000"
Sep 22 23:14:30.126 INFO Move directory  "/tmp/downstairs-08law0EE/00/000/001.replace" to "/tmp/downstairs-08law0EE/00/000/001.completed"
Sep 22 23:14:30.126 DEBG fsync completed for: "/tmp/downstairs-08law0EE/00/000"
Sep 22 23:14:30.126 DEBG fsync completed for: "/tmp/downstairs-08law0EE/00/000"
Sep 22 23:14:30.126 DEBG LiveRepair:1007 extent 1 deps:[JobId(1006)] res:true
Sep 22 23:14:30.126 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:30.126 WARN 1009 job ELiveReopen for connection UpstairsConnection { upstairs_id: e94af85e-3796-4fab-91a1-f12add9c3020, session_id: a86ae319-11ad-4b2d-a80b-330efc92ebef, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:14:30.126 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:47266, task: repair
Sep 22 23:14:30.126 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:47266, task: repair
Sep 22 23:14:30.126 INFO listening, local_addr: 127.0.0.1:47266, task: repair
Sep 22 23:14:30.126 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:47266, task: repair
Sep 22 23:14:30.126 DEBG [0] ExtentLiveRepair AckReady 1007, : downstairs
Sep 22 23:14:30.126 INFO Using repair address: 127.0.0.1:47266, task: main
Sep 22 23:14:30.126 INFO No SSL acceptor configured, task: main
Sep 22 23:14:30.126 DEBG up_ds_listen was notified
Sep 22 23:14:30.126 DEBG up_ds_listen process 1007
Sep 22 23:14:30.126 DEBG [A] ack job 1007:8, : downstairs
Sep 22 23:14:30.126 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:30.126 DEBG Extent 1 id:1007 Done
Sep 22 23:14:30.127 DEBG Enqueue repair job 1008, : downstairs
Sep 22 23:14:30.127 INFO RE:1 Wait for result from NoOp command 1008:9
Sep 22 23:14:30.127 DEBG [0] 1008 Remove check skipped:{JobId(1001)} from deps:[JobId(1006), JobId(1007)], : downstairs
Sep 22 23:14:30.127 DEBG [0] 1008 Remove check < min repaired:1002 from deps:[JobId(1006), JobId(1007)], : downstairs
Sep 22 23:14:30.127 INFO [0] 1008 final dependency list [JobId(1006), JobId(1007)], : downstairs
Sep 22 23:14:30.127 DEBG Received NoOP 1008
Sep 22 23:14:30.127 DEBG Received NoOP 1008
Sep 22 23:14:30.127 DEBG Received NoOP 1008
Sep 22 23:14:30.127 DEBG Work of: LiveNoOp 1008
Sep 22 23:14:30.127 DEBG LiveNoOp  :1008 deps:[JobId(1006), JobId(1007)] res:true
Sep 22 23:14:30.129 DEBG LiveReopen:1009 extent 1 deps:[JobId(1006), JobId(1007), JobId(1008)] res:true
Sep 22 23:14:30.129 DEBG Work of: LiveNoOp 1008
Sep 22 23:14:30.129 DEBG LiveNoOp  :1008 deps:[JobId(1006), JobId(1007)] res:true
Sep 22 23:14:30.131 DEBG LiveReopen:1009 extent 1 deps:[JobId(1006), JobId(1007), JobId(1008)] res:true
Sep 22 23:14:30.131 DEBG Work of: LiveNoOp 1008
Sep 22 23:14:30.131 DEBG LiveNoOp  :1008 deps:[JobId(1006), JobId(1007)] res:true
Sep 22 23:14:30.133 DEBG LiveReopen:1009 extent 1 deps:[JobId(1006), JobId(1007), JobId(1008)] res:true
Sep 22 23:14:30.133 DEBG [0] ExtentLiveNoOp AckReady 1008, : downstairs
Sep 22 23:14:30.134 DEBG [0] ExtentLiveReopen AckReady 1009, : downstairs
Sep 22 23:14:30.134 DEBG up_ds_listen was notified
Sep 22 23:14:30.134 DEBG up_ds_listen process 1008
Sep 22 23:14:30.134 DEBG [A] ack job 1008:9, : downstairs
Sep 22 23:14:30.134 DEBG up_ds_listen process 1009
Sep 22 23:14:30.134 DEBG [A] ack job 1009:10, : downstairs
Sep 22 23:14:30.134 DEBG up_ds_listen checked 2 jobs, back to waiting
Sep 22 23:14:30.134 DEBG up_ds_listen was notified
Sep 22 23:14:30.134 DEBG up_ds_listen checked 0 jobs, back to waiting
Sep 22 23:14:30.134 DEBG Extent 1 id:1008 Done
Sep 22 23:14:30.134 INFO RE:1 Wait for result from reopen command 1009:10
Sep 22 23:14:30.134 DEBG Extent 1 id:1009 Done
Sep 22 23:14:30.134 DEBG IO Flush 1010 has deps [JobId(1009), JobId(1008), JobId(1007), JobId(1006), JobId(1005), JobId(1004), JobId(1003), JobId(1002)]
Sep 22 23:14:30.134 INFO LiveRepair final flush submitted
Sep 22 23:14:30.134 DEBG [0] 1010 Remove check skipped:{JobId(1001)} from deps:[JobId(1009), JobId(1008), JobId(1007), JobId(1006), JobId(1005), JobId(1004), JobId(1003), JobId(1002)], : downstairs
Sep 22 23:14:30.134 DEBG [0] 1010 Remove check < min repaired:1002 from deps:[JobId(1009), JobId(1008), JobId(1007), JobId(1006), JobId(1005), JobId(1004), JobId(1003), JobId(1002)], : downstairs
Sep 22 23:14:30.134 INFO [0] 1010 final dependency list [JobId(1009), JobId(1008), JobId(1007), JobId(1006), JobId(1005), JobId(1004), JobId(1003), JobId(1002)], : downstairs
Sep 22 23:14:30.135 DEBG Flush     :1010 extent_limit None deps:[JobId(1009), JobId(1008), JobId(1007), JobId(1006), JobId(1005), JobId(1004), JobId(1003), JobId(1002)] res:true f:4 g:1
Sep 22 23:14:30.135 DEBG Flush     :1010 extent_limit None deps:[JobId(1009), JobId(1008), JobId(1007), JobId(1006), JobId(1005), JobId(1004), JobId(1003), JobId(1002)] res:true f:4 g:1
Sep 22 23:14:30.135 DEBG Flush     :1010 extent_limit Some(1) deps:[JobId(1009), JobId(1008), JobId(1007), JobId(1006), JobId(1005), JobId(1004), JobId(1003), JobId(1002)] res:true f:4 g:1
Sep 22 23:14:30.135 DEBG up_ds_listen was notified
Sep 22 23:14:30.135 DEBG up_ds_listen process 1010
Sep 22 23:14:30.135 DEBG [A] ack job 1010:11, : downstairs
Sep 22 23:14:30.135 DEBG [rc] retire 1010 clears [JobId(1002), JobId(1003), JobId(1004), JobId(1005), JobId(1006), JobId(1007), JobId(1008), JobId(1009), JobId(1010)], : downstairs
Sep 22 23:14:30.135 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:30.135 INFO LiveRepair final flush completed
Sep 22 23:14:30.135 INFO [0] e94af85e-3796-4fab-91a1-f12add9c3020 (a86ae319-11ad-4b2d-a80b-330efc92ebef) LiveRepair Active Active ds_transition to Active
Sep 22 23:14:30.135 INFO [0] Transition from LiveRepair to Active
Sep 22 23:14:30.135 WARN Live Repair returns Ok(())
Sep 22 23:14:30.158 INFO UUID: 6e66ae97-3dfc-40c6-a1d0-7d1c5297fdbf
Sep 22 23:14:30.158 INFO Blocks per extent:512 Total Extents: 188
Sep 22 23:14:30.159 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:30.159 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:30.159 INFO Using address: 127.0.0.1:44532, task: main
Sep 22 23:14:30.159 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:30.159 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:35145, task: repair
Sep 22 23:14:30.159 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:35145, task: repair
Sep 22 23:14:30.159 INFO listening, local_addr: 127.0.0.1:35145, task: repair
Sep 22 23:14:30.159 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:35145, task: repair
Sep 22 23:14:30.159 INFO Using repair address: 127.0.0.1:35145, task: main
Sep 22 23:14:30.159 INFO No SSL acceptor configured, task: main
note: configured to log to "/dev/stdout"
{"msg":"listening","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.17327078Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"local_addr":"127.0.0.1:45536","component":"dropshot"}
{"msg":"listen IP: 127.0.0.1:45536","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.173473694Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
note: configured to log to "/dev/stdout"
{"msg":"listening","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.196422565Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"local_addr":"127.0.0.1:34477","component":"dropshot"}
{"msg":"listen IP: 127.0.0.1:34477","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.196620883Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
Sep 22 23:14:30.213 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:30.213 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:30.213 INFO Opened existing region file "/tmp/downstairs-1HhZJsN0/region.json"
Sep 22 23:14:30.213 INFO Database read version 1
Sep 22 23:14:30.213 INFO Database write version 1
Sep 22 23:14:30.256 INFO UUID: 9da41dd6-4cdf-4b03-8df1-d13ee59384a3
Sep 22 23:14:30.256 INFO Blocks per extent:512 Total Extents: 188
Sep 22 23:14:30.256 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:30.257 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:30.257 INFO Using address: 127.0.0.1:51108, task: main
Sep 22 23:14:30.257 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:30.257 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:55881, task: repair
Sep 22 23:14:30.257 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:55881, task: repair
Sep 22 23:14:30.257 INFO listening, local_addr: 127.0.0.1:55881, task: repair
Sep 22 23:14:30.257 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:55881, task: repair
Sep 22 23:14:30.257 INFO Using repair address: 127.0.0.1:55881, task: main
Sep 22 23:14:30.257 INFO No SSL acceptor configured, task: main
Sep 22 23:14:30.267 INFO listening on 127.0.0.1:0, task: main
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.267621329Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"local_addr":"127.0.0.1:45536","component":"dropshot","remote_addr":"127.0.0.1:41882"}
{"msg":"no entry exists for volume a9fc305b-73b1-411e-9d00-c6edbf74297f, constructing...","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.268311387Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Upstairs starts","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.268522142Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Crucible Version: BuildInfo {\n    version: \"0.0.1\",\n    git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n    git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n    git_branch: \"main\",\n    rustc_semver: \"1.70.0\",\n    rustc_channel: \"stable\",\n    rustc_host_triple: \"x86_64-unknown-illumos\",\n    rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n    cargo_triple: \"x86_64-unknown-illumos\",\n    debug: true,\n    opt_level: 0,\n}","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.268599861Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.268647178Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Crucible stats registered with UUID: d37ce371-0ed6-44c3-bb8f-e1d703ba2c92","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.268698689Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Crucible d37ce371-0ed6-44c3-bb8f-e1d703ba2c92 has session id: 86116c5e-f3b4-46a4-818a-c22a616d06ea","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.268746628Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] connecting to 127.0.0.1:62776","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.268790251Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"0","component":"datafile"}
{"msg":"[1] connecting to 127.0.0.1:60728","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.268963291Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"1","component":"datafile"}
{"msg":"[2] connecting to 127.0.0.1:62859","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.269113185Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"2","component":"datafile"}
{"msg":"up_listen starts","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.269273477Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile","task":"up_listen"}
{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.26932309Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Flush timeout: 0.5","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.269358863Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"volume a9fc305b-73b1-411e-9d00-c6edbf74297f constructed ok","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.269596463Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
The guest has requested activation
{"msg":"d37ce371-0ed6-44c3-bb8f-e1d703ba2c92 active request set","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.269931269Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:30.270 INFO accepted connection from 127.0.0.1:56292, task: main
{"msg":"[0] d37ce371-0ed6-44c3-bb8f-e1d703ba2c92 looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.270179307Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"0","component":"datafile"}
{"msg":"[0] Proc runs for 127.0.0.1:62776 in state New","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.270233431Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] d37ce371-0ed6-44c3-bb8f-e1d703ba2c92 looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.270276957Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"1","component":"datafile"}
{"msg":"[1] Proc runs for 127.0.0.1:60728 in state New","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.270320621Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] d37ce371-0ed6-44c3-bb8f-e1d703ba2c92 looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.270362061Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"2","component":"datafile"}
{"msg":"[2] Proc runs for 127.0.0.1:62859 in state New","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.270400459Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:30.270 INFO accepted connection from 127.0.0.1:61683, task: main
Sep 22 23:14:30.270 INFO accepted connection from 127.0.0.1:52790, task: main
Sep 22 23:14:30.270 INFO Connection request from d37ce371-0ed6-44c3-bb8f-e1d703ba2c92 with version 4, task: proc
Sep 22 23:14:30.270 INFO upstairs UpstairsConnection { upstairs_id: d37ce371-0ed6-44c3-bb8f-e1d703ba2c92, session_id: 1e2cd9af-31e3-4c00-a3ff-558b5ef140ef, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:30.270 INFO Connection request from d37ce371-0ed6-44c3-bb8f-e1d703ba2c92 with version 4, task: proc
Sep 22 23:14:30.270 INFO upstairs UpstairsConnection { upstairs_id: d37ce371-0ed6-44c3-bb8f-e1d703ba2c92, session_id: 1e2cd9af-31e3-4c00-a3ff-558b5ef140ef, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:30.271 INFO Connection request from d37ce371-0ed6-44c3-bb8f-e1d703ba2c92 with version 4, task: proc
Sep 22 23:14:30.271 INFO upstairs UpstairsConnection { upstairs_id: d37ce371-0ed6-44c3-bb8f-e1d703ba2c92, session_id: 1e2cd9af-31e3-4c00-a3ff-558b5ef140ef, gen: 1 } connected, version 4, task: proc
{"msg":"[0] d37ce371-0ed6-44c3-bb8f-e1d703ba2c92 (1e2cd9af-31e3-4c00-a3ff-558b5ef140ef) New New New ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.271272839Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.27132551Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] client is_active_req TRUE, promote! session 1e2cd9af-31e3-4c00-a3ff-558b5ef140ef","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.271369241Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] d37ce371-0ed6-44c3-bb8f-e1d703ba2c92 (1e2cd9af-31e3-4c00-a3ff-558b5ef140ef) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.271415219Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.271454917Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] client is_active_req TRUE, promote! session 1e2cd9af-31e3-4c00-a3ff-558b5ef140ef","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.271492641Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] d37ce371-0ed6-44c3-bb8f-e1d703ba2c92 (1e2cd9af-31e3-4c00-a3ff-558b5ef140ef) WaitActive WaitActive New ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.271532227Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.271573546Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] client is_active_req TRUE, promote! session 1e2cd9af-31e3-4c00-a3ff-558b5ef140ef","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.271611436Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:30.272 INFO UpstairsConnection { upstairs_id: d37ce371-0ed6-44c3-bb8f-e1d703ba2c92, session_id: 1e2cd9af-31e3-4c00-a3ff-558b5ef140ef, gen: 1 } is now active (read-write)
Sep 22 23:14:30.272 INFO UpstairsConnection { upstairs_id: d37ce371-0ed6-44c3-bb8f-e1d703ba2c92, session_id: 1e2cd9af-31e3-4c00-a3ff-558b5ef140ef, gen: 1 } is now active (read-write)
Sep 22 23:14:30.272 INFO UpstairsConnection { upstairs_id: d37ce371-0ed6-44c3-bb8f-e1d703ba2c92, session_id: 1e2cd9af-31e3-4c00-a3ff-558b5ef140ef, gen: 1 } is now active (read-write)
note: configured to log to "/dev/stdout"
{"msg":"[0] downstairs client at 127.0.0.1:62776 has UUID 10109090-7e8f-489a-9e4a-65ce6167d00b","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.2731826Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 512, shift: 9 }, extent_count: 188, uuid: 10109090-7e8f-489a-9e4a-65ce6167d00b, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.273250842Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"d37ce371-0ed6-44c3-bb8f-e1d703ba2c92 WaitActive WaitActive WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.273290949Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] downstairs client at 127.0.0.1:60728 has UUID 02d6cbbb-c0cd-42b6-b416-45900a2ec4e0","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.273328224Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 512, shift: 9 }, extent_count: 188, uuid: 02d6cbbb-c0cd-42b6-b416-45900a2ec4e0, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.273375829Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"d37ce371-0ed6-44c3-bb8f-e1d703ba2c92 WaitActive WaitActive WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.273421821Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] downstairs client at 127.0.0.1:62859 has UUID 65afeef9-2fd5-4a00-9b98-8d6c60e46268","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.273463591Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 512, shift: 9 }, extent_count: 188, uuid: 65afeef9-2fd5-4a00-9b98-8d6c60e46268, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.273511409Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"d37ce371-0ed6-44c3-bb8f-e1d703ba2c92 WaitActive WaitActive WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.273554837Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.274342084Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"local_addr":"127.0.0.1:61933","component":"dropshot"}
{"msg":"listen IP: 127.0.0.1:61933","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.274558364Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
Sep 22 23:14:30.292 INFO Current flush_numbers [0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:14:30.295 INFO Current flush_numbers [0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:14:30.298 INFO Current flush_numbers [0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:14:30.301 INFO UUID: ef580430-d0f3-4a91-9bfb-605bdf8f791b
Sep 22 23:14:30.301 INFO Blocks per extent:512 Total Extents: 188
Sep 22 23:14:30.301 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:30.301 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:30.301 INFO Using address: 127.0.0.1:47631, task: main
Sep 22 23:14:30.302 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:30.302 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:52204, task: repair
Sep 22 23:14:30.302 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:52204, task: repair
Sep 22 23:14:30.302 INFO listening, local_addr: 127.0.0.1:52204, task: repair
Sep 22 23:14:30.302 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:52204, task: repair
Sep 22 23:14:30.302 INFO Using repair address: 127.0.0.1:52204, task: main
Sep 22 23:14:30.302 INFO No SSL acceptor configured, task: main
note: configured to log to "/dev/stdout"
{"msg":"listening","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.316329405Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"local_addr":"127.0.0.1:33566","component":"dropshot"}
{"msg":"listen IP: 127.0.0.1:33566","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.316531665Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
Sep 22 23:14:30.326 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:30.328 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:30.329 INFO Downstairs has completed Negotiation, task: proc
{"msg":"[0] d37ce371-0ed6-44c3-bb8f-e1d703ba2c92 (1e2cd9af-31e3-4c00-a3ff-558b5ef140ef) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.329961026Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.330016392Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:14:30.330055871Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.330091414Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] d37ce371-0ed6-44c3-bb8f-e1d703ba2c92 (1e2cd9af-31e3-4c00-a3ff-558b5ef140ef) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.330327442Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.330372308Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:14:30.330411055Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.330443786Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] d37ce371-0ed6-44c3-bb8f-e1d703ba2c92 (1e2cd9af-31e3-4c00-a3ff-558b5ef140ef) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.330684565Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.330727256Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:14:30.330760255Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.330791792Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] 127.0.0.1:62776 task reports connection:true","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.330831696Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"d37ce371-0ed6-44c3-bb8f-e1d703ba2c92 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.330866463Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.330899845Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.330932227Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.330966242Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.331005211Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.331046686Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.331087686Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
The guest has finished waiting for activation
{"msg":"[2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.331130911Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.331171738Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.331209951Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Max found gen is 1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.331244371Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.331279412Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Next flush: 1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.331313564Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"All extents match","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.331348415Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"No downstairs repair required","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.331383855Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"No initial repair work was required","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.331419596Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.331462632Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"d37ce371-0ed6-44c3-bb8f-e1d703ba2c92 is now active with session: 1e2cd9af-31e3-4c00-a3ff-558b5ef140ef","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.331502245Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"d37ce371-0ed6-44c3-bb8f-e1d703ba2c92 Set Active after no repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.33153965Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.331576884Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Set check for repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.331612729Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] 127.0.0.1:60728 task reports connection:true","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.331649734Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"d37ce371-0ed6-44c3-bb8f-e1d703ba2c92 Active Active Active","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.331686857Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Set check for repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.331722578Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] 127.0.0.1:62859 task reports connection:true","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.331760302Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"d37ce371-0ed6-44c3-bb8f-e1d703ba2c92 Active Active Active","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.331797835Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Set check for repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.331834492Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] received reconcile message","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.331871064Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.331907459Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.331943437Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] received reconcile message","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.331979778Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.332015997Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.332051927Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] received reconcile message","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.332088192Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.332125303Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.332167865Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"volume a9fc305b-73b1-411e-9d00-c6edbf74297f activated ok","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.332206573Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"volume a9fc305b-73b1-411e-9d00-c6edbf74297f constructed and inserted ok","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.332245448Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.33228175Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/a9fc305b-73b1-411e-9d00-c6edbf74297f","method":"POST","req_id":"26f10781-7085-42d0-af89-960debed8e37","remote_addr":"127.0.0.1:41882","local_addr":"127.0.0.1:45536","component":"dropshot","latency_us":63626,"response_code":"200"}
Sep 22 23:14:30.333 INFO listening on 127.0.0.1:0, task: main
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.333529582Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"local_addr":"127.0.0.1:34477","component":"dropshot","remote_addr":"127.0.0.1:51700"}
{"msg":"no entry exists for volume cc43b43c-a066-4496-841d-0f340d8a3b1d, constructing...","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.334144751Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Upstairs starts","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.334306425Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Crucible Version: BuildInfo {\n    version: \"0.0.1\",\n    git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n    git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n    git_branch: \"main\",\n    rustc_semver: \"1.70.0\",\n    rustc_channel: \"stable\",\n    rustc_host_triple: \"x86_64-unknown-illumos\",\n    rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n    cargo_triple: \"x86_64-unknown-illumos\",\n    debug: true,\n    opt_level: 0,\n}","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.334386245Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.334429752Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Crucible stats registered with UUID: 18b01c87-5587-4e4c-af37-0e404569a230","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.334471812Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Crucible 18b01c87-5587-4e4c-af37-0e404569a230 has session id: 06e81f5d-ec2e-4b30-923b-a0b5fee55cdb","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.334515136Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] connecting to 127.0.0.1:62296","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.334555016Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"0","component":"datafile"}
{"msg":"[1] connecting to 127.0.0.1:62299","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.334596716Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"1","component":"datafile"}
{"msg":"[2] connecting to 127.0.0.1:44532","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.334648499Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"2","component":"datafile"}
{"msg":"up_listen starts","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.334718938Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile","task":"up_listen"}
{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.334764343Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Flush timeout: 0.5","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.334803643Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"volume cc43b43c-a066-4496-841d-0f340d8a3b1d constructed ok","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.334938825Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
The guest has requested activation
{"msg":"18b01c87-5587-4e4c-af37-0e404569a230 active request set","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.33513964Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] 18b01c87-5587-4e4c-af37-0e404569a230 looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.335208498Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"0","component":"datafile"}
{"msg":"[0] Proc runs for 127.0.0.1:62296 in state New","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.335254018Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] 18b01c87-5587-4e4c-af37-0e404569a230 looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.335295403Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"1","component":"datafile"}
{"msg":"[1] Proc runs for 127.0.0.1:62299 in state New","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.335337736Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:30.335 INFO accepted connection from 127.0.0.1:35197, task: main
{"msg":"[2] 18b01c87-5587-4e4c-af37-0e404569a230 looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.335378481Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"2","component":"datafile"}
{"msg":"[2] Proc runs for 127.0.0.1:44532 in state New","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.335420838Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:30.335 INFO accepted connection from 127.0.0.1:57231, task: main
Sep 22 23:14:30.335 INFO accepted connection from 127.0.0.1:41210, task: main
Sep 22 23:14:30.335 INFO Connection request from 18b01c87-5587-4e4c-af37-0e404569a230 with version 4, task: proc
Sep 22 23:14:30.335 INFO upstairs UpstairsConnection { upstairs_id: 18b01c87-5587-4e4c-af37-0e404569a230, session_id: 12531aa6-b93c-4bc2-9fe7-f954b135051a, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:30.335 INFO Connection request from 18b01c87-5587-4e4c-af37-0e404569a230 with version 4, task: proc
Sep 22 23:14:30.335 INFO upstairs UpstairsConnection { upstairs_id: 18b01c87-5587-4e4c-af37-0e404569a230, session_id: 12531aa6-b93c-4bc2-9fe7-f954b135051a, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:30.335 INFO Connection request from 18b01c87-5587-4e4c-af37-0e404569a230 with version 4, task: proc
Sep 22 23:14:30.335 INFO upstairs UpstairsConnection { upstairs_id: 18b01c87-5587-4e4c-af37-0e404569a230, session_id: 12531aa6-b93c-4bc2-9fe7-f954b135051a, gen: 1 } connected, version 4, task: proc
{"msg":"[0] 18b01c87-5587-4e4c-af37-0e404569a230 (12531aa6-b93c-4bc2-9fe7-f954b135051a) New New New ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.336019667Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.336075228Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] client is_active_req TRUE, promote! session 12531aa6-b93c-4bc2-9fe7-f954b135051a","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.336119721Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] 18b01c87-5587-4e4c-af37-0e404569a230 (12531aa6-b93c-4bc2-9fe7-f954b135051a) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.336163899Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.336203987Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] client is_active_req TRUE, promote! session 12531aa6-b93c-4bc2-9fe7-f954b135051a","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.336245398Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] 18b01c87-5587-4e4c-af37-0e404569a230 (12531aa6-b93c-4bc2-9fe7-f954b135051a) WaitActive WaitActive New ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.336288983Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.336328185Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] client is_active_req TRUE, promote! session 12531aa6-b93c-4bc2-9fe7-f954b135051a","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.336369381Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:30.336 INFO UpstairsConnection { upstairs_id: 18b01c87-5587-4e4c-af37-0e404569a230, session_id: 12531aa6-b93c-4bc2-9fe7-f954b135051a, gen: 1 } is now active (read-write)
Sep 22 23:14:30.336 INFO UpstairsConnection { upstairs_id: 18b01c87-5587-4e4c-af37-0e404569a230, session_id: 12531aa6-b93c-4bc2-9fe7-f954b135051a, gen: 1 } is now active (read-write)
Sep 22 23:14:30.336 INFO UpstairsConnection { upstairs_id: 18b01c87-5587-4e4c-af37-0e404569a230, session_id: 12531aa6-b93c-4bc2-9fe7-f954b135051a, gen: 1 } is now active (read-write)
{"msg":"[0] downstairs client at 127.0.0.1:62296 has UUID bec31c6c-c0ce-43db-8467-8d94b73527a2","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.337250804Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 512, shift: 9 }, extent_count: 188, uuid: bec31c6c-c0ce-43db-8467-8d94b73527a2, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.33731028Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"18b01c87-5587-4e4c-af37-0e404569a230 WaitActive WaitActive WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.337353585Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] downstairs client at 127.0.0.1:62299 has UUID ede072e9-d3f2-4884-bb86-eea53898f9d9","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.337396117Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 512, shift: 9 }, extent_count: 188, uuid: ede072e9-d3f2-4884-bb86-eea53898f9d9, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.337444434Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"18b01c87-5587-4e4c-af37-0e404569a230 WaitActive WaitActive WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.337492704Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] downstairs client at 127.0.0.1:44532 has UUID 6e66ae97-3dfc-40c6-a1d0-7d1c5297fdbf","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.337535731Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 512, shift: 9 }, extent_count: 188, uuid: 6e66ae97-3dfc-40c6-a1d0-7d1c5297fdbf, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.337583861Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"18b01c87-5587-4e4c-af37-0e404569a230 WaitActive WaitActive WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.337624909Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:30.349 INFO Current flush_numbers [0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:14:30.351 INFO Current flush_numbers [0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:14:30.353 INFO Current flush_numbers [0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:14:30.371 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:30.372 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:30.373 INFO Downstairs has completed Negotiation, task: proc
{"msg":"[0] 18b01c87-5587-4e4c-af37-0e404569a230 (12531aa6-b93c-4bc2-9fe7-f954b135051a) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.373829912Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.373885394Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:14:30.373927002Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.373966518Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] 18b01c87-5587-4e4c-af37-0e404569a230 (12531aa6-b93c-4bc2-9fe7-f954b135051a) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.374030757Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.374073285Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:14:30.374112974Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.374151997Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] 18b01c87-5587-4e4c-af37-0e404569a230 (12531aa6-b93c-4bc2-9fe7-f954b135051a) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.374282334Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.374337793Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:14:30.374379481Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.374418912Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] 127.0.0.1:62296 task reports connection:true","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.374459232Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"18b01c87-5587-4e4c-af37-0e404569a230 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.374500639Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
The guest has finished waiting for activation
{"msg":"[0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.37454145Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.374647961Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.374695113Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.374736618Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.374777362Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.374819585Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.374860356Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.374900448Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.374942412Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Max found gen is 1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.374980693Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.375019772Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Next flush: 1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.375057739Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"All extents match","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.375102578Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"No downstairs repair required","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.375142459Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"No initial repair work was required","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.375181801Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.375220788Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"18b01c87-5587-4e4c-af37-0e404569a230 is now active with session: 12531aa6-b93c-4bc2-9fe7-f954b135051a","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.375262748Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"18b01c87-5587-4e4c-af37-0e404569a230 Set Active after no repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.375302889Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.375342721Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Set check for repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.375380841Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] 127.0.0.1:62299 task reports connection:true","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.37542022Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"18b01c87-5587-4e4c-af37-0e404569a230 Active Active Active","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.3754596Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Set check for repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.375498057Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] 127.0.0.1:44532 task reports connection:true","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.375537606Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"18b01c87-5587-4e4c-af37-0e404569a230 Active Active Active","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.375577646Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Set check for repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.375615974Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] received reconcile message","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.375654772Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.375693618Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.375732996Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] received reconcile message","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.375771921Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.375875606Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.375924009Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] received reconcile message","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.375964349Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.376003342Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.376041402Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"volume cc43b43c-a066-4496-841d-0f340d8a3b1d activated ok","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.376081568Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"volume cc43b43c-a066-4496-841d-0f340d8a3b1d constructed and inserted ok","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.376121944Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.376159825Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/cc43b43c-a066-4496-841d-0f340d8a3b1d","method":"POST","req_id":"7cf62e10-5608-45f3-a8e8-04d75ee2f23f","remote_addr":"127.0.0.1:51700","local_addr":"127.0.0.1:34477","component":"dropshot","latency_us":41099,"response_code":"200"}
Sep 22 23:14:30.441 INFO listening on 127.0.0.1:0, task: main
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.441486484Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"local_addr":"127.0.0.1:61933","component":"dropshot","remote_addr":"127.0.0.1:58570"}
{"msg":"no entry exists for volume d6736968-99ba-4ebf-b4ff-15afb0904fed, constructing...","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.442161617Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Upstairs starts","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.442408574Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Crucible Version: BuildInfo {\n    version: \"0.0.1\",\n    git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n    git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n    git_branch: \"main\",\n    rustc_semver: \"1.70.0\",\n    rustc_channel: \"stable\",\n    rustc_host_triple: \"x86_64-unknown-illumos\",\n    rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n    cargo_triple: \"x86_64-unknown-illumos\",\n    debug: true,\n    opt_level: 0,\n}","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.442488296Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.44253129Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Crucible stats registered with UUID: 0d1229ab-bd36-49b6-95e1-d6287586b840","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.442573894Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Crucible 0d1229ab-bd36-49b6-95e1-d6287586b840 has session id: e49fc84a-8167-42c1-b25f-144be76533fa","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.442618149Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] connecting to 127.0.0.1:38920","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.442658585Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"0","component":"datafile"}
{"msg":"[1] connecting to 127.0.0.1:44435","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.442796167Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"1","component":"datafile"}
{"msg":"[2] connecting to 127.0.0.1:51108","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.442953274Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"2","component":"datafile"}
{"msg":"up_listen starts","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.443026457Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile","task":"up_listen"}
{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.443072027Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Flush timeout: 0.5","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.443111695Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"volume d6736968-99ba-4ebf-b4ff-15afb0904fed constructed ok","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.443315316Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
The guest has requested activation
{"msg":"0d1229ab-bd36-49b6-95e1-d6287586b840 active request set","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.443607911Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] 0d1229ab-bd36-49b6-95e1-d6287586b840 looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.443754712Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"0","component":"datafile"}
{"msg":"[0] Proc runs for 127.0.0.1:38920 in state New","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.443805029Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] 0d1229ab-bd36-49b6-95e1-d6287586b840 looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.443847182Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"1","component":"datafile"}
{"msg":"[1] Proc runs for 127.0.0.1:44435 in state New","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.443889755Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] 0d1229ab-bd36-49b6-95e1-d6287586b840 looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.44393623Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"2","component":"datafile"}
{"msg":"[2] Proc runs for 127.0.0.1:51108 in state New","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.443978455Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:30.444 INFO accepted connection from 127.0.0.1:59183, task: main
Sep 22 23:14:30.444 INFO accepted connection from 127.0.0.1:49569, task: main
Sep 22 23:14:30.444 INFO accepted connection from 127.0.0.1:64659, task: main
Sep 22 23:14:30.444 INFO Connection request from 0d1229ab-bd36-49b6-95e1-d6287586b840 with version 4, task: proc
Sep 22 23:14:30.444 INFO upstairs UpstairsConnection { upstairs_id: 0d1229ab-bd36-49b6-95e1-d6287586b840, session_id: 16060f7a-9986-4db1-95c8-74c4346b949d, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:30.444 INFO Connection request from 0d1229ab-bd36-49b6-95e1-d6287586b840 with version 4, task: proc
Sep 22 23:14:30.444 INFO upstairs UpstairsConnection { upstairs_id: 0d1229ab-bd36-49b6-95e1-d6287586b840, session_id: 16060f7a-9986-4db1-95c8-74c4346b949d, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:30.444 INFO Connection request from 0d1229ab-bd36-49b6-95e1-d6287586b840 with version 4, task: proc
Sep 22 23:14:30.444 INFO upstairs UpstairsConnection { upstairs_id: 0d1229ab-bd36-49b6-95e1-d6287586b840, session_id: 16060f7a-9986-4db1-95c8-74c4346b949d, gen: 1 } connected, version 4, task: proc
{"msg":"[0] 0d1229ab-bd36-49b6-95e1-d6287586b840 (16060f7a-9986-4db1-95c8-74c4346b949d) New New New ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.444922121Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.444970657Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] client is_active_req TRUE, promote! session 16060f7a-9986-4db1-95c8-74c4346b949d","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.445014182Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] 0d1229ab-bd36-49b6-95e1-d6287586b840 (16060f7a-9986-4db1-95c8-74c4346b949d) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.445059377Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.445099124Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] client is_active_req TRUE, promote! session 16060f7a-9986-4db1-95c8-74c4346b949d","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.445140357Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] 0d1229ab-bd36-49b6-95e1-d6287586b840 (16060f7a-9986-4db1-95c8-74c4346b949d) WaitActive WaitActive New ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.445183732Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.445223132Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] client is_active_req TRUE, promote! session 16060f7a-9986-4db1-95c8-74c4346b949d","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.445264354Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:30.445 INFO UpstairsConnection { upstairs_id: 0d1229ab-bd36-49b6-95e1-d6287586b840, session_id: 16060f7a-9986-4db1-95c8-74c4346b949d, gen: 1 } is now active (read-write)
Sep 22 23:14:30.446 INFO UpstairsConnection { upstairs_id: 0d1229ab-bd36-49b6-95e1-d6287586b840, session_id: 16060f7a-9986-4db1-95c8-74c4346b949d, gen: 1 } is now active (read-write)
Sep 22 23:14:30.446 INFO UpstairsConnection { upstairs_id: 0d1229ab-bd36-49b6-95e1-d6287586b840, session_id: 16060f7a-9986-4db1-95c8-74c4346b949d, gen: 1 } is now active (read-write)
{"msg":"[0] downstairs client at 127.0.0.1:38920 has UUID b38dafac-2da7-4558-aae1-959c564193b2","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.446738218Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 512, shift: 9 }, extent_count: 188, uuid: b38dafac-2da7-4558-aae1-959c564193b2, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.44679535Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"0d1229ab-bd36-49b6-95e1-d6287586b840 WaitActive WaitActive WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.446838117Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] downstairs client at 127.0.0.1:44435 has UUID 14399b3b-6e6e-4995-afe0-74c1441f0833","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.446887027Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 512, shift: 9 }, extent_count: 188, uuid: 14399b3b-6e6e-4995-afe0-74c1441f0833, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.446936466Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"0d1229ab-bd36-49b6-95e1-d6287586b840 WaitActive WaitActive WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.446978081Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] downstairs client at 127.0.0.1:51108 has UUID 9da41dd6-4cdf-4b03-8df1-d13ee59384a3","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.447019801Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 512, shift: 9 }, extent_count: 188, uuid: 9da41dd6-4cdf-4b03-8df1-d13ee59384a3, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.447067819Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"0d1229ab-bd36-49b6-95e1-d6287586b840 WaitActive WaitActive WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.447108491Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:30.461 INFO listening on 127.0.0.1:0, task: main
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.461477253Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"local_addr":"127.0.0.1:33566","component":"dropshot","remote_addr":"127.0.0.1:49478"}
{"msg":"no entry exists for volume 1e088c74-6c1b-4846-90a3-1bfe70bc4f7f, constructing...","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.462109168Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Upstairs starts","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.462329399Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Crucible Version: BuildInfo {\n    version: \"0.0.1\",\n    git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n    git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n    git_branch: \"main\",\n    rustc_semver: \"1.70.0\",\n    rustc_channel: \"stable\",\n    rustc_host_triple: \"x86_64-unknown-illumos\",\n    rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n    cargo_triple: \"x86_64-unknown-illumos\",\n    debug: true,\n    opt_level: 0,\n}","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.462408807Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.462452242Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Crucible stats registered with UUID: 982ad9a6-37b1-4b89-8804-8ba0c1300cac","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.462494742Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Crucible 982ad9a6-37b1-4b89-8804-8ba0c1300cac has session id: 92eb453c-9169-4bc7-aded-578f614d6e24","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.462538481Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] connecting to 127.0.0.1:46850","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.462585392Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"0","component":"datafile"}
{"msg":"[1] connecting to 127.0.0.1:50214","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.462739764Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"1","component":"datafile"}
{"msg":"[2] connecting to 127.0.0.1:47631","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.462889563Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"2","component":"datafile"}
{"msg":"up_listen starts","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.462964122Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile","task":"up_listen"}
{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.46300973Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Flush timeout: 0.5","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.463049577Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"volume 1e088c74-6c1b-4846-90a3-1bfe70bc4f7f constructed ok","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.463242792Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
The guest has requested activation
{"msg":"982ad9a6-37b1-4b89-8804-8ba0c1300cac active request set","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.463521429Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] 982ad9a6-37b1-4b89-8804-8ba0c1300cac looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.46367217Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"0","component":"datafile"}
{"msg":"[0] Proc runs for 127.0.0.1:46850 in state New","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.463722717Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] 982ad9a6-37b1-4b89-8804-8ba0c1300cac looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.463764671Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"1","component":"datafile"}
{"msg":"[1] Proc runs for 127.0.0.1:50214 in state New","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.463807223Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] 982ad9a6-37b1-4b89-8804-8ba0c1300cac looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.463847982Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"2","component":"datafile"}
{"msg":"[2] Proc runs for 127.0.0.1:47631 in state New","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.463890046Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:30.463 INFO accepted connection from 127.0.0.1:61789, task: main
Sep 22 23:14:30.464 INFO accepted connection from 127.0.0.1:58381, task: main
Sep 22 23:14:30.464 INFO accepted connection from 127.0.0.1:50053, task: main
Sep 22 23:14:30.464 INFO Connection request from 982ad9a6-37b1-4b89-8804-8ba0c1300cac with version 4, task: proc
Sep 22 23:14:30.464 INFO upstairs UpstairsConnection { upstairs_id: 982ad9a6-37b1-4b89-8804-8ba0c1300cac, session_id: 1dc57d2b-9138-4f1d-8c86-6d444ab88226, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:30.464 INFO Connection request from 982ad9a6-37b1-4b89-8804-8ba0c1300cac with version 4, task: proc
Sep 22 23:14:30.464 INFO upstairs UpstairsConnection { upstairs_id: 982ad9a6-37b1-4b89-8804-8ba0c1300cac, session_id: 1dc57d2b-9138-4f1d-8c86-6d444ab88226, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:30.464 INFO Connection request from 982ad9a6-37b1-4b89-8804-8ba0c1300cac with version 4, task: proc
Sep 22 23:14:30.464 INFO upstairs UpstairsConnection { upstairs_id: 982ad9a6-37b1-4b89-8804-8ba0c1300cac, session_id: 1dc57d2b-9138-4f1d-8c86-6d444ab88226, gen: 1 } connected, version 4, task: proc
{"msg":"[0] 982ad9a6-37b1-4b89-8804-8ba0c1300cac (1dc57d2b-9138-4f1d-8c86-6d444ab88226) New New New ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.464859546Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.464909989Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] client is_active_req TRUE, promote! session 1dc57d2b-9138-4f1d-8c86-6d444ab88226","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.46495327Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] 982ad9a6-37b1-4b89-8804-8ba0c1300cac (1dc57d2b-9138-4f1d-8c86-6d444ab88226) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.464997168Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.465037121Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] client is_active_req TRUE, promote! session 1dc57d2b-9138-4f1d-8c86-6d444ab88226","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.465078469Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] 982ad9a6-37b1-4b89-8804-8ba0c1300cac (1dc57d2b-9138-4f1d-8c86-6d444ab88226) WaitActive WaitActive New ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.465122149Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.465161766Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] client is_active_req TRUE, promote! session 1dc57d2b-9138-4f1d-8c86-6d444ab88226","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.465203133Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:30.465 INFO Current flush_numbers [0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:14:30.465 INFO UpstairsConnection { upstairs_id: 982ad9a6-37b1-4b89-8804-8ba0c1300cac, session_id: 1dc57d2b-9138-4f1d-8c86-6d444ab88226, gen: 1 } is now active (read-write)
Sep 22 23:14:30.465 INFO UpstairsConnection { upstairs_id: 982ad9a6-37b1-4b89-8804-8ba0c1300cac, session_id: 1dc57d2b-9138-4f1d-8c86-6d444ab88226, gen: 1 } is now active (read-write)
Sep 22 23:14:30.466 INFO UpstairsConnection { upstairs_id: 982ad9a6-37b1-4b89-8804-8ba0c1300cac, session_id: 1dc57d2b-9138-4f1d-8c86-6d444ab88226, gen: 1 } is now active (read-write)
{"msg":"[0] downstairs client at 127.0.0.1:46850 has UUID ed5caa6d-3818-4a41-afcf-7deb29720fa0","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.466673776Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 512, shift: 9 }, extent_count: 188, uuid: ed5caa6d-3818-4a41-afcf-7deb29720fa0, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.466731205Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"982ad9a6-37b1-4b89-8804-8ba0c1300cac WaitActive WaitActive WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.466780485Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] downstairs client at 127.0.0.1:50214 has UUID 2b517ebe-5910-495c-8a3d-f94697b298ea","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.466823818Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 512, shift: 9 }, extent_count: 188, uuid: 2b517ebe-5910-495c-8a3d-f94697b298ea, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.46687233Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"982ad9a6-37b1-4b89-8804-8ba0c1300cac WaitActive WaitActive WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.466913968Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] downstairs client at 127.0.0.1:47631 has UUID ef580430-d0f3-4a91-9bfb-605bdf8f791b","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.466955616Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 512, shift: 9 }, extent_count: 188, uuid: ef580430-d0f3-4a91-9bfb-605bdf8f791b, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.467003899Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"982ad9a6-37b1-4b89-8804-8ba0c1300cac WaitActive WaitActive WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.467044804Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:30.468 INFO Current flush_numbers [0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:14:30.470 INFO Current flush_numbers [0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:14:30.485 INFO Current flush_numbers [0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:14:30.487 INFO Current flush_numbers [0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:14:30.490 INFO Current flush_numbers [0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:14:30.500 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:30.502 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:30.503 INFO Downstairs has completed Negotiation, task: proc
{"msg":"[0] 0d1229ab-bd36-49b6-95e1-d6287586b840 (16060f7a-9986-4db1-95c8-74c4346b949d) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.504424784Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.504478707Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:14:30.504519667Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.504559221Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] 0d1229ab-bd36-49b6-95e1-d6287586b840 (16060f7a-9986-4db1-95c8-74c4346b949d) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.504833574Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.504888996Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:14:30.50493049Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.504969693Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] 0d1229ab-bd36-49b6-95e1-d6287586b840 (16060f7a-9986-4db1-95c8-74c4346b949d) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.505234332Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.505283052Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:14:30.505323542Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.505363063Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] 127.0.0.1:38920 task reports connection:true","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.505403623Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"0d1229ab-bd36-49b6-95e1-d6287586b840 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.505445619Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.505486258Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.505526599Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.50556882Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.505608917Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.505648954Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
The guest has finished waiting for activation
{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.505690833Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.505731224Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.50577122Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.505834863Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Max found gen is 1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.505874696Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.505914127Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Next flush: 1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.50595252Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"All extents match","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.505990943Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"No downstairs repair required","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.506029746Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"No initial repair work was required","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.506068857Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.506107847Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"0d1229ab-bd36-49b6-95e1-d6287586b840 is now active with session: 16060f7a-9986-4db1-95c8-74c4346b949d","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.50615002Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"0d1229ab-bd36-49b6-95e1-d6287586b840 Set Active after no repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.506190554Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.506230639Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Set check for repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.506268998Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] 127.0.0.1:44435 task reports connection:true","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.506308877Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"0d1229ab-bd36-49b6-95e1-d6287586b840 Active Active Active","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.506348767Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Set check for repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.506387031Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] 127.0.0.1:51108 task reports connection:true","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.506426704Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"0d1229ab-bd36-49b6-95e1-d6287586b840 Active Active Active","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.506466994Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Set check for repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.50650531Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] received reconcile message","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.506548785Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.506588706Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.506626834Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] received reconcile message","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.506665517Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.506704052Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.506742429Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] received reconcile message","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.50678127Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.506819935Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.506857895Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"volume d6736968-99ba-4ebf-b4ff-15afb0904fed activated ok","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.506897704Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"volume d6736968-99ba-4ebf-b4ff-15afb0904fed constructed and inserted ok","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.506938079Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.506976037Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/d6736968-99ba-4ebf-b4ff-15afb0904fed","method":"POST","req_id":"67684343-1827-49ac-9b89-0c30bb14fbb3","remote_addr":"127.0.0.1:58570","local_addr":"127.0.0.1:61933","component":"dropshot","latency_us":64358,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.507321271Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/d6736968-99ba-4ebf-b4ff-15afb0904fed/import_from_url","method":"POST","req_id":"96eae84f-8a8c-4cc9-bd62-02581d9bae36","remote_addr":"127.0.0.1:58570","local_addr":"127.0.0.1:61933","component":"dropshot","latency_us":285,"response_code":"200"}
Sep 22 23:14:30.518 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:30.519 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:30.521 INFO Downstairs has completed Negotiation, task: proc
{"msg":"[0] 982ad9a6-37b1-4b89-8804-8ba0c1300cac (1dc57d2b-9138-4f1d-8c86-6d444ab88226) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.521884777Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.521935812Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:14:30.521977026Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.522023388Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] 982ad9a6-37b1-4b89-8804-8ba0c1300cac (1dc57d2b-9138-4f1d-8c86-6d444ab88226) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.522288878Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.522337465Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:14:30.522378008Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.522416894Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] 982ad9a6-37b1-4b89-8804-8ba0c1300cac (1dc57d2b-9138-4f1d-8c86-6d444ab88226) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.522680269Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.522728255Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:14:30.522769737Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.522809014Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] 127.0.0.1:46850 task reports connection:true","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.522849063Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"982ad9a6-37b1-4b89-8804-8ba0c1300cac WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.522889711Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.522930322Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.522970481Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.523012643Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.523052677Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.523093036Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
The guest has finished waiting for activation
{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.523134991Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.523182187Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.523223511Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.523265596Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Max found gen is 1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.523303905Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.523342718Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Next flush: 1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.523380898Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"All extents match","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.523419347Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"No downstairs repair required","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.523458229Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"No initial repair work was required","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.52349734Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.523536535Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"982ad9a6-37b1-4b89-8804-8ba0c1300cac is now active with session: 1dc57d2b-9138-4f1d-8c86-6d444ab88226","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.523578292Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"982ad9a6-37b1-4b89-8804-8ba0c1300cac Set Active after no repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.523618474Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.523658254Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Set check for repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.523696736Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] 127.0.0.1:50214 task reports connection:true","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.523736072Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"982ad9a6-37b1-4b89-8804-8ba0c1300cac Active Active Active","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.523775997Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Set check for repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.523814414Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] 127.0.0.1:47631 task reports connection:true","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.523859151Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"982ad9a6-37b1-4b89-8804-8ba0c1300cac Active Active Active","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.523899915Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Set check for repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.523938218Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] received reconcile message","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.52397754Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.524016584Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.524055113Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] received reconcile message","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.524093761Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.524132627Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.524170894Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] received reconcile message","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.524209805Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.524248575Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.524287Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"volume 1e088c74-6c1b-4846-90a3-1bfe70bc4f7f activated ok","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.524327505Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"volume 1e088c74-6c1b-4846-90a3-1bfe70bc4f7f constructed and inserted ok","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.524368194Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.524406046Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/1e088c74-6c1b-4846-90a3-1bfe70bc4f7f","method":"POST","req_id":"37479699-b3cc-4173-8cc6-c0412bade8aa","remote_addr":"127.0.0.1:49478","local_addr":"127.0.0.1:33566","component":"dropshot","latency_us":61734,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.524565032Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/1e088c74-6c1b-4846-90a3-1bfe70bc4f7f/import_from_url","method":"POST","req_id":"54248e53-6b49-4caa-ae27-adf1891e3262","remote_addr":"127.0.0.1:49478","local_addr":"127.0.0.1:33566","component":"dropshot","latency_us":257,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.623933759Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/job/a52bd47d-76c9-4928-9712-218eb57bfeb7/is_finished","method":"GET","req_id":"078a732a-b54e-44e7-9aaa-34ba8eaef819","remote_addr":"127.0.0.1:49478","local_addr":"127.0.0.1:33566","component":"dropshot","latency_us":256,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.638097036Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/job/de4c6089-f970-4d99-9581-6bd61244a7f8/is_finished","method":"GET","req_id":"3197f6f8-dc99-4bf0-9eb3-3fe2fe0b15c9","remote_addr":"127.0.0.1:58570","local_addr":"127.0.0.1:61933","component":"dropshot","latency_us":196,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.65411087Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/a9fc305b-73b1-411e-9d00-c6edbf74297f/bulk_write","method":"POST","req_id":"736f6533-2ad4-4087-a4e2-eadc39266dcd","remote_addr":"127.0.0.1:41882","local_addr":"127.0.0.1:45536","component":"dropshot","latency_us":276862,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.697652614Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/cc43b43c-a066-4496-841d-0f340d8a3b1d/bulk_write","method":"POST","req_id":"4f5a16bb-68a7-4f21-abea-ef2b76fa4866","remote_addr":"127.0.0.1:51700","local_addr":"127.0.0.1:34477","component":"dropshot","latency_us":293338,"response_code":"204"}
{"msg":"detach removing entry for volume cc43b43c-a066-4496-841d-0f340d8a3b1d","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.701186482Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"detaching volume cc43b43c-a066-4496-841d-0f340d8a3b1d","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.701238311Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:30.768 DEBG Write     :1000 deps:[] res:true
Sep 22 23:14:30.776 DEBG Read      :1001 deps:[JobId(1000)] res:true
Sep 22 23:14:30.794 DEBG Write     :1000 deps:[] res:true
Sep 22 23:14:30.810 DEBG Write     :1000 deps:[] res:true
Sep 22 23:14:30.823 DEBG Write     :1000 deps:[] res:true
Sep 22 23:14:30.825 DEBG Write     :1000 deps:[] res:true
Sep 22 23:14:30.830 DEBG Read      :1001 deps:[JobId(1000)] res:true
Sep 22 23:14:30.830 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:1 g:1
Sep 22 23:14:30.831 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:1 g:1
Sep 22 23:14:30.831 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:1 g:1
{"msg":"Request to deactivate this guest","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.832230202Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"18b01c87-5587-4e4c-af37-0e404569a230 set deactivating.","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.832280118Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.832463663Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/cc43b43c-a066-4496-841d-0f340d8a3b1d","method":"DELETE","req_id":"9ed47dc6-fe17-4e98-8cef-19b14e551325","remote_addr":"127.0.0.1:51700","local_addr":"127.0.0.1:34477","component":"dropshot","latency_us":131485,"response_code":"204"}
Sep 22 23:14:30.832 INFO Upstairs starts
Sep 22 23:14:30.832 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:14:30.832 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:14:30.832 INFO Crucible stats registered with UUID: 18b01c87-5587-4e4c-af37-0e404569a230
Sep 22 23:14:30.832 INFO Crucible 18b01c87-5587-4e4c-af37-0e404569a230 has session id: 9fb860c1-25bf-4ac3-8286-f7b4bbced8f6
Sep 22 23:14:30.833 INFO [0] connecting to 127.0.0.1:62296, looper: 0
Sep 22 23:14:30.833 INFO [1] connecting to 127.0.0.1:62299, looper: 1
Sep 22 23:14:30.833 INFO [2] connecting to 127.0.0.1:44532, looper: 2
Sep 22 23:14:30.833 INFO up_listen starts, task: up_listen
Sep 22 23:14:30.833 INFO Wait for all three downstairs to come online
Sep 22 23:14:30.833 INFO Flush timeout: 0.5
Sep 22 23:14:30.833 INFO accepted connection from 127.0.0.1:59426, task: main
Sep 22 23:14:30.833 INFO accepted connection from 127.0.0.1:53704, task: main
Sep 22 23:14:30.833 INFO [0] 18b01c87-5587-4e4c-af37-0e404569a230 looper connected, looper: 0
Sep 22 23:14:30.833 INFO [0] Proc runs for 127.0.0.1:62296 in state New
Sep 22 23:14:30.833 INFO [1] 18b01c87-5587-4e4c-af37-0e404569a230 looper connected, looper: 1
Sep 22 23:14:30.833 INFO [1] Proc runs for 127.0.0.1:62299 in state New
Sep 22 23:14:30.833 INFO [2] 18b01c87-5587-4e4c-af37-0e404569a230 looper connected, looper: 2
Sep 22 23:14:30.833 INFO [2] Proc runs for 127.0.0.1:44532 in state New
Sep 22 23:14:30.833 INFO accepted connection from 127.0.0.1:45845, task: main
Sep 22 23:14:30.834 INFO Connection request from 18b01c87-5587-4e4c-af37-0e404569a230 with version 4, task: proc
Sep 22 23:14:30.834 INFO upstairs UpstairsConnection { upstairs_id: 18b01c87-5587-4e4c-af37-0e404569a230, session_id: e8b82b10-5353-499d-a44d-e0dd5d13af34, gen: 2 } connected, version 4, task: proc
Sep 22 23:14:30.834 INFO Connection request from 18b01c87-5587-4e4c-af37-0e404569a230 with version 4, task: proc
Sep 22 23:14:30.834 INFO upstairs UpstairsConnection { upstairs_id: 18b01c87-5587-4e4c-af37-0e404569a230, session_id: e8b82b10-5353-499d-a44d-e0dd5d13af34, gen: 2 } connected, version 4, task: proc
Sep 22 23:14:30.834 INFO Connection request from 18b01c87-5587-4e4c-af37-0e404569a230 with version 4, task: proc
Sep 22 23:14:30.834 INFO upstairs UpstairsConnection { upstairs_id: 18b01c87-5587-4e4c-af37-0e404569a230, session_id: e8b82b10-5353-499d-a44d-e0dd5d13af34, gen: 2 } connected, version 4, task: proc
Sep 22 23:14:30.834 INFO [0] 18b01c87-5587-4e4c-af37-0e404569a230 (e8b82b10-5353-499d-a44d-e0dd5d13af34) New New New ds_transition to WaitActive
Sep 22 23:14:30.834 INFO [0] Transition from New to WaitActive
Sep 22 23:14:30.834 INFO [1] 18b01c87-5587-4e4c-af37-0e404569a230 (e8b82b10-5353-499d-a44d-e0dd5d13af34) WaitActive New New ds_transition to WaitActive
Sep 22 23:14:30.834 INFO [1] Transition from New to WaitActive
Sep 22 23:14:30.834 INFO [2] 18b01c87-5587-4e4c-af37-0e404569a230 (e8b82b10-5353-499d-a44d-e0dd5d13af34) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:14:30.834 INFO [2] Transition from New to WaitActive
The guest has requested activation
Sep 22 23:14:30.835 INFO 18b01c87-5587-4e4c-af37-0e404569a230 active request set
Sep 22 23:14:30.835 INFO [0] received activate with gen 2
Sep 22 23:14:30.835 INFO [0] client got ds_active_rx, promote! session e8b82b10-5353-499d-a44d-e0dd5d13af34
Sep 22 23:14:30.835 INFO [1] received activate with gen 2
Sep 22 23:14:30.835 INFO [1] client got ds_active_rx, promote! session e8b82b10-5353-499d-a44d-e0dd5d13af34
Sep 22 23:14:30.835 INFO [2] received activate with gen 2
Sep 22 23:14:30.835 INFO [2] client got ds_active_rx, promote! session e8b82b10-5353-499d-a44d-e0dd5d13af34
Sep 22 23:14:30.835 WARN Attempting RW takeover from UpstairsConnection { upstairs_id: 18b01c87-5587-4e4c-af37-0e404569a230, session_id: 12531aa6-b93c-4bc2-9fe7-f954b135051a, gen: 1 } to UpstairsConnection { upstairs_id: 18b01c87-5587-4e4c-af37-0e404569a230, session_id: e8b82b10-5353-499d-a44d-e0dd5d13af34, gen: 2 }
Sep 22 23:14:30.835 WARN Signaling to UpstairsConnection { upstairs_id: 18b01c87-5587-4e4c-af37-0e404569a230, session_id: 12531aa6-b93c-4bc2-9fe7-f954b135051a, gen: 1 } thread that UpstairsConnection { upstairs_id: 18b01c87-5587-4e4c-af37-0e404569a230, session_id: e8b82b10-5353-499d-a44d-e0dd5d13af34, gen: 2 } is being promoted (read-write)
Sep 22 23:14:30.835 WARN Attempting RW takeover from UpstairsConnection { upstairs_id: 18b01c87-5587-4e4c-af37-0e404569a230, session_id: 12531aa6-b93c-4bc2-9fe7-f954b135051a, gen: 1 } to UpstairsConnection { upstairs_id: 18b01c87-5587-4e4c-af37-0e404569a230, session_id: e8b82b10-5353-499d-a44d-e0dd5d13af34, gen: 2 }
Sep 22 23:14:30.835 WARN Signaling to UpstairsConnection { upstairs_id: 18b01c87-5587-4e4c-af37-0e404569a230, session_id: 12531aa6-b93c-4bc2-9fe7-f954b135051a, gen: 1 } thread that UpstairsConnection { upstairs_id: 18b01c87-5587-4e4c-af37-0e404569a230, session_id: e8b82b10-5353-499d-a44d-e0dd5d13af34, gen: 2 } is being promoted (read-write)
Sep 22 23:14:30.835 WARN Attempting RW takeover from UpstairsConnection { upstairs_id: 18b01c87-5587-4e4c-af37-0e404569a230, session_id: 12531aa6-b93c-4bc2-9fe7-f954b135051a, gen: 1 } to UpstairsConnection { upstairs_id: 18b01c87-5587-4e4c-af37-0e404569a230, session_id: e8b82b10-5353-499d-a44d-e0dd5d13af34, gen: 2 }
Sep 22 23:14:30.835 WARN Signaling to UpstairsConnection { upstairs_id: 18b01c87-5587-4e4c-af37-0e404569a230, session_id: 12531aa6-b93c-4bc2-9fe7-f954b135051a, gen: 1 } thread that UpstairsConnection { upstairs_id: 18b01c87-5587-4e4c-af37-0e404569a230, session_id: e8b82b10-5353-499d-a44d-e0dd5d13af34, gen: 2 } is being promoted (read-write)
Sep 22 23:14:30.836 WARN Another upstairs UpstairsConnection { upstairs_id: 18b01c87-5587-4e4c-af37-0e404569a230, session_id: e8b82b10-5353-499d-a44d-e0dd5d13af34, gen: 2 } promoted to active, shutting down connection for UpstairsConnection { upstairs_id: 18b01c87-5587-4e4c-af37-0e404569a230, session_id: 12531aa6-b93c-4bc2-9fe7-f954b135051a, gen: 1 }, task: main
Sep 22 23:14:30.836 INFO UpstairsConnection { upstairs_id: 18b01c87-5587-4e4c-af37-0e404569a230, session_id: e8b82b10-5353-499d-a44d-e0dd5d13af34, gen: 2 } is now active (read-write)
Sep 22 23:14:30.836 WARN Another upstairs UpstairsConnection { upstairs_id: 18b01c87-5587-4e4c-af37-0e404569a230, session_id: e8b82b10-5353-499d-a44d-e0dd5d13af34, gen: 2 } promoted to active, shutting down connection for UpstairsConnection { upstairs_id: 18b01c87-5587-4e4c-af37-0e404569a230, session_id: 12531aa6-b93c-4bc2-9fe7-f954b135051a, gen: 1 }, task: main
Sep 22 23:14:30.836 INFO UpstairsConnection { upstairs_id: 18b01c87-5587-4e4c-af37-0e404569a230, session_id: e8b82b10-5353-499d-a44d-e0dd5d13af34, gen: 2 } is now active (read-write)
Sep 22 23:14:30.836 WARN Another upstairs UpstairsConnection { upstairs_id: 18b01c87-5587-4e4c-af37-0e404569a230, session_id: e8b82b10-5353-499d-a44d-e0dd5d13af34, gen: 2 } promoted to active, shutting down connection for UpstairsConnection { upstairs_id: 18b01c87-5587-4e4c-af37-0e404569a230, session_id: 12531aa6-b93c-4bc2-9fe7-f954b135051a, gen: 1 }, task: main
Sep 22 23:14:30.836 INFO UpstairsConnection { upstairs_id: 18b01c87-5587-4e4c-af37-0e404569a230, session_id: e8b82b10-5353-499d-a44d-e0dd5d13af34, gen: 2 } is now active (read-write)
Sep 22 23:14:30.836 INFO connection (127.0.0.1:35197): all done
Sep 22 23:14:30.836 INFO connection (127.0.0.1:57231): all done
Sep 22 23:14:30.836 INFO connection (127.0.0.1:41210): all done
{"msg":"[0] 18b01c87-5587-4e4c-af37-0e404569a230 (12531aa6-b93c-4bc2-9fe7-f954b135051a) cmd_loop saw YouAreNoLongerActive 18b01c87-5587-4e4c-af37-0e404569a230 e8b82b10-5353-499d-a44d-e0dd5d13af34 2","v":0,"name":"crucible-pantry","level":50,"time":"2023-09-22T23:14:30.836937435Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] 18b01c87-5587-4e4c-af37-0e404569a230 (12531aa6-b93c-4bc2-9fe7-f954b135051a) Active Active Active ds_transition to Disabled","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.836996899Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] Transition from Active to Disabled","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.837047035Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"18b01c87-5587-4e4c-af37-0e404569a230 set inactive, session 12531aa6-b93c-4bc2-9fe7-f954b135051a","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.837085529Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"127.0.0.1:62296: proc: Generation number is too low: saw YouAreNoLongerActive with larger gen 2 than ours 1","v":0,"name":"crucible-pantry","level":50,"time":"2023-09-22T23:14:30.837116421Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"0","component":"datafile"}
{"msg":"[0] 18b01c87-5587-4e4c-af37-0e404569a230 Gone missing, transition from Disabled to Disconnected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.837141932Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] 18b01c87-5587-4e4c-af37-0e404569a230 connection to 127.0.0.1:62296 closed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.837164953Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"0","component":"datafile"}
{"msg":"[1] 18b01c87-5587-4e4c-af37-0e404569a230 (12531aa6-b93c-4bc2-9fe7-f954b135051a) cmd_loop saw YouAreNoLongerActive 18b01c87-5587-4e4c-af37-0e404569a230 e8b82b10-5353-499d-a44d-e0dd5d13af34 2","v":0,"name":"crucible-pantry","level":50,"time":"2023-09-22T23:14:30.83723374Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] 18b01c87-5587-4e4c-af37-0e404569a230 (12531aa6-b93c-4bc2-9fe7-f954b135051a) Disconnected Active Active ds_transition to Disabled","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.837265633Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] Transition from Active to Disabled","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.837288336Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"18b01c87-5587-4e4c-af37-0e404569a230 set inactive, session 12531aa6-b93c-4bc2-9fe7-f954b135051a","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.837326151Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"127.0.0.1:62299: proc: Generation number is too low: saw YouAreNoLongerActive with larger gen 2 than ours 1","v":0,"name":"crucible-pantry","level":50,"time":"2023-09-22T23:14:30.837354063Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"1","component":"datafile"}
{"msg":"[1] 18b01c87-5587-4e4c-af37-0e404569a230 Gone missing, transition from Disabled to Disconnected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.837379223Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] 18b01c87-5587-4e4c-af37-0e404569a230 connection to 127.0.0.1:62299 closed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.837402213Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"1","component":"datafile"}
{"msg":"[2] 18b01c87-5587-4e4c-af37-0e404569a230 (12531aa6-b93c-4bc2-9fe7-f954b135051a) cmd_loop saw YouAreNoLongerActive 18b01c87-5587-4e4c-af37-0e404569a230 e8b82b10-5353-499d-a44d-e0dd5d13af34 2","v":0,"name":"crucible-pantry","level":50,"time":"2023-09-22T23:14:30.83742909Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] 18b01c87-5587-4e4c-af37-0e404569a230 (12531aa6-b93c-4bc2-9fe7-f954b135051a) Disconnected Disconnected Active ds_transition to Disabled","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.837453717Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] Transition from Active to Disabled","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.837481088Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"18b01c87-5587-4e4c-af37-0e404569a230 set inactive, session 12531aa6-b93c-4bc2-9fe7-f954b135051a","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.837505263Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"127.0.0.1:44532: proc: Generation number is too low: saw YouAreNoLongerActive with larger gen 2 than ours 1","v":0,"name":"crucible-pantry","level":50,"time":"2023-09-22T23:14:30.837528829Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"2","component":"datafile"}
{"msg":"[2] 18b01c87-5587-4e4c-af37-0e404569a230 Gone missing, transition from Disabled to Disconnected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.837553199Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] 18b01c87-5587-4e4c-af37-0e404569a230 connection to 127.0.0.1:44532 closed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.83757603Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"2","component":"datafile"}
{"msg":"[0] 127.0.0.1:62296 task reports connection:false","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.837799288Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"18b01c87-5587-4e4c-af37-0e404569a230 Disconnected Disconnected Disconnected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.837829788Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] 127.0.0.1:62296 task reports offline","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.83785244Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] 127.0.0.1:62299 task reports connection:false","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.837874655Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"18b01c87-5587-4e4c-af37-0e404569a230 Disconnected Disconnected Disconnected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.837897104Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] 127.0.0.1:62299 task reports offline","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.837918595Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] 127.0.0.1:44532 task reports connection:false","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.837940262Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"18b01c87-5587-4e4c-af37-0e404569a230 Disconnected Disconnected Disconnected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.837962754Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] 127.0.0.1:44532 task reports offline","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:30.837984437Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] pm_task rx.recv() is None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:14:30.838005702Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] pm_task rx.recv() is None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:14:30.838027065Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] pm_task rx.recv() is None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:14:30.838048343Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:30.838 INFO [0] downstairs client at 127.0.0.1:62296 has UUID bec31c6c-c0ce-43db-8467-8d94b73527a2
Sep 22 23:14:30.838 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 512, shift: 9 }, extent_count: 188, uuid: bec31c6c-c0ce-43db-8467-8d94b73527a2, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:30.838 INFO 18b01c87-5587-4e4c-af37-0e404569a230 WaitActive WaitActive WaitActive
Sep 22 23:14:30.838 INFO [1] downstairs client at 127.0.0.1:62299 has UUID ede072e9-d3f2-4884-bb86-eea53898f9d9
Sep 22 23:14:30.838 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 512, shift: 9 }, extent_count: 188, uuid: ede072e9-d3f2-4884-bb86-eea53898f9d9, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:30.838 INFO 18b01c87-5587-4e4c-af37-0e404569a230 WaitActive WaitActive WaitActive
Sep 22 23:14:30.838 INFO [2] downstairs client at 127.0.0.1:44532 has UUID 6e66ae97-3dfc-40c6-a1d0-7d1c5297fdbf
Sep 22 23:14:30.838 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 512, shift: 9 }, extent_count: 188, uuid: 6e66ae97-3dfc-40c6-a1d0-7d1c5297fdbf, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:30.838 INFO 18b01c87-5587-4e4c-af37-0e404569a230 WaitActive WaitActive WaitActive
Sep 22 23:14:30.850 INFO Current flush_numbers [0..12]: [1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:14:30.852 INFO Current flush_numbers [0..12]: [1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:14:30.854 INFO Current flush_numbers [0..12]: [1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:14:30.858 DEBG Write     :1000 deps:[] res:true
Sep 22 23:14:30.861 DEBG Read      :1001 deps:[JobId(1000)] res:true
Sep 22 23:14:30.867 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:30.868 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:30.869 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:30.869 INFO [0] 18b01c87-5587-4e4c-af37-0e404569a230 (e8b82b10-5353-499d-a44d-e0dd5d13af34) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:30.869 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:14:30.869 WARN [0] new RM replaced this: None
Sep 22 23:14:30.869 INFO [0] Starts reconcile loop
Sep 22 23:14:30.869 INFO [1] 18b01c87-5587-4e4c-af37-0e404569a230 (e8b82b10-5353-499d-a44d-e0dd5d13af34) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:30.869 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:14:30.869 WARN [1] new RM replaced this: None
Sep 22 23:14:30.869 INFO [1] Starts reconcile loop
Sep 22 23:14:30.869 INFO [2] 18b01c87-5587-4e4c-af37-0e404569a230 (e8b82b10-5353-499d-a44d-e0dd5d13af34) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:14:30.870 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:14:30.870 WARN [2] new RM replaced this: None
Sep 22 23:14:30.870 INFO [2] Starts reconcile loop
Sep 22 23:14:30.870 INFO [0] 127.0.0.1:62296 task reports connection:true
Sep 22 23:14:30.870 INFO 18b01c87-5587-4e4c-af37-0e404569a230 WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:14:30.870 INFO [0]R flush_numbers[0..12]: [1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:14:30.870 INFO [0]R generation[0..12]: [1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:14:30.870 INFO [0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]
Sep 22 23:14:30.870 INFO [1]R flush_numbers[0..12]: [1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:14:30.870 INFO [1]R generation[0..12]: [1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:14:30.870 INFO [1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]
Sep 22 23:14:30.870 INFO [2]R flush_numbers[0..12]: [1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:14:30.870 INFO [2]R generation[0..12]: [1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:14:30.870 INFO [2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]
Sep 22 23:14:30.870 INFO Max found gen is 2
Sep 22 23:14:30.870 INFO Generation requested: 2 >= found:2
Sep 22 23:14:30.870 INFO Next flush: 2
Sep 22 23:14:30.870 INFO All extents match
Sep 22 23:14:30.870 INFO No downstairs repair required
Sep 22 23:14:30.870 INFO No initial repair work was required
Sep 22 23:14:30.870 INFO Set Downstairs and Upstairs active
Sep 22 23:14:30.870 INFO 18b01c87-5587-4e4c-af37-0e404569a230 is now active with session: e8b82b10-5353-499d-a44d-e0dd5d13af34
Sep 22 23:14:30.870 INFO 18b01c87-5587-4e4c-af37-0e404569a230 Set Active after no repair
Sep 22 23:14:30.870 INFO Notify all downstairs, region set compare is done.
Sep 22 23:14:30.870 INFO Set check for repair
Sep 22 23:14:30.870 INFO [1] 127.0.0.1:62299 task reports connection:true
Sep 22 23:14:30.870 INFO 18b01c87-5587-4e4c-af37-0e404569a230 Active Active Active
Sep 22 23:14:30.870 INFO Set check for repair
Sep 22 23:14:30.870 INFO [2] 127.0.0.1:44532 task reports connection:true
Sep 22 23:14:30.870 INFO 18b01c87-5587-4e4c-af37-0e404569a230 Active Active Active
Sep 22 23:14:30.870 INFO Set check for repair
Sep 22 23:14:30.870 INFO [0] received reconcile message
Sep 22 23:14:30.870 INFO [0] All repairs completed, exit
Sep 22 23:14:30.870 INFO [0] Starts cmd_loop
Sep 22 23:14:30.870 INFO [1] received reconcile message
Sep 22 23:14:30.870 INFO [1] All repairs completed, exit
Sep 22 23:14:30.870 INFO [1] Starts cmd_loop
Sep 22 23:14:30.870 INFO [2] received reconcile message
Sep 22 23:14:30.870 INFO [2] All repairs completed, exit
Sep 22 23:14:30.870 INFO [2] Starts cmd_loop
The guest has finished waiting for activation
Sep 22 23:14:30.872 DEBG IO Read  1000 has deps []
Sep 22 23:14:30.882 DEBG Read      :1000 deps:[] res:true
Sep 22 23:14:30.882 DEBG Flush     :1002 extent_limit None deps:[JobId(1001), JobId(1000)] res:true f:1 g:1
Sep 22 23:14:30.882 DEBG Flush     :1002 extent_limit None deps:[JobId(1001), JobId(1000)] res:true f:1 g:1
Sep 22 23:14:30.882 DEBG Flush     :1002 extent_limit None deps:[JobId(1001), JobId(1000)] res:true f:1 g:1
Sep 22 23:14:30.896 DEBG Read      :1000 deps:[] res:true
Sep 22 23:14:30.910 DEBG Read      :1000 deps:[] res:true
Sep 22 23:14:31.172 DEBG [0] Read AckReady 1000, : downstairs
Sep 22 23:14:31.337 DEBG [1] Read already AckReady 1000, : downstairs
Sep 22 23:14:31.503 DEBG [2] Read already AckReady 1000, : downstairs
Sep 22 23:14:31.504 DEBG up_ds_listen was notified
Sep 22 23:14:31.504 DEBG up_ds_listen process 1000
Sep 22 23:14:31.504 DEBG [A] ack job 1000:1, : downstairs
{"msg":"Checking if live repair is needed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.5071637Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"No Live Repair required at this time","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.507236007Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.513389986Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/a9fc305b-73b1-411e-9d00-c6edbf74297f/bulk_read","method":"POST","req_id":"7a8495a2-592a-48de-9223-77f8950ceb7e","remote_addr":"127.0.0.1:41882","local_addr":"127.0.0.1:45536","component":"dropshot","latency_us":855720,"response_code":"200"}
{"msg":"Checking if live repair is needed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.513671667Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"No Live Repair required at this time","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.51370539Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Checking if live repair is needed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.523855126Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"No Live Repair required at this time","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.523939355Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"detach removing entry for volume a9fc305b-73b1-411e-9d00-c6edbf74297f","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.529931137Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"detaching volume a9fc305b-73b1-411e-9d00-c6edbf74297f","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.529991215Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:31.530 DEBG Flush     :1003 extent_limit None deps:[] res:true f:2 g:1
Sep 22 23:14:31.530 DEBG Flush     :1003 extent_limit None deps:[] res:true f:2 g:1
Sep 22 23:14:31.530 DEBG Flush     :1003 extent_limit None deps:[] res:true f:2 g:1
{"msg":"Request to deactivate this guest","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.531162144Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"d37ce371-0ed6-44c3-bb8f-e1d703ba2c92 set deactivating.","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.531207544Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.531383055Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/a9fc305b-73b1-411e-9d00-c6edbf74297f","method":"DELETE","req_id":"9498cace-6405-475e-88bd-9f70ef9db15f","remote_addr":"127.0.0.1:41882","local_addr":"127.0.0.1:45536","component":"dropshot","latency_us":1624,"response_code":"204"}
Sep 22 23:14:31.531 DEBG up_ds_listen checked 1 jobs, back to waiting
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.625608408Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/job/a52bd47d-76c9-4928-9712-218eb57bfeb7/is_finished","method":"GET","req_id":"f86ce50d-aab8-41b6-8c6c-ca52ece334e5","remote_addr":"127.0.0.1:49478","local_addr":"127.0.0.1:33566","component":"dropshot","latency_us":301,"response_code":"200"}
Sep 22 23:14:31.637 WARN a50fc3fb-3de2-4743-9c50-cc80cfba77db request to replace downstairs 127.0.0.1:42970 with 127.0.0.1:62845
Sep 22 23:14:31.637 INFO a50fc3fb-3de2-4743-9c50-cc80cfba77db found new target: 127.0.0.1:62845 at 0
Sep 22 23:14:31.637 INFO Downstairs replacement completed
Sep 22 23:14:31.637 DEBG IO Read  1011 has deps []
Sep 22 23:14:31.639 DEBG Read      :1011 deps:[] res:true
Sep 22 23:14:31.640 DEBG Read      :1011 deps:[] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.640599513Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/job/de4c6089-f970-4d99-9581-6bd61244a7f8/is_finished","method":"GET","req_id":"d67ef902-5b0d-4fde-b094-2d028ae346fd","remote_addr":"127.0.0.1:58570","local_addr":"127.0.0.1:61933","component":"dropshot","latency_us":232,"response_code":"200"}
Sep 22 23:14:31.640 DEBG Read      :1011 deps:[] res:true
Sep 22 23:14:31.645 DEBG [1] Read AckReady 1011, : downstairs
Sep 22 23:14:31.647 DEBG [2] Read already AckReady 1011, : downstairs
Sep 22 23:14:31.650 DEBG [0] Read already AckReady 1011, : downstairs
Sep 22 23:14:31.650 DEBG up_ds_listen was notified
Sep 22 23:14:31.650 DEBG up_ds_listen process 1011
Sep 22 23:14:31.650 DEBG [A] ack job 1011:12, : downstairs
Sep 22 23:14:31.650 DEBG up_ds_listen checked 1 jobs, back to waiting
test test::integration_test_guest_replace_downstairs ... ok
test test::test_pantry_bulk_read_max_chunk_size ... ok
Sep 22 23:14:31.712 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:31.712 INFO Created new region file "/tmp/downstairs-don5KOuC/region.json"
Sep 22 23:14:31.718 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:31.718 INFO Opened existing region file "/tmp/downstairs-don5KOuC/region.json"
Sep 22 23:14:31.718 INFO Database read version 1
Sep 22 23:14:31.718 INFO Database write version 1
Sep 22 23:14:31.719 INFO UUID: 38f16af0-b890-4595-9750-0a66b8b08b29
Sep 22 23:14:31.719 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:31.719 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:31.719 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:31.719 INFO Using address: 127.0.0.1:56365, task: main
Sep 22 23:14:31.719 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:31.720 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:51524, task: repair
Sep 22 23:14:31.720 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:51524, task: repair
Sep 22 23:14:31.720 INFO listening, local_addr: 127.0.0.1:51524, task: repair
Sep 22 23:14:31.720 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:51524, task: repair
Sep 22 23:14:31.720 INFO Using repair address: 127.0.0.1:51524, task: main
Sep 22 23:14:31.720 INFO No SSL acceptor configured, task: main
Sep 22 23:14:31.720 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:31.720 INFO Created new region file "/tmp/downstairs-jd0MhT1q/region.json"
test test::test_pantry_bulk_write_max_chunk_size ... ok
Sep 22 23:14:31.722 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:31.722 INFO Created new region file "/tmp/downstairs-S24QnP1S/region.json"
Sep 22 23:14:31.723 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:31.723 INFO Opened existing region file "/tmp/downstairs-jd0MhT1q/region.json"
Sep 22 23:14:31.723 INFO Database read version 1
Sep 22 23:14:31.723 INFO Database write version 1
Sep 22 23:14:31.725 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:31.725 INFO Opened existing region file "/tmp/downstairs-S24QnP1S/region.json"
Sep 22 23:14:31.725 INFO Database read version 1
Sep 22 23:14:31.725 INFO Database write version 1
Sep 22 23:14:31.725 INFO UUID: cc08515f-5167-4ce8-af58-a1c8873a38fe
Sep 22 23:14:31.725 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:31.725 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:31.725 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:31.725 INFO Using address: 127.0.0.1:43819, task: main
Sep 22 23:14:31.725 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:31.725 INFO UUID: 1535f28b-0d0c-40c8-8224-3bcf2b9ad4d7
Sep 22 23:14:31.725 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:31.725 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:36231, task: repair
Sep 22 23:14:31.725 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:36231, task: repair
Sep 22 23:14:31.725 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:31.725 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:31.725 INFO Using address: 127.0.0.1:45844, task: main
Sep 22 23:14:31.725 INFO listening, local_addr: 127.0.0.1:36231, task: repair
Sep 22 23:14:31.726 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:36231, task: repair
Sep 22 23:14:31.726 INFO Using repair address: 127.0.0.1:36231, task: main
Sep 22 23:14:31.726 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:31.726 INFO No SSL acceptor configured, task: main
Sep 22 23:14:31.726 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:40664, task: repair
Sep 22 23:14:31.726 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:40664, task: repair
Sep 22 23:14:31.726 INFO listening, local_addr: 127.0.0.1:40664, task: repair
Sep 22 23:14:31.726 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:31.726 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:40664, task: repair
Sep 22 23:14:31.726 INFO Created new region file "/tmp/downstairs-VwMKzFw9/region.json"
Sep 22 23:14:31.726 INFO Using repair address: 127.0.0.1:40664, task: main
Sep 22 23:14:31.726 INFO No SSL acceptor configured, task: main
Sep 22 23:14:31.726 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:31.726 INFO Created new region file "/tmp/downstairs-eC0rwx3k/region.json"
Sep 22 23:14:31.729 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:31.729 INFO Opened existing region file "/tmp/downstairs-eC0rwx3k/region.json"
Sep 22 23:14:31.729 INFO Database read version 1
Sep 22 23:14:31.729 INFO Database write version 1
Sep 22 23:14:31.730 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:31.730 INFO Opened existing region file "/tmp/downstairs-VwMKzFw9/region.json"
Sep 22 23:14:31.730 INFO Database read version 1
Sep 22 23:14:31.730 INFO Database write version 1
Sep 22 23:14:31.730 INFO UUID: 77eb89cb-93f6-46c5-be2e-fd5390eb0b19
Sep 22 23:14:31.730 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:31.730 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:31.730 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:31.730 INFO Using address: 127.0.0.1:34614, task: main
Sep 22 23:14:31.730 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:31.730 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:37689, task: repair
Sep 22 23:14:31.730 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:37689, task: repair
Sep 22 23:14:31.730 INFO listening, local_addr: 127.0.0.1:37689, task: repair
Sep 22 23:14:31.730 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:37689, task: repair
Sep 22 23:14:31.730 INFO Using repair address: 127.0.0.1:37689, task: main
Sep 22 23:14:31.730 INFO No SSL acceptor configured, task: main
Sep 22 23:14:31.730 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:31.730 INFO UUID: 420155b0-a798-48b0-aa4b-1710ab0f9b49
Sep 22 23:14:31.730 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:31.730 INFO Created new region file "/tmp/downstairs-CCCKCF6f/region.json"
Sep 22 23:14:31.731 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:31.731 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:31.731 INFO Using address: 127.0.0.1:64951, task: main
Sep 22 23:14:31.731 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:31.731 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:50389, task: repair
Sep 22 23:14:31.731 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:50389, task: repair
Sep 22 23:14:31.731 INFO listening, local_addr: 127.0.0.1:50389, task: repair
Sep 22 23:14:31.731 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:50389, task: repair
Sep 22 23:14:31.731 INFO Using repair address: 127.0.0.1:50389, task: main
Sep 22 23:14:31.731 INFO No SSL acceptor configured, task: main
note: configured to log to "/dev/stdout"
Sep 22 23:14:31.733 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:31.733 INFO Opened existing region file "/tmp/downstairs-CCCKCF6f/region.json"
Sep 22 23:14:31.733 INFO Database read version 1
Sep 22 23:14:31.733 INFO Database write version 1
{"msg":"listening","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.73358451Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"local_addr":"127.0.0.1:51521","component":"dropshot"}
Sep 22 23:14:31.733 INFO UUID: 4c7c4d38-93a3-4971-b5c4-80ee60a119d2
Sep 22 23:14:31.733 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:31.733 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:31.733 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:31.733 INFO Using address: 127.0.0.1:49185, task: main
{"msg":"listen IP: 127.0.0.1:51521","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.733790232Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
Sep 22 23:14:31.734 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:31.734 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:39643, task: repair
Sep 22 23:14:31.734 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:39643, task: repair
Sep 22 23:14:31.734 INFO listening, local_addr: 127.0.0.1:39643, task: repair
Sep 22 23:14:31.734 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:39643, task: repair
Sep 22 23:14:31.734 INFO Using repair address: 127.0.0.1:39643, task: main
Sep 22 23:14:31.734 INFO No SSL acceptor configured, task: main
note: configured to log to "/dev/stdout"
{"msg":"listening","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.735577184Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"local_addr":"127.0.0.1:51522","component":"dropshot"}
{"msg":"listen IP: 127.0.0.1:51522","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.735827016Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
Sep 22 23:14:31.827 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:31.827 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:31.827 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:31.827 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:31.827 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:31.827 INFO listening on 127.0.0.1:0, task: main
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.82760124Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"local_addr":"127.0.0.1:51522","component":"dropshot","remote_addr":"127.0.0.1:43032"}
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.827680923Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"local_addr":"127.0.0.1:51521","component":"dropshot","remote_addr":"127.0.0.1:60724"}
{"msg":"no entry exists for volume b9c1e9d5-7588-4cfd-ba21-899d3f748374, constructing...","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.828211504Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"no entry exists for volume 79c71868-5af1-4b01-8453-182bf4ff9124, constructing...","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.828217593Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Upstairs starts","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.828366849Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Upstairs starts","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.828425672Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Crucible Version: BuildInfo {\n    version: \"0.0.1\",\n    git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n    git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n    git_branch: \"main\",\n    rustc_semver: \"1.70.0\",\n    rustc_channel: \"stable\",\n    rustc_host_triple: \"x86_64-unknown-illumos\",\n    rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n    cargo_triple: \"x86_64-unknown-illumos\",\n    debug: true,\n    opt_level: 0,\n}","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.82845375Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Crucible Version: BuildInfo {\n    version: \"0.0.1\",\n    git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n    git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n    git_branch: \"main\",\n    rustc_semver: \"1.70.0\",\n    rustc_channel: \"stable\",\n    rustc_host_triple: \"x86_64-unknown-illumos\",\n    rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n    cargo_triple: \"x86_64-unknown-illumos\",\n    debug: true,\n    opt_level: 0,\n}","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.828505204Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.828504389Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.828546849Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Crucible stats registered with UUID: 7fa1ceb3-2853-4b6f-aa54-bc7d3f1bd8a3","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.828674752Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Crucible 7fa1ceb3-2853-4b6f-aa54-bc7d3f1bd8a3 has session id: 7773880a-bf4e-4e9d-af22-d332b059734c","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.828722117Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] connecting to 127.0.0.1:56365","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.828758943Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"0","component":"datafile"}
{"msg":"Crucible stats registered with UUID: 451049c0-e2b7-41c3-872a-e5be40bc5510","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.828552713Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Crucible 451049c0-e2b7-41c3-872a-e5be40bc5510 has session id: da82b289-0e3b-42f1-837a-a883c1bd3768","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.828875139Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] connecting to 127.0.0.1:45844","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.828918167Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"0","component":"datafile"}
{"msg":"[1] connecting to 127.0.0.1:34614","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.828955384Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"1","component":"datafile"}
{"msg":"[2] connecting to 127.0.0.1:49185","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.828982352Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"2","component":"datafile"}
{"msg":"[1] connecting to 127.0.0.1:43819","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.82896695Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"1","component":"datafile"}
{"msg":"up_listen starts","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.829014306Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile","task":"up_listen"}
{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.829040358Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Flush timeout: 0.5","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.829062147Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] connecting to 127.0.0.1:64951","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.829158269Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"2","component":"datafile"}
{"msg":"up_listen starts","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.829231233Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile","task":"up_listen"}
{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.829275478Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"volume 79c71868-5af1-4b01-8453-182bf4ff9124 constructed ok","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.829269703Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Flush timeout: 0.5","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.829341989Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
The guest has requested activation
{"msg":"volume b9c1e9d5-7588-4cfd-ba21-899d3f748374 constructed ok","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.829498188Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"451049c0-e2b7-41c3-872a-e5be40bc5510 active request set","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.829537617Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:31.829 INFO accepted connection from 127.0.0.1:47008, task: main
The guest has requested activation
Sep 22 23:14:31.829 INFO accepted connection from 127.0.0.1:52904, task: main
{"msg":"7fa1ceb3-2853-4b6f-aa54-bc7d3f1bd8a3 active request set","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.829786634Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:31.829 INFO accepted connection from 127.0.0.1:60505, task: main
Sep 22 23:14:31.829 INFO accepted connection from 127.0.0.1:48109, task: main
{"msg":"[0] 451049c0-e2b7-41c3-872a-e5be40bc5510 looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.829929453Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"0","component":"datafile"}
{"msg":"[0] Proc runs for 127.0.0.1:45844 in state New","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.829961409Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:31.829 INFO accepted connection from 127.0.0.1:63156, task: main
Sep 22 23:14:31.830 INFO accepted connection from 127.0.0.1:54276, task: main
{"msg":"[1] 451049c0-e2b7-41c3-872a-e5be40bc5510 looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.830073249Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"1","component":"datafile"}
{"msg":"[1] Proc runs for 127.0.0.1:34614 in state New","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.830103911Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] 451049c0-e2b7-41c3-872a-e5be40bc5510 looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.830148228Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"2","component":"datafile"}
{"msg":"[0] 7fa1ceb3-2853-4b6f-aa54-bc7d3f1bd8a3 looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.830158705Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"0","component":"datafile"}
{"msg":"[2] Proc runs for 127.0.0.1:49185 in state New","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.830189255Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] Proc runs for 127.0.0.1:56365 in state New","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.830198599Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] 7fa1ceb3-2853-4b6f-aa54-bc7d3f1bd8a3 looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.830232704Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"1","component":"datafile"}
{"msg":"[1] Proc runs for 127.0.0.1:43819 in state New","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.830266281Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] 7fa1ceb3-2853-4b6f-aa54-bc7d3f1bd8a3 looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.830290376Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"2","component":"datafile"}
{"msg":"[2] Proc runs for 127.0.0.1:64951 in state New","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.830313516Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:31.830 INFO Connection request from 451049c0-e2b7-41c3-872a-e5be40bc5510 with version 4, task: proc
Sep 22 23:14:31.830 INFO upstairs UpstairsConnection { upstairs_id: 451049c0-e2b7-41c3-872a-e5be40bc5510, session_id: 193f0536-0e45-44aa-a329-53c0a3eabf73, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:31.830 INFO Connection request from 7fa1ceb3-2853-4b6f-aa54-bc7d3f1bd8a3 with version 4, task: proc
Sep 22 23:14:31.830 INFO Connection request from 451049c0-e2b7-41c3-872a-e5be40bc5510 with version 4, task: proc
Sep 22 23:14:31.830 INFO upstairs UpstairsConnection { upstairs_id: 7fa1ceb3-2853-4b6f-aa54-bc7d3f1bd8a3, session_id: a5beedbf-47b7-46b1-9353-7d08dc5cf76e, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:31.830 INFO upstairs UpstairsConnection { upstairs_id: 451049c0-e2b7-41c3-872a-e5be40bc5510, session_id: 193f0536-0e45-44aa-a329-53c0a3eabf73, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:31.830 INFO Connection request from 451049c0-e2b7-41c3-872a-e5be40bc5510 with version 4, task: proc
Sep 22 23:14:31.830 INFO Connection request from 7fa1ceb3-2853-4b6f-aa54-bc7d3f1bd8a3 with version 4, task: proc
Sep 22 23:14:31.830 INFO upstairs UpstairsConnection { upstairs_id: 7fa1ceb3-2853-4b6f-aa54-bc7d3f1bd8a3, session_id: a5beedbf-47b7-46b1-9353-7d08dc5cf76e, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:31.830 INFO upstairs UpstairsConnection { upstairs_id: 451049c0-e2b7-41c3-872a-e5be40bc5510, session_id: 193f0536-0e45-44aa-a329-53c0a3eabf73, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:31.830 INFO Connection request from 7fa1ceb3-2853-4b6f-aa54-bc7d3f1bd8a3 with version 4, task: proc
Sep 22 23:14:31.830 INFO upstairs UpstairsConnection { upstairs_id: 7fa1ceb3-2853-4b6f-aa54-bc7d3f1bd8a3, session_id: a5beedbf-47b7-46b1-9353-7d08dc5cf76e, gen: 1 } connected, version 4, task: proc
{"msg":"[0] 451049c0-e2b7-41c3-872a-e5be40bc5510 (193f0536-0e45-44aa-a329-53c0a3eabf73) New New New ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.830807749Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.830843014Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] client is_active_req TRUE, promote! session 193f0536-0e45-44aa-a329-53c0a3eabf73","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.830867862Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] 451049c0-e2b7-41c3-872a-e5be40bc5510 (193f0536-0e45-44aa-a329-53c0a3eabf73) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.830892191Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.830927677Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] 7fa1ceb3-2853-4b6f-aa54-bc7d3f1bd8a3 (a5beedbf-47b7-46b1-9353-7d08dc5cf76e) New New New ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.830932593Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] client is_active_req TRUE, promote! session 193f0536-0e45-44aa-a329-53c0a3eabf73","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.830966546Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.830970708Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] client is_active_req TRUE, promote! session a5beedbf-47b7-46b1-9353-7d08dc5cf76e","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.831006382Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] 451049c0-e2b7-41c3-872a-e5be40bc5510 (193f0536-0e45-44aa-a329-53c0a3eabf73) WaitActive WaitActive New ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.831006386Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] 7fa1ceb3-2853-4b6f-aa54-bc7d3f1bd8a3 (a5beedbf-47b7-46b1-9353-7d08dc5cf76e) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.831068798Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.831095785Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:31.831 INFO UpstairsConnection { upstairs_id: 451049c0-e2b7-41c3-872a-e5be40bc5510, session_id: 193f0536-0e45-44aa-a329-53c0a3eabf73, gen: 1 } is now active (read-write)
{"msg":"[1] client is_active_req TRUE, promote! session a5beedbf-47b7-46b1-9353-7d08dc5cf76e","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.831119376Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.831099193Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] 7fa1ceb3-2853-4b6f-aa54-bc7d3f1bd8a3 (a5beedbf-47b7-46b1-9353-7d08dc5cf76e) WaitActive WaitActive New ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.831143997Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.83116619Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] client is_active_req TRUE, promote! session 193f0536-0e45-44aa-a329-53c0a3eabf73","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.831155595Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] client is_active_req TRUE, promote! session a5beedbf-47b7-46b1-9353-7d08dc5cf76e","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.831222475Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:31.831 INFO UpstairsConnection { upstairs_id: 7fa1ceb3-2853-4b6f-aa54-bc7d3f1bd8a3, session_id: a5beedbf-47b7-46b1-9353-7d08dc5cf76e, gen: 1 } is now active (read-write)
Sep 22 23:14:31.831 INFO UpstairsConnection { upstairs_id: 451049c0-e2b7-41c3-872a-e5be40bc5510, session_id: 193f0536-0e45-44aa-a329-53c0a3eabf73, gen: 1 } is now active (read-write)
Sep 22 23:14:31.831 INFO UpstairsConnection { upstairs_id: 451049c0-e2b7-41c3-872a-e5be40bc5510, session_id: 193f0536-0e45-44aa-a329-53c0a3eabf73, gen: 1 } is now active (read-write)
Sep 22 23:14:31.831 INFO UpstairsConnection { upstairs_id: 7fa1ceb3-2853-4b6f-aa54-bc7d3f1bd8a3, session_id: a5beedbf-47b7-46b1-9353-7d08dc5cf76e, gen: 1 } is now active (read-write)
Sep 22 23:14:31.831 INFO UpstairsConnection { upstairs_id: 7fa1ceb3-2853-4b6f-aa54-bc7d3f1bd8a3, session_id: a5beedbf-47b7-46b1-9353-7d08dc5cf76e, gen: 1 } is now active (read-write)
{"msg":"[0] downstairs client at 127.0.0.1:45844 has UUID 1535f28b-0d0c-40c8-8224-3bcf2b9ad4d7","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.832021844Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 1535f28b-0d0c-40c8-8224-3bcf2b9ad4d7, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.832058442Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"451049c0-e2b7-41c3-872a-e5be40bc5510 WaitActive WaitActive WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.832082202Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] downstairs client at 127.0.0.1:34614 has UUID 77eb89cb-93f6-46c5-be2e-fd5390eb0b19","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.832123783Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] downstairs client at 127.0.0.1:56365 has UUID 38f16af0-b890-4595-9750-0a66b8b08b29","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.832121639Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 77eb89cb-93f6-46c5-be2e-fd5390eb0b19, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.832202704Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 38f16af0-b890-4595-9750-0a66b8b08b29, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.832211602Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"451049c0-e2b7-41c3-872a-e5be40bc5510 WaitActive WaitActive WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.832243425Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"7fa1ceb3-2853-4b6f-aa54-bc7d3f1bd8a3 WaitActive WaitActive WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.832248021Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] downstairs client at 127.0.0.1:49185 has UUID 4c7c4d38-93a3-4971-b5c4-80ee60a119d2","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.832315173Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] downstairs client at 127.0.0.1:43819 has UUID cc08515f-5167-4ce8-af58-a1c8873a38fe","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.832323526Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 4c7c4d38-93a3-4971-b5c4-80ee60a119d2, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.832362685Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: cc08515f-5167-4ce8-af58-a1c8873a38fe, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.83236676Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"451049c0-e2b7-41c3-872a-e5be40bc5510 WaitActive WaitActive WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.832399873Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"7fa1ceb3-2853-4b6f-aa54-bc7d3f1bd8a3 WaitActive WaitActive WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.832400751Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:31.832 INFO Current flush_numbers [0..12]: [0, 0]
{"msg":"[2] downstairs client at 127.0.0.1:64951 has UUID 420155b0-a798-48b0-aa4b-1710ab0f9b49","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.832512796Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:31.832 INFO Current flush_numbers [0..12]: [0, 0]
{"msg":"[2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 420155b0-a798-48b0-aa4b-1710ab0f9b49, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.832546653Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"7fa1ceb3-2853-4b6f-aa54-bc7d3f1bd8a3 WaitActive WaitActive WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.832570318Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:31.832 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:31.832 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:31.832 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:31.832 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:31.833 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:31.833 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:31.833 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:31.833 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:31.833 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:31.833 INFO Downstairs has completed Negotiation, task: proc
{"msg":"[0] 451049c0-e2b7-41c3-872a-e5be40bc5510 (193f0536-0e45-44aa-a329-53c0a3eabf73) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.833720338Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] 7fa1ceb3-2853-4b6f-aa54-bc7d3f1bd8a3 (a5beedbf-47b7-46b1-9353-7d08dc5cf76e) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.833724576Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.833798565Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.833811541Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:14:31.833839063Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:14:31.833847073Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.833873428Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.833879121Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] 451049c0-e2b7-41c3-872a-e5be40bc5510 (193f0536-0e45-44aa-a329-53c0a3eabf73) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.83391264Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] 7fa1ceb3-2853-4b6f-aa54-bc7d3f1bd8a3 (a5beedbf-47b7-46b1-9353-7d08dc5cf76e) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.833916125Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.833948568Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.833948005Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:14:31.834014316Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:14:31.834027256Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.834052328Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.834062731Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] 7fa1ceb3-2853-4b6f-aa54-bc7d3f1bd8a3 (a5beedbf-47b7-46b1-9353-7d08dc5cf76e) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.834091278Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] 451049c0-e2b7-41c3-872a-e5be40bc5510 (193f0536-0e45-44aa-a329-53c0a3eabf73) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.834100209Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
The guest has finished waiting for activation
The guest has finished waiting for activation
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.834125466Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.834139558Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:14:31.834160272Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:14:31.834172756Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.834193791Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.834205354Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] 127.0.0.1:56365 task reports connection:true","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.834227639Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] 127.0.0.1:45844 task reports connection:true","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.834238726Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"7fa1ceb3-2853-4b6f-aa54-bc7d3f1bd8a3 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.834262904Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"451049c0-e2b7-41c3-872a-e5be40bc5510 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.834272418Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0]R  flush_numbers: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.834296237Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0]R  flush_numbers: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.83430447Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0]R  generation: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.834329556Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0]R  generation: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.834336051Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0]R  dirty: [false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.83436327Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0]R  dirty: [false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.834367718Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1]R  flush_numbers: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.834396824Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1]R  flush_numbers: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.834399221Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1]R  generation: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.834459784Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1]R  generation: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.834506964Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1]R  dirty: [false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.834508827Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1]R  dirty: [false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.834577307Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2]R  flush_numbers: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.834584738Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2]R  flush_numbers: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.834616083Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2]R  generation: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.834618528Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2]R  generation: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.834650596Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2]R  dirty: [false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.834650925Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2]R  dirty: [false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.834713778Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Max found gen is 1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.834794685Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Max found gen is 1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.834802468Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.834837407Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.834837482Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Next flush: 1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.834903322Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"All extents match","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.834938411Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Next flush: 1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.834933317Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"No downstairs repair required","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.834999863Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"All extents match","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.835009845Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"No initial repair work was required","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.835037753Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"No downstairs repair required","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.835050597Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.835072807Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"No initial repair work was required","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.835084248Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"451049c0-e2b7-41c3-872a-e5be40bc5510 is now active with session: 193f0536-0e45-44aa-a329-53c0a3eabf73","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.835109671Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.835117099Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"451049c0-e2b7-41c3-872a-e5be40bc5510 Set Active after no repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.835145044Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"7fa1ceb3-2853-4b6f-aa54-bc7d3f1bd8a3 is now active with session: a5beedbf-47b7-46b1-9353-7d08dc5cf76e","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.83515219Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.835179404Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"7fa1ceb3-2853-4b6f-aa54-bc7d3f1bd8a3 Set Active after no repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.835185813Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Set check for repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.835212726Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.83521873Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] 127.0.0.1:34614 task reports connection:true","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.835247634Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Set check for repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.835250359Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"451049c0-e2b7-41c3-872a-e5be40bc5510 Active Active Active","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.835283233Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] 127.0.0.1:43819 task reports connection:true","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.835283548Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Set check for repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.835346593Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"7fa1ceb3-2853-4b6f-aa54-bc7d3f1bd8a3 Active Active Active","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.83535983Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] 127.0.0.1:49185 task reports connection:true","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.835385241Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Set check for repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.835400598Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"451049c0-e2b7-41c3-872a-e5be40bc5510 Active Active Active","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.835421441Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] 127.0.0.1:64951 task reports connection:true","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.835435169Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Set check for repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.83545477Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"7fa1ceb3-2853-4b6f-aa54-bc7d3f1bd8a3 Active Active Active","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.835469672Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] received reconcile message","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.835488229Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Set check for repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.835502424Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.835521549Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] received reconcile message","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.835535813Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.835554631Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.835569108Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] received reconcile message","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.835588155Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.835601675Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.835621609Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] received reconcile message","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.835634681Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.835654564Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.835667724Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] received reconcile message","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.835687847Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.835726746Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.835700465Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.83576404Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] received reconcile message","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.835798084Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"volume 79c71868-5af1-4b01-8453-182bf4ff9124 activated ok","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.835798784Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.835863598Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"volume 79c71868-5af1-4b01-8453-182bf4ff9124 constructed and inserted ok","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.835898105Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.835899105Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"volume b9c1e9d5-7588-4cfd-ba21-899d3f748374 activated ok","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.835990681Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.835966074Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/79c71868-5af1-4b01-8453-182bf4ff9124","method":"POST","req_id":"a3141e3f-e0f9-4c25-8b21-d6e793fc27b4","remote_addr":"127.0.0.1:43032","local_addr":"127.0.0.1:51522","component":"dropshot","latency_us":6635,"response_code":"200"}
{"msg":"volume b9c1e9d5-7588-4cfd-ba21-899d3f748374 constructed and inserted ok","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.836030073Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.836065675Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/b9c1e9d5-7588-4cfd-ba21-899d3f748374","method":"POST","req_id":"d8376324-72a1-4e60-9b98-73d722e6a2fb","remote_addr":"127.0.0.1:60724","local_addr":"127.0.0.1:51521","component":"dropshot","latency_us":6599,"response_code":"200"}
{"msg":"flush with snap requested","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.836108517Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:31.836 ERRO Snapshot request received on unsupported binary
Sep 22 23:14:31.836 DEBG Flush     :1000 extent_limit None deps:[] res:true f:1 g:1
Sep 22 23:14:31.836 ERRO Snapshot request received on unsupported binary
Sep 22 23:14:31.836 DEBG Flush     :1000 extent_limit None deps:[] res:true f:1 g:1
Sep 22 23:14:31.836 ERRO Snapshot request received on unsupported binary
Sep 22 23:14:31.836 DEBG Flush     :1000 extent_limit None deps:[] res:true f:1 g:1
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.837152955Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/b9c1e9d5-7588-4cfd-ba21-899d3f748374/snapshot","method":"POST","req_id":"c5106f12-9782-40fd-b3d0-52c334dff322","remote_addr":"127.0.0.1:60724","local_addr":"127.0.0.1:51521","component":"dropshot","latency_us":1846,"response_code":"204"}
{"msg":"detach removing entry for volume b9c1e9d5-7588-4cfd-ba21-899d3f748374","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.837893203Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"detaching volume b9c1e9d5-7588-4cfd-ba21-899d3f748374","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.837926096Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:31.838 DEBG Flush     :1001 extent_limit None deps:[] res:true f:2 g:1
Sep 22 23:14:31.838 DEBG Flush     :1001 extent_limit None deps:[] res:true f:2 g:1
Sep 22 23:14:31.838 DEBG Flush     :1001 extent_limit None deps:[] res:true f:2 g:1
{"msg":"Request to deactivate this guest","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.839389941Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"7fa1ceb3-2853-4b6f-aa54-bc7d3f1bd8a3 set deactivating.","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.839424717Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.839678192Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/b9c1e9d5-7588-4cfd-ba21-899d3f748374","method":"DELETE","req_id":"e4821359-efff-4694-8eb5-9a0c244e389d","remote_addr":"127.0.0.1:60724","local_addr":"127.0.0.1:51521","component":"dropshot","latency_us":1938,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.840066162Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"local_addr":"127.0.0.1:51522","component":"dropshot","remote_addr":"127.0.0.1:49688"}
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.840104059Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"local_addr":"127.0.0.1:51522","component":"dropshot","remote_addr":"127.0.0.1:45421"}
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.840131917Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"local_addr":"127.0.0.1:51522","component":"dropshot","remote_addr":"127.0.0.1:51046"}
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.840161237Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"local_addr":"127.0.0.1:51522","component":"dropshot","remote_addr":"127.0.0.1:60061"}
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.840187797Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"local_addr":"127.0.0.1:51522","component":"dropshot","remote_addr":"127.0.0.1:54313"}
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.840216523Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"local_addr":"127.0.0.1:51522","component":"dropshot","remote_addr":"127.0.0.1:37651"}
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.840242972Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"local_addr":"127.0.0.1:51522","component":"dropshot","remote_addr":"127.0.0.1:46952"}
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.840270479Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"local_addr":"127.0.0.1:51522","component":"dropshot","remote_addr":"127.0.0.1:58452"}
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.840297335Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"local_addr":"127.0.0.1:51522","component":"dropshot","remote_addr":"127.0.0.1:41946"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.842863888Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/79c71868-5af1-4b01-8453-182bf4ff9124/bulk_write","method":"POST","req_id":"a1900ba1-380b-41cd-9073-b78f2f897184","remote_addr":"127.0.0.1:43032","local_addr":"127.0.0.1:51522","component":"dropshot","latency_us":3508,"response_code":"204"}
test test::test_pantry_snapshot ... ok
Sep 22 23:14:31.845 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:31.845 INFO Created new region file "/tmp/downstairs-kXOwbSjj/region.json"
Sep 22 23:14:31.846 DEBG Write     :1000 deps:[] res:true
Sep 22 23:14:31.846 DEBG Write     :1000 deps:[] res:true
Sep 22 23:14:31.847 DEBG Write     :1000 deps:[] res:true
Sep 22 23:14:31.847 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:31.847 INFO Opened existing region file "/tmp/downstairs-kXOwbSjj/region.json"
Sep 22 23:14:31.847 INFO Database read version 1
Sep 22 23:14:31.847 INFO Database write version 1
Sep 22 23:14:31.848 INFO UUID: 0023ec13-80c8-426d-b3f0-fea7668a3f39
Sep 22 23:14:31.848 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:31.848 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:31.848 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:31.848 INFO Using address: 127.0.0.1:45781, task: main
Sep 22 23:14:31.848 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:31.848 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:64829, task: repair
Sep 22 23:14:31.848 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:64829, task: repair
Sep 22 23:14:31.848 INFO listening, local_addr: 127.0.0.1:64829, task: repair
Sep 22 23:14:31.848 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:64829, task: repair
Sep 22 23:14:31.848 INFO Using repair address: 127.0.0.1:64829, task: main
Sep 22 23:14:31.848 INFO No SSL acceptor configured, task: main
Sep 22 23:14:31.848 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:31.849 INFO Created new region file "/tmp/downstairs-8upV5Efn/region.json"
Sep 22 23:14:31.851 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:31.851 INFO Opened existing region file "/tmp/downstairs-8upV5Efn/region.json"
Sep 22 23:14:31.851 INFO Database read version 1
Sep 22 23:14:31.851 INFO Database write version 1
Sep 22 23:14:31.851 INFO UUID: f811b648-86cf-4cf5-927b-38fc90064f11
Sep 22 23:14:31.851 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:31.851 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:31.851 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:31.851 INFO Using address: 127.0.0.1:52134, task: main
Sep 22 23:14:31.851 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:31.852 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:62441, task: repair
Sep 22 23:14:31.852 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:62441, task: repair
Sep 22 23:14:31.852 INFO listening, local_addr: 127.0.0.1:62441, task: repair
Sep 22 23:14:31.852 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:62441, task: repair
Sep 22 23:14:31.852 INFO Using repair address: 127.0.0.1:62441, task: main
Sep 22 23:14:31.852 INFO No SSL acceptor configured, task: main
Sep 22 23:14:31.852 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:31.852 INFO Created new region file "/tmp/downstairs-Wku7LpKv/region.json"
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.852965784Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/79c71868-5af1-4b01-8453-182bf4ff9124/bulk_write","method":"POST","req_id":"95300955-d478-4550-8427-49b16d182763","remote_addr":"127.0.0.1:49688","local_addr":"127.0.0.1:51522","component":"dropshot","latency_us":9480,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.853048068Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/79c71868-5af1-4b01-8453-182bf4ff9124/bulk_write","method":"POST","req_id":"0f4a5c8f-9f77-4485-878a-226746cbcff2","remote_addr":"127.0.0.1:45421","local_addr":"127.0.0.1:51522","component":"dropshot","latency_us":9385,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.853121172Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/79c71868-5af1-4b01-8453-182bf4ff9124/bulk_write","method":"POST","req_id":"189538a7-9338-4d47-8009-05ad93e8f632","remote_addr":"127.0.0.1:51046","local_addr":"127.0.0.1:51522","component":"dropshot","latency_us":9259,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.853192865Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/79c71868-5af1-4b01-8453-182bf4ff9124/bulk_write","method":"POST","req_id":"d5963a47-84a6-4d9e-a606-b65f0d070384","remote_addr":"127.0.0.1:60061","local_addr":"127.0.0.1:51522","component":"dropshot","latency_us":9131,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.853263558Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/79c71868-5af1-4b01-8453-182bf4ff9124/bulk_write","method":"POST","req_id":"20c44313-ac23-4c07-9233-45b7ecbc8502","remote_addr":"127.0.0.1:54313","local_addr":"127.0.0.1:51522","component":"dropshot","latency_us":9011,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.853334394Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/79c71868-5af1-4b01-8453-182bf4ff9124/bulk_write","method":"POST","req_id":"30ea465d-adb6-4362-979c-a76a1943245d","remote_addr":"127.0.0.1:37651","local_addr":"127.0.0.1:51522","component":"dropshot","latency_us":8885,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.853404417Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/79c71868-5af1-4b01-8453-182bf4ff9124/bulk_write","method":"POST","req_id":"02ef650c-4e27-41d8-a668-8fecd754d689","remote_addr":"127.0.0.1:46952","local_addr":"127.0.0.1:51522","component":"dropshot","latency_us":8766,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.853491027Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/79c71868-5af1-4b01-8453-182bf4ff9124/bulk_write","method":"POST","req_id":"63c9d6b3-87ad-4ed9-b8d9-c36ab148a547","remote_addr":"127.0.0.1:58452","local_addr":"127.0.0.1:51522","component":"dropshot","latency_us":8644,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.853563294Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/79c71868-5af1-4b01-8453-182bf4ff9124/bulk_write","method":"POST","req_id":"1a30ef41-202e-4e81-b5ee-196a04e3f431","remote_addr":"127.0.0.1:41946","local_addr":"127.0.0.1:51522","component":"dropshot","latency_us":8539,"response_code":"204"}
Sep 22 23:14:31.853 DEBG Write     :1001 deps:[] res:true
Sep 22 23:14:31.853 DEBG Write     :1002 deps:[] res:true
Sep 22 23:14:31.854 DEBG Write     :1003 deps:[] res:true
Sep 22 23:14:31.854 DEBG Write     :1004 deps:[] res:true
Sep 22 23:14:31.854 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:31.854 INFO Opened existing region file "/tmp/downstairs-Wku7LpKv/region.json"
Sep 22 23:14:31.854 INFO Database read version 1
Sep 22 23:14:31.854 INFO Database write version 1
Sep 22 23:14:31.854 DEBG Write     :1005 deps:[] res:true
Sep 22 23:14:31.854 DEBG Write     :1006 deps:[] res:true
Sep 22 23:14:31.855 DEBG Write     :1007 deps:[] res:true
Sep 22 23:14:31.855 INFO UUID: 4cdee3da-6bae-4f3e-9cc5-e3e96652865a
Sep 22 23:14:31.855 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:31.855 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:31.855 DEBG Write     :1008 deps:[] res:true
Sep 22 23:14:31.855 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:31.855 INFO Using address: 127.0.0.1:60333, task: main
Sep 22 23:14:31.855 DEBG Write     :1009 deps:[] res:true
Sep 22 23:14:31.855 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:31.855 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:37396, task: repair
Sep 22 23:14:31.855 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:37396, task: repair
Sep 22 23:14:31.855 INFO listening, local_addr: 127.0.0.1:37396, task: repair
Sep 22 23:14:31.855 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:37396, task: repair
Sep 22 23:14:31.855 INFO Using repair address: 127.0.0.1:37396, task: main
Sep 22 23:14:31.855 INFO No SSL acceptor configured, task: main
note: configured to log to "/dev/stdout"
{"msg":"listening","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.856924414Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"local_addr":"127.0.0.1:39234","component":"dropshot"}
{"msg":"listen IP: 127.0.0.1:39234","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.857072013Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
Sep 22 23:14:31.857 DEBG Write     :1001 deps:[] res:true
Sep 22 23:14:31.857 DEBG Write     :1002 deps:[] res:true
Sep 22 23:14:31.857 DEBG Write     :1003 deps:[] res:true
Sep 22 23:14:31.857 DEBG Write     :1004 deps:[] res:true
Sep 22 23:14:31.858 DEBG Write     :1005 deps:[] res:true
Sep 22 23:14:31.858 DEBG Write     :1006 deps:[] res:true
Sep 22 23:14:31.858 DEBG Write     :1007 deps:[] res:true
Sep 22 23:14:31.858 DEBG Write     :1008 deps:[] res:true
Sep 22 23:14:31.858 DEBG Write     :1009 deps:[] res:true
Sep 22 23:14:31.858 DEBG Write     :1001 deps:[] res:true
Sep 22 23:14:31.859 DEBG Write     :1002 deps:[] res:true
Sep 22 23:14:31.859 DEBG Write     :1003 deps:[] res:true
Sep 22 23:14:31.859 DEBG Write     :1004 deps:[] res:true
Sep 22 23:14:31.859 DEBG Write     :1005 deps:[] res:true
Sep 22 23:14:31.859 DEBG Write     :1006 deps:[] res:true
Sep 22 23:14:31.859 DEBG Write     :1007 deps:[] res:true
Sep 22 23:14:31.860 DEBG Write     :1008 deps:[] res:true
Sep 22 23:14:31.860 DEBG Write     :1009 deps:[] res:true
Sep 22 23:14:31.861 DEBG Read      :1010 deps:[JobId(1000)] res:true
Sep 22 23:14:31.862 DEBG Read      :1010 deps:[JobId(1000)] res:true
Sep 22 23:14:31.862 DEBG Read      :1010 deps:[JobId(1000)] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.863683698Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/79c71868-5af1-4b01-8453-182bf4ff9124/bulk_read","method":"POST","req_id":"48c1c518-5f5f-4de9-94a5-a50d0e3198ce","remote_addr":"127.0.0.1:41946","local_addr":"127.0.0.1:51522","component":"dropshot","latency_us":2643,"response_code":"200"}
Sep 22 23:14:31.865 DEBG Read      :1011 deps:[JobId(1001)] res:true
Sep 22 23:14:31.865 DEBG Read      :1011 deps:[JobId(1001)] res:true
Sep 22 23:14:31.865 DEBG Read      :1011 deps:[JobId(1001)] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.866863701Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/79c71868-5af1-4b01-8453-182bf4ff9124/bulk_read","method":"POST","req_id":"709db523-f743-4a43-b2f1-b21923a676da","remote_addr":"127.0.0.1:41946","local_addr":"127.0.0.1:51522","component":"dropshot","latency_us":2560,"response_code":"200"}
Sep 22 23:14:31.868 DEBG Read      :1012 deps:[JobId(1002)] res:true
Sep 22 23:14:31.868 DEBG Read      :1012 deps:[JobId(1002)] res:true
Sep 22 23:14:31.868 DEBG Read      :1012 deps:[JobId(1002)] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.869983457Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/79c71868-5af1-4b01-8453-182bf4ff9124/bulk_read","method":"POST","req_id":"c93c0b26-3196-42cc-b62f-199e2d47c195","remote_addr":"127.0.0.1:41946","local_addr":"127.0.0.1:51522","component":"dropshot","latency_us":2536,"response_code":"200"}
Sep 22 23:14:31.871 DEBG Read      :1013 deps:[JobId(1003)] res:true
Sep 22 23:14:31.871 DEBG Read      :1013 deps:[JobId(1003)] res:true
Sep 22 23:14:31.871 DEBG Read      :1013 deps:[JobId(1003)] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.873245851Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/79c71868-5af1-4b01-8453-182bf4ff9124/bulk_read","method":"POST","req_id":"af4f704a-38df-439d-bc56-77e9d715fe68","remote_addr":"127.0.0.1:41946","local_addr":"127.0.0.1:51522","component":"dropshot","latency_us":2576,"response_code":"200"}
Sep 22 23:14:31.874 DEBG Read      :1014 deps:[JobId(1004)] res:true
Sep 22 23:14:31.874 DEBG Read      :1014 deps:[JobId(1004)] res:true
Sep 22 23:14:31.875 DEBG Read      :1014 deps:[JobId(1004)] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.876418996Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/79c71868-5af1-4b01-8453-182bf4ff9124/bulk_read","method":"POST","req_id":"a06d3977-c87e-4c94-b5b4-1351f3d7c709","remote_addr":"127.0.0.1:41946","local_addr":"127.0.0.1:51522","component":"dropshot","latency_us":2571,"response_code":"200"}
Sep 22 23:14:31.878 DEBG Read      :1015 deps:[JobId(1005)] res:true
Sep 22 23:14:31.878 DEBG Read      :1015 deps:[JobId(1005)] res:true
Sep 22 23:14:31.878 DEBG Read      :1015 deps:[JobId(1005)] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.879882526Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/79c71868-5af1-4b01-8453-182bf4ff9124/bulk_read","method":"POST","req_id":"e17d203d-a6a9-4d13-a86d-775d2ded5dbc","remote_addr":"127.0.0.1:41946","local_addr":"127.0.0.1:51522","component":"dropshot","latency_us":2616,"response_code":"200"}
Sep 22 23:14:31.881 DEBG Read      :1016 deps:[JobId(1006)] res:true
Sep 22 23:14:31.881 DEBG Read      :1016 deps:[JobId(1006)] res:true
Sep 22 23:14:31.881 DEBG Read      :1016 deps:[JobId(1006)] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.88300441Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/79c71868-5af1-4b01-8453-182bf4ff9124/bulk_read","method":"POST","req_id":"da7750e8-2504-4af8-8ec4-968cb3eb56c7","remote_addr":"127.0.0.1:41946","local_addr":"127.0.0.1:51522","component":"dropshot","latency_us":2538,"response_code":"200"}
Sep 22 23:14:31.884 DEBG Read      :1017 deps:[JobId(1007)] res:true
Sep 22 23:14:31.884 DEBG Read      :1017 deps:[JobId(1007)] res:true
Sep 22 23:14:31.884 DEBG Read      :1017 deps:[JobId(1007)] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.886119654Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/79c71868-5af1-4b01-8453-182bf4ff9124/bulk_read","method":"POST","req_id":"6d4a7526-0c20-410a-9e4c-1707bc703366","remote_addr":"127.0.0.1:41946","local_addr":"127.0.0.1:51522","component":"dropshot","latency_us":2533,"response_code":"200"}
Sep 22 23:14:31.887 DEBG Read      :1018 deps:[JobId(1008)] res:true
Sep 22 23:14:31.887 DEBG Read      :1018 deps:[JobId(1008)] res:true
Sep 22 23:14:31.887 DEBG Read      :1018 deps:[JobId(1008)] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.889249163Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/79c71868-5af1-4b01-8453-182bf4ff9124/bulk_read","method":"POST","req_id":"b0f51dc2-99e4-4464-86ea-a51e5ad2a994","remote_addr":"127.0.0.1:41946","local_addr":"127.0.0.1:51522","component":"dropshot","latency_us":2548,"response_code":"200"}
Sep 22 23:14:31.890 WARN e94af85e-3796-4fab-91a1-f12add9c3020 request to replace downstairs 127.0.0.1:52905 with 127.0.0.1:56689
Sep 22 23:14:31.890 INFO e94af85e-3796-4fab-91a1-f12add9c3020 found new target: 127.0.0.1:56689 at 0
Downstairs replacement completed
Sep 22 23:14:31.890 DEBG Read      :1019 deps:[JobId(1009)] res:true
Sep 22 23:14:31.891 DEBG IO Read  1011 has deps []
Sep 22 23:14:31.891 DEBG Read      :1019 deps:[JobId(1009)] res:true
Sep 22 23:14:31.891 DEBG Read      :1019 deps:[JobId(1009)] res:true
Sep 22 23:14:31.891 DEBG Read      :1011 deps:[] res:true
Sep 22 23:14:31.892 DEBG Read      :1011 deps:[] res:true
Sep 22 23:14:31.893 DEBG Read      :1011 deps:[] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.893272031Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/79c71868-5af1-4b01-8453-182bf4ff9124/bulk_read","method":"POST","req_id":"953a44d0-b480-4c38-879c-c0ed4e5f96c8","remote_addr":"127.0.0.1:41946","local_addr":"127.0.0.1:51522","component":"dropshot","latency_us":3330,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.89503642Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/79c71868-5af1-4b01-8453-182bf4ff9124/validate","method":"POST","req_id":"4bd8ca14-e4b3-4ecd-a12d-4bcb290ed780","remote_addr":"127.0.0.1:41946","local_addr":"127.0.0.1:51522","component":"dropshot","latency_us":324,"response_code":"200"}
Sep 22 23:14:31.896 DEBG Read      :1020 deps:[JobId(1009), JobId(1008), JobId(1007), JobId(1006), JobId(1005), JobId(1004), JobId(1003), JobId(1002), JobId(1001), JobId(1000)] res:true
Sep 22 23:14:31.896 DEBG Read      :1020 deps:[JobId(1009), JobId(1008), JobId(1007), JobId(1006), JobId(1005), JobId(1004), JobId(1003), JobId(1002), JobId(1001), JobId(1000)] res:true
Sep 22 23:14:31.897 DEBG Read      :1020 deps:[JobId(1009), JobId(1008), JobId(1007), JobId(1006), JobId(1005), JobId(1004), JobId(1003), JobId(1002), JobId(1001), JobId(1000)] res:true
Sep 22 23:14:31.897 DEBG [1] Read AckReady 1011, : downstairs
Sep 22 23:14:31.900 DEBG [2] Read already AckReady 1011, : downstairs
Sep 22 23:14:31.903 DEBG [0] Read already AckReady 1011, : downstairs
Sep 22 23:14:31.903 DEBG up_ds_listen was notified
Sep 22 23:14:31.903 DEBG up_ds_listen process 1011
Sep 22 23:14:31.903 DEBG [A] ack job 1011:12, : downstairs
Sep 22 23:14:31.903 DEBG up_ds_listen checked 1 jobs, back to waiting
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.908591597Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/job/023f91c6-d6ff-4bd9-a4de-d41b53664b0d/is_finished","method":"GET","req_id":"08326023-eaa4-4f4d-96a1-659c8f8d680b","remote_addr":"127.0.0.1:41946","local_addr":"127.0.0.1:51522","component":"dropshot","latency_us":179,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.909314159Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/job/023f91c6-d6ff-4bd9-a4de-d41b53664b0d/ok","method":"GET","req_id":"09580edb-a327-4adc-a4da-d4a3e0f0e6db","remote_addr":"127.0.0.1:41946","local_addr":"127.0.0.1:51522","component":"dropshot","latency_us":184,"response_code":"200"}
test test::integration_test_volume_replace_downstairs ... ok
Sep 22 23:14:31.909 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:31.909 INFO Created new region file "/tmp/downstairs-V3hLVw1a/region.json"
{"msg":"detach removing entry for volume 79c71868-5af1-4b01-8453-182bf4ff9124","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.910012472Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"detaching volume 79c71868-5af1-4b01-8453-182bf4ff9124","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.910070129Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:31.911 DEBG Flush     :1021 extent_limit None deps:[JobId(1020), JobId(1019), JobId(1018), JobId(1017), JobId(1016), JobId(1015), JobId(1014), JobId(1013), JobId(1012), JobId(1011), JobId(1010), JobId(1009), JobId(1008), JobId(1007), JobId(1006), JobId(1005), JobId(1004), JobId(1003), JobId(1002), JobId(1001), JobId(1000)] res:true f:1 g:1
Sep 22 23:14:31.912 DEBG Flush     :1021 extent_limit None deps:[JobId(1020), JobId(1019), JobId(1018), JobId(1017), JobId(1016), JobId(1015), JobId(1014), JobId(1013), JobId(1012), JobId(1011), JobId(1010), JobId(1009), JobId(1008), JobId(1007), JobId(1006), JobId(1005), JobId(1004), JobId(1003), JobId(1002), JobId(1001), JobId(1000)] res:true f:1 g:1
Sep 22 23:14:31.912 DEBG Flush     :1021 extent_limit None deps:[JobId(1020), JobId(1019), JobId(1018), JobId(1017), JobId(1016), JobId(1015), JobId(1014), JobId(1013), JobId(1012), JobId(1011), JobId(1010), JobId(1009), JobId(1008), JobId(1007), JobId(1006), JobId(1005), JobId(1004), JobId(1003), JobId(1002), JobId(1001), JobId(1000)] res:true f:1 g:1
{"msg":"Request to deactivate this guest","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.912717668Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"451049c0-e2b7-41c3-872a-e5be40bc5510 set deactivating.","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.912774616Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:31.912 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:31.912 INFO Opened existing region file "/tmp/downstairs-V3hLVw1a/region.json"
Sep 22 23:14:31.912 INFO Database read version 1
Sep 22 23:14:31.912 INFO Database write version 1
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.912989407Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/79c71868-5af1-4b01-8453-182bf4ff9124","method":"DELETE","req_id":"7eba7968-1a2b-4c28-a1b8-06ea8cdc136c","remote_addr":"127.0.0.1:41946","local_addr":"127.0.0.1:51522","component":"dropshot","latency_us":3091,"response_code":"204"}
Sep 22 23:14:31.913 INFO UUID: fc04a5c8-8424-4fbf-8cbf-45e4299fa04d
Sep 22 23:14:31.913 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:31.913 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:31.913 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:31.913 INFO Using address: 127.0.0.1:62247, task: main
Sep 22 23:14:31.914 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:31.914 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:57817, task: repair
Sep 22 23:14:31.914 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:57817, task: repair
Sep 22 23:14:31.914 INFO listening, local_addr: 127.0.0.1:57817, task: repair
Sep 22 23:14:31.914 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:57817, task: repair
Sep 22 23:14:31.914 INFO Using repair address: 127.0.0.1:57817, task: main
Sep 22 23:14:31.914 INFO No SSL acceptor configured, task: main
Sep 22 23:14:31.914 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:31.914 INFO Created new region file "/tmp/downstairs-vWZk4NRb/region.json"
test test::test_pantry_validate ... ok
Sep 22 23:14:31.918 INFO test_volume_replace of a volume
Sep 22 23:14:31.918 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:31.918 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:31.918 INFO Opened existing region file "/tmp/downstairs-vWZk4NRb/region.json"
Sep 22 23:14:31.918 INFO Database read version 1
Sep 22 23:14:31.918 INFO Database write version 1
Sep 22 23:14:31.918 INFO Created new region file "/tmp/downstairs-DqWKEajh/region.json"
Sep 22 23:14:31.919 INFO UUID: 64eb55ec-79b8-4aa0-81d6-70fa12931989
Sep 22 23:14:31.919 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:31.919 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:31.919 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:31.919 INFO Using address: 127.0.0.1:39430, task: main
Sep 22 23:14:31.919 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:31.919 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:37977, task: repair
Sep 22 23:14:31.919 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:37977, task: repair
Sep 22 23:14:31.919 INFO listening, local_addr: 127.0.0.1:37977, task: repair
Sep 22 23:14:31.920 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:37977, task: repair
Sep 22 23:14:31.920 INFO Using repair address: 127.0.0.1:37977, task: main
Sep 22 23:14:31.920 INFO No SSL acceptor configured, task: main
Sep 22 23:14:31.920 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:31.920 INFO Created new region file "/tmp/downstairs-PfkZQLmG/region.json"
Sep 22 23:14:31.921 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:31.921 INFO Opened existing region file "/tmp/downstairs-DqWKEajh/region.json"
Sep 22 23:14:31.921 INFO Database read version 1
Sep 22 23:14:31.921 INFO Database write version 1
Sep 22 23:14:31.921 INFO UUID: 399f1d0b-a42b-4a95-9d11-4686487cbd7a
Sep 22 23:14:31.921 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:31.921 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:31.921 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:31.921 INFO Using address: 127.0.0.1:61980, task: main
Sep 22 23:14:31.922 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:31.922 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:58454, task: repair
Sep 22 23:14:31.922 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:58454, task: repair
Sep 22 23:14:31.922 INFO listening, local_addr: 127.0.0.1:58454, task: repair
Sep 22 23:14:31.922 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:58454, task: repair
Sep 22 23:14:31.922 INFO Using repair address: 127.0.0.1:58454, task: main
Sep 22 23:14:31.922 INFO No SSL acceptor configured, task: main
Sep 22 23:14:31.922 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:31.922 INFO Created new region file "/tmp/downstairs-pVoHEn8Q/region.json"
Sep 22 23:14:31.924 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:31.924 INFO Opened existing region file "/tmp/downstairs-PfkZQLmG/region.json"
Sep 22 23:14:31.924 INFO Database read version 1
Sep 22 23:14:31.924 INFO Database write version 1
Sep 22 23:14:31.925 INFO UUID: 6e47f0ac-72f2-4910-8085-d56269d7e5b1
Sep 22 23:14:31.925 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:31.925 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:31.925 INFO Opened existing region file "/tmp/downstairs-pVoHEn8Q/region.json"
Sep 22 23:14:31.925 INFO Database read version 1
Sep 22 23:14:31.925 INFO Database write version 1
Sep 22 23:14:31.925 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:31.925 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:31.925 INFO Using address: 127.0.0.1:44923, task: main
Sep 22 23:14:31.925 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:31.925 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:65121, task: repair
Sep 22 23:14:31.926 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:65121, task: repair
Sep 22 23:14:31.926 INFO listening, local_addr: 127.0.0.1:65121, task: repair
Sep 22 23:14:31.926 INFO UUID: 43c441f1-cf2f-4bad-b83b-ff7ae6f4603d
Sep 22 23:14:31.926 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:31.926 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:31.926 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:31.926 INFO Using address: 127.0.0.1:54419, task: main
Sep 22 23:14:31.926 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:65121, task: repair
Sep 22 23:14:31.926 INFO Using repair address: 127.0.0.1:65121, task: main
Sep 22 23:14:31.926 INFO No SSL acceptor configured, task: main
Sep 22 23:14:31.926 INFO Repair listens on 127.0.0.1:0, task: repair
note: configured to log to "/dev/stdout"
Sep 22 23:14:31.926 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:38668, task: repair
Sep 22 23:14:31.926 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:38668, task: repair
Sep 22 23:14:31.926 INFO listening, local_addr: 127.0.0.1:38668, task: repair
Sep 22 23:14:31.926 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:38668, task: repair
Sep 22 23:14:31.926 INFO Using repair address: 127.0.0.1:38668, task: main
Sep 22 23:14:31.926 INFO No SSL acceptor configured, task: main
Sep 22 23:14:31.926 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:31.926 INFO Created new region file "/tmp/downstairs-TEMyQKsJ/region.json"
{"msg":"listening","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.927980758Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"local_addr":"127.0.0.1:54057","component":"dropshot"}
{"msg":"listen IP: 127.0.0.1:54057","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.928202989Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
Sep 22 23:14:31.929 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:31.929 INFO Opened existing region file "/tmp/downstairs-TEMyQKsJ/region.json"
Sep 22 23:14:31.929 INFO Database read version 1
Sep 22 23:14:31.929 INFO Database write version 1
Sep 22 23:14:31.929 INFO UUID: a2bfa752-7a6e-4ff9-854c-32e43259f2a8
Sep 22 23:14:31.929 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:31.929 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:31.929 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:31.929 INFO Using address: 127.0.0.1:58078, task: main
Sep 22 23:14:31.930 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:31.930 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:43913, task: repair
Sep 22 23:14:31.930 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:43913, task: repair
Sep 22 23:14:31.930 INFO listening, local_addr: 127.0.0.1:43913, task: repair
Sep 22 23:14:31.930 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:43913, task: repair
Sep 22 23:14:31.930 INFO Using repair address: 127.0.0.1:43913, task: main
Sep 22 23:14:31.930 INFO No SSL acceptor configured, task: main
Sep 22 23:14:31.930 INFO Upstairs starts
Sep 22 23:14:31.930 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:14:31.930 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:14:31.930 INFO Crucible stats registered with UUID: 68728c39-2b9d-474c-a894-338e842ff7ac
Sep 22 23:14:31.930 INFO Crucible 68728c39-2b9d-474c-a894-338e842ff7ac has session id: 0a32346b-7a73-4634-a8f6-e4b15f18f823
Sep 22 23:14:31.930 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:31.930 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:31.931 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:31.931 INFO [0] connecting to 127.0.0.1:61980, looper: 0
Sep 22 23:14:31.931 INFO [1] connecting to 127.0.0.1:54419, looper: 1
Sep 22 23:14:31.931 INFO [2] connecting to 127.0.0.1:58078, looper: 2
Sep 22 23:14:31.931 INFO up_listen starts, task: up_listen
Sep 22 23:14:31.931 INFO Wait for all three downstairs to come online
Sep 22 23:14:31.931 INFO Flush timeout: 0.5
Sep 22 23:14:31.931 INFO [0] 68728c39-2b9d-474c-a894-338e842ff7ac looper connected, looper: 0
Sep 22 23:14:31.931 INFO [0] Proc runs for 127.0.0.1:61980 in state New
Sep 22 23:14:31.931 INFO accepted connection from 127.0.0.1:43584, task: main
Sep 22 23:14:31.931 INFO [1] 68728c39-2b9d-474c-a894-338e842ff7ac looper connected, looper: 1
Sep 22 23:14:31.931 INFO [1] Proc runs for 127.0.0.1:54419 in state New
Sep 22 23:14:31.931 INFO [2] 68728c39-2b9d-474c-a894-338e842ff7ac looper connected, looper: 2
Sep 22 23:14:31.931 INFO [2] Proc runs for 127.0.0.1:58078 in state New
Sep 22 23:14:31.931 INFO accepted connection from 127.0.0.1:39878, task: main
Sep 22 23:14:31.931 INFO accepted connection from 127.0.0.1:36978, task: main
Sep 22 23:14:31.932 INFO Connection request from 68728c39-2b9d-474c-a894-338e842ff7ac with version 4, task: proc
Sep 22 23:14:31.932 INFO upstairs UpstairsConnection { upstairs_id: 68728c39-2b9d-474c-a894-338e842ff7ac, session_id: b1e7d83f-1c6e-44a2-a060-99d0bf74c596, gen: 2 } connected, version 4, task: proc
Sep 22 23:14:31.932 INFO Connection request from 68728c39-2b9d-474c-a894-338e842ff7ac with version 4, task: proc
Sep 22 23:14:31.932 INFO upstairs UpstairsConnection { upstairs_id: 68728c39-2b9d-474c-a894-338e842ff7ac, session_id: b1e7d83f-1c6e-44a2-a060-99d0bf74c596, gen: 2 } connected, version 4, task: proc
Sep 22 23:14:31.932 INFO Connection request from 68728c39-2b9d-474c-a894-338e842ff7ac with version 4, task: proc
Sep 22 23:14:31.932 INFO upstairs UpstairsConnection { upstairs_id: 68728c39-2b9d-474c-a894-338e842ff7ac, session_id: b1e7d83f-1c6e-44a2-a060-99d0bf74c596, gen: 2 } connected, version 4, task: proc
Sep 22 23:14:31.932 INFO [0] 68728c39-2b9d-474c-a894-338e842ff7ac (b1e7d83f-1c6e-44a2-a060-99d0bf74c596) New New New ds_transition to WaitActive
Sep 22 23:14:31.932 INFO [0] Transition from New to WaitActive
Sep 22 23:14:31.932 INFO [1] 68728c39-2b9d-474c-a894-338e842ff7ac (b1e7d83f-1c6e-44a2-a060-99d0bf74c596) WaitActive New New ds_transition to WaitActive
Sep 22 23:14:31.932 INFO [1] Transition from New to WaitActive
Sep 22 23:14:31.932 INFO [2] 68728c39-2b9d-474c-a894-338e842ff7ac (b1e7d83f-1c6e-44a2-a060-99d0bf74c596) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:14:31.932 INFO [2] Transition from New to WaitActive
The guest has requested activation
Sep 22 23:14:31.932 INFO 68728c39-2b9d-474c-a894-338e842ff7ac active request set
Sep 22 23:14:31.932 INFO [0] received activate with gen 2
Sep 22 23:14:31.932 INFO [0] client got ds_active_rx, promote! session b1e7d83f-1c6e-44a2-a060-99d0bf74c596
Sep 22 23:14:31.932 INFO [1] received activate with gen 2
Sep 22 23:14:31.932 INFO [1] client got ds_active_rx, promote! session b1e7d83f-1c6e-44a2-a060-99d0bf74c596
Sep 22 23:14:31.932 INFO [2] received activate with gen 2
Sep 22 23:14:31.933 INFO [2] client got ds_active_rx, promote! session b1e7d83f-1c6e-44a2-a060-99d0bf74c596
Sep 22 23:14:31.933 INFO UpstairsConnection { upstairs_id: 68728c39-2b9d-474c-a894-338e842ff7ac, session_id: b1e7d83f-1c6e-44a2-a060-99d0bf74c596, gen: 2 } is now active (read-write)
Sep 22 23:14:31.933 INFO UpstairsConnection { upstairs_id: 68728c39-2b9d-474c-a894-338e842ff7ac, session_id: b1e7d83f-1c6e-44a2-a060-99d0bf74c596, gen: 2 } is now active (read-write)
Sep 22 23:14:31.933 INFO UpstairsConnection { upstairs_id: 68728c39-2b9d-474c-a894-338e842ff7ac, session_id: b1e7d83f-1c6e-44a2-a060-99d0bf74c596, gen: 2 } is now active (read-write)
Sep 22 23:14:31.933 INFO [0] downstairs client at 127.0.0.1:61980 has UUID 399f1d0b-a42b-4a95-9d11-4686487cbd7a
Sep 22 23:14:31.933 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 399f1d0b-a42b-4a95-9d11-4686487cbd7a, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:31.933 INFO 68728c39-2b9d-474c-a894-338e842ff7ac WaitActive WaitActive WaitActive
Sep 22 23:14:31.933 INFO [1] downstairs client at 127.0.0.1:54419 has UUID 43c441f1-cf2f-4bad-b83b-ff7ae6f4603d
Sep 22 23:14:31.933 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 43c441f1-cf2f-4bad-b83b-ff7ae6f4603d, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:31.933 INFO 68728c39-2b9d-474c-a894-338e842ff7ac WaitActive WaitActive WaitActive
Sep 22 23:14:31.933 INFO [2] downstairs client at 127.0.0.1:58078 has UUID a2bfa752-7a6e-4ff9-854c-32e43259f2a8
Sep 22 23:14:31.933 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: a2bfa752-7a6e-4ff9-854c-32e43259f2a8, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:31.933 INFO 68728c39-2b9d-474c-a894-338e842ff7ac WaitActive WaitActive WaitActive
Sep 22 23:14:31.934 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:31.934 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:31.934 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:31.934 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:31.934 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:31.934 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:31.934 INFO [0] 68728c39-2b9d-474c-a894-338e842ff7ac (b1e7d83f-1c6e-44a2-a060-99d0bf74c596) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:31.934 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:14:31.934 WARN [0] new RM replaced this: None
Sep 22 23:14:31.934 INFO [0] Starts reconcile loop
Sep 22 23:14:31.935 INFO [1] 68728c39-2b9d-474c-a894-338e842ff7ac (b1e7d83f-1c6e-44a2-a060-99d0bf74c596) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:31.935 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:14:31.935 WARN [1] new RM replaced this: None
Sep 22 23:14:31.935 INFO [1] Starts reconcile loop
Sep 22 23:14:31.935 INFO [2] 68728c39-2b9d-474c-a894-338e842ff7ac (b1e7d83f-1c6e-44a2-a060-99d0bf74c596) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:14:31.935 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:14:31.935 WARN [2] new RM replaced this: None
Sep 22 23:14:31.935 INFO [2] Starts reconcile loop
Sep 22 23:14:31.935 INFO [0] 127.0.0.1:61980 task reports connection:true
Sep 22 23:14:31.935 INFO 68728c39-2b9d-474c-a894-338e842ff7ac WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:14:31.935 INFO [0]R  flush_numbers: [0, 0]
Sep 22 23:14:31.935 INFO [0]R  generation: [0, 0]
Sep 22 23:14:31.935 INFO [0]R  dirty: [false, false]
Sep 22 23:14:31.935 INFO [1]R  flush_numbers: [0, 0]
Sep 22 23:14:31.935 INFO [1]R  generation: [0, 0]
Sep 22 23:14:31.935 INFO [1]R  dirty: [false, false]
Sep 22 23:14:31.935 INFO [2]R  flush_numbers: [0, 0]
Sep 22 23:14:31.935 INFO [2]R  generation: [0, 0]
Sep 22 23:14:31.935 INFO [2]R  dirty: [false, false]
Sep 22 23:14:31.935 INFO Max found gen is 1
Sep 22 23:14:31.935 INFO Generation requested: 2 >= found:1
Sep 22 23:14:31.935 INFO Next flush: 1
Sep 22 23:14:31.935 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:31.935 INFO All extents match
Sep 22 23:14:31.935 INFO No downstairs repair required
Sep 22 23:14:31.935 INFO No initial repair work was required
Sep 22 23:14:31.935 INFO Set Downstairs and Upstairs active
Sep 22 23:14:31.935 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:31.935 INFO 68728c39-2b9d-474c-a894-338e842ff7ac is now active with session: b1e7d83f-1c6e-44a2-a060-99d0bf74c596
Sep 22 23:14:31.935 INFO 68728c39-2b9d-474c-a894-338e842ff7ac Set Active after no repair
Sep 22 23:14:31.935 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:31.935 INFO Notify all downstairs, region set compare is done.
Sep 22 23:14:31.935 INFO Set check for repair
Sep 22 23:14:31.935 INFO [1] 127.0.0.1:54419 task reports connection:true
Sep 22 23:14:31.935 INFO 68728c39-2b9d-474c-a894-338e842ff7ac Active Active Active
Sep 22 23:14:31.935 INFO Set check for repair
Sep 22 23:14:31.935 INFO [2] 127.0.0.1:58078 task reports connection:true
Sep 22 23:14:31.935 INFO 68728c39-2b9d-474c-a894-338e842ff7ac Active Active Active
Sep 22 23:14:31.935 INFO Set check for repair
Sep 22 23:14:31.935 INFO [0] received reconcile message
Sep 22 23:14:31.935 INFO [0] All repairs completed, exit
Sep 22 23:14:31.935 INFO [0] Starts cmd_loop
Sep 22 23:14:31.935 INFO [1] received reconcile message
Sep 22 23:14:31.935 INFO [1] All repairs completed, exit
Sep 22 23:14:31.935 INFO [1] Starts cmd_loop
Sep 22 23:14:31.935 INFO [2] received reconcile message
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.935795147Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"local_addr":"127.0.0.1:39234","component":"dropshot","remote_addr":"127.0.0.1:43629"}
Sep 22 23:14:31.935 INFO [2] All repairs completed, exit
Sep 22 23:14:31.935 INFO [2] Starts cmd_loop
The guest has finished waiting for activation
{"msg":"no entry exists for volume 7cd46074-355e-4bc3-ab28-d527993e88ad, constructing...","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.93639816Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Upstairs starts","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.936610268Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Crucible Version: BuildInfo {\n    version: \"0.0.1\",\n    git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n    git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n    git_branch: \"main\",\n    rustc_semver: \"1.70.0\",\n    rustc_channel: \"stable\",\n    rustc_host_triple: \"x86_64-unknown-illumos\",\n    rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n    cargo_triple: \"x86_64-unknown-illumos\",\n    debug: true,\n    opt_level: 0,\n}","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.936697285Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.936737668Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Crucible stats registered with UUID: c3b92b8f-1217-4240-b3aa-0b4e2d139bb1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.936776325Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Crucible c3b92b8f-1217-4240-b3aa-0b4e2d139bb1 has session id: b4f0e22e-6012-4edf-bac7-dd4160d48302","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.936814921Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] connecting to 127.0.0.1:45781","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.936851499Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"0","component":"datafile"}
{"msg":"[1] connecting to 127.0.0.1:52134","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.936909222Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"1","component":"datafile"}
{"msg":"[2] connecting to 127.0.0.1:60333","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.937047599Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"2","component":"datafile"}
{"msg":"up_listen starts","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.937211278Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile","task":"up_listen"}
{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.937262343Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Flush timeout: 0.5","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.93729848Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"volume 7cd46074-355e-4bc3-ab28-d527993e88ad constructed ok","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.937463176Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
The guest has requested activation
{"msg":"c3b92b8f-1217-4240-b3aa-0b4e2d139bb1 active request set","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.937735948Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:31.937 INFO accepted connection from 127.0.0.1:64285, task: main
Sep 22 23:14:31.937 INFO accepted connection from 127.0.0.1:56568, task: main
Sep 22 23:14:31.937 INFO accepted connection from 127.0.0.1:44587, task: main
{"msg":"[0] c3b92b8f-1217-4240-b3aa-0b4e2d139bb1 looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.938122609Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"0","component":"datafile"}
{"msg":"[0] Proc runs for 127.0.0.1:45781 in state New","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.938177379Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] c3b92b8f-1217-4240-b3aa-0b4e2d139bb1 looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.938227416Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"1","component":"datafile"}
Sep 22 23:14:31.938 DEBG IO Write 1000 has deps []
{"msg":"[1] Proc runs for 127.0.0.1:52134 in state New","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.938269745Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] c3b92b8f-1217-4240-b3aa-0b4e2d139bb1 looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.938306486Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"2","component":"datafile"}
{"msg":"[2] Proc runs for 127.0.0.1:60333 in state New","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.93834421Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:31.938 DEBG up_ds_listen was notified
Sep 22 23:14:31.938 DEBG up_ds_listen process 1000
Sep 22 23:14:31.938 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:14:31.938 INFO Connection request from c3b92b8f-1217-4240-b3aa-0b4e2d139bb1 with version 4, task: proc
Sep 22 23:14:31.938 INFO upstairs UpstairsConnection { upstairs_id: c3b92b8f-1217-4240-b3aa-0b4e2d139bb1, session_id: 0f529401-1886-4b80-b1e1-a309115d7cef, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:31.938 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:31.938 INFO Connection request from c3b92b8f-1217-4240-b3aa-0b4e2d139bb1 with version 4, task: proc
Sep 22 23:14:31.938 INFO upstairs UpstairsConnection { upstairs_id: c3b92b8f-1217-4240-b3aa-0b4e2d139bb1, session_id: 0f529401-1886-4b80-b1e1-a309115d7cef, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:31.938 INFO Connection request from c3b92b8f-1217-4240-b3aa-0b4e2d139bb1 with version 4, task: proc
Sep 22 23:14:31.938 INFO upstairs UpstairsConnection { upstairs_id: c3b92b8f-1217-4240-b3aa-0b4e2d139bb1, session_id: 0f529401-1886-4b80-b1e1-a309115d7cef, gen: 1 } connected, version 4, task: proc
{"msg":"[0] c3b92b8f-1217-4240-b3aa-0b4e2d139bb1 (0f529401-1886-4b80-b1e1-a309115d7cef) New New New ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.938939162Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.938989296Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] client is_active_req TRUE, promote! session 0f529401-1886-4b80-b1e1-a309115d7cef","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.939031867Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] c3b92b8f-1217-4240-b3aa-0b4e2d139bb1 (0f529401-1886-4b80-b1e1-a309115d7cef) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.939071302Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.939106807Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:31.939 INFO UpstairsConnection { upstairs_id: c3b92b8f-1217-4240-b3aa-0b4e2d139bb1, session_id: 0f529401-1886-4b80-b1e1-a309115d7cef, gen: 1 } is now active (read-write)
{"msg":"[1] client is_active_req TRUE, promote! session 0f529401-1886-4b80-b1e1-a309115d7cef","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.939261185Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] c3b92b8f-1217-4240-b3aa-0b4e2d139bb1 (0f529401-1886-4b80-b1e1-a309115d7cef) WaitActive WaitActive New ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.939313298Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:31.939 INFO UpstairsConnection { upstairs_id: c3b92b8f-1217-4240-b3aa-0b4e2d139bb1, session_id: 0f529401-1886-4b80-b1e1-a309115d7cef, gen: 1 } is now active (read-write)
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.939358352Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] client is_active_req TRUE, promote! session 0f529401-1886-4b80-b1e1-a309115d7cef","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.939397769Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:31.939 INFO UpstairsConnection { upstairs_id: c3b92b8f-1217-4240-b3aa-0b4e2d139bb1, session_id: 0f529401-1886-4b80-b1e1-a309115d7cef, gen: 1 } is now active (read-write)
{"msg":"[0] downstairs client at 127.0.0.1:45781 has UUID 0023ec13-80c8-426d-b3f0-fea7668a3f39","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.940037621Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 0023ec13-80c8-426d-b3f0-fea7668a3f39, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.940095017Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"c3b92b8f-1217-4240-b3aa-0b4e2d139bb1 WaitActive WaitActive WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.940133581Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] downstairs client at 127.0.0.1:52134 has UUID f811b648-86cf-4cf5-927b-38fc90064f11","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.940171177Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: f811b648-86cf-4cf5-927b-38fc90064f11, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.94021457Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"c3b92b8f-1217-4240-b3aa-0b4e2d139bb1 WaitActive WaitActive WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.94025232Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] downstairs client at 127.0.0.1:60333 has UUID 4cdee3da-6bae-4f3e-9cc5-e3e96652865a","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.940295694Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 4cdee3da-6bae-4f3e-9cc5-e3e96652865a, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.940341179Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"c3b92b8f-1217-4240-b3aa-0b4e2d139bb1 WaitActive WaitActive WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.940378022Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:31.940 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:31.940 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:31.940 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:31.941 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:31.941 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:31.941 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:31.941 DEBG Write     :1000 deps:[] res:true
{"msg":"[0] c3b92b8f-1217-4240-b3aa-0b4e2d139bb1 (0f529401-1886-4b80-b1e1-a309115d7cef) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.941672453Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.941721277Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:14:31.94175716Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.94179195Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] c3b92b8f-1217-4240-b3aa-0b4e2d139bb1 (0f529401-1886-4b80-b1e1-a309115d7cef) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.941831511Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.9418672Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:14:31.941902951Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.941943779Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] c3b92b8f-1217-4240-b3aa-0b4e2d139bb1 (0f529401-1886-4b80-b1e1-a309115d7cef) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.941985279Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.9420208Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:14:31.94205599Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.942090543Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] 127.0.0.1:45781 task reports connection:true","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.942125857Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
The guest has finished waiting for activation
{"msg":"c3b92b8f-1217-4240-b3aa-0b4e2d139bb1 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.942162418Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0]R  flush_numbers: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.942200709Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0]R  generation: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.942239148Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0]R  dirty: [false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.942273993Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1]R  flush_numbers: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.942316064Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1]R  generation: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.942351632Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:31.942 DEBG Write     :1000 deps:[] res:true
{"msg":"[1]R  dirty: [false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.942386258Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2]R  flush_numbers: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.942420691Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2]R  generation: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.942455506Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2]R  dirty: [false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.942495125Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Max found gen is 1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.942530642Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.942565657Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Next flush: 1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.942599598Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"All extents match","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.942633252Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"No downstairs repair required","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.942667826Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"No initial repair work was required","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.942702672Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.942738941Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"c3b92b8f-1217-4240-b3aa-0b4e2d139bb1 is now active with session: 0f529401-1886-4b80-b1e1-a309115d7cef","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.942781723Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"c3b92b8f-1217-4240-b3aa-0b4e2d139bb1 Set Active after no repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.942819467Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.9428555Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Set check for repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.942889898Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] 127.0.0.1:52134 task reports connection:true","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.942925165Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"c3b92b8f-1217-4240-b3aa-0b4e2d139bb1 Active Active Active","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.94296071Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Set check for repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.943000884Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] 127.0.0.1:60333 task reports connection:true","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.943043258Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"c3b92b8f-1217-4240-b3aa-0b4e2d139bb1 Active Active Active","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.943081598Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Set check for repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.943115955Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] received reconcile message","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.943150821Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:31.943 DEBG Write     :1000 deps:[] res:true
{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.943185141Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.943219641Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] received reconcile message","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.943254059Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.943289647Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.943328588Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] received reconcile message","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.943364434Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.943399037Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.943433152Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"volume 7cd46074-355e-4bc3-ab28-d527993e88ad activated ok","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.943468979Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:31.943 DEBG IO Read  1001 has deps [JobId(1000)]
{"msg":"volume 7cd46074-355e-4bc3-ab28-d527993e88ad constructed and inserted ok","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.943505468Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.943615922Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/7cd46074-355e-4bc3-ab28-d527993e88ad","method":"POST","req_id":"778caea1-1ec0-4e49-901f-b79a2efc0b03","remote_addr":"127.0.0.1:43629","local_addr":"127.0.0.1:39234","component":"dropshot","latency_us":6523,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.943689184Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/7cd46074-355e-4bc3-ab28-d527993e88ad/validate","method":"POST","req_id":"c9ec6d84-9e90-4647-9c4e-3377db738304","remote_addr":"127.0.0.1:43629","local_addr":"127.0.0.1:39234","component":"dropshot","latency_us":283,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.944529649Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/job/61daaa4e-2831-4f75-abc5-514826d12648/is_finished","method":"GET","req_id":"cc741c30-1c70-4362-bd61-0f3521df38b6","remote_addr":"127.0.0.1:43629","local_addr":"127.0.0.1:39234","component":"dropshot","latency_us":211,"response_code":"200"}
Sep 22 23:14:31.944 DEBG Read      :1001 deps:[JobId(1000)] res:true
Sep 22 23:14:31.945 DEBG Read      :1001 deps:[JobId(1000)] res:true
{"msg":"job 61daaa4e-2831-4f75-abc5-514826d12648 failed with size to validate 100 not divisible by block size 512!","v":0,"name":"crucible-pantry","level":50,"time":"2023-09-22T23:14:31.945354198Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.945411451Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/job/61daaa4e-2831-4f75-abc5-514826d12648/ok","method":"GET","req_id":"a9e7cd82-6c3a-4b1e-9466-f154b27894fa","remote_addr":"127.0.0.1:43629","local_addr":"127.0.0.1:39234","component":"dropshot","latency_us":271,"response_code":"200"}
Sep 22 23:14:31.945 DEBG Read      :1001 deps:[JobId(1000)] res:true
{"msg":"detach removing entry for volume 7cd46074-355e-4bc3-ab28-d527993e88ad","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.946255757Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"detaching volume 7cd46074-355e-4bc3-ab28-d527993e88ad","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.946306026Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:31.946 DEBG Flush     :1000 extent_limit None deps:[] res:true f:1 g:1
Sep 22 23:14:31.947 DEBG Flush     :1000 extent_limit None deps:[] res:true f:1 g:1
Sep 22 23:14:31.947 DEBG Flush     :1000 extent_limit None deps:[] res:true f:1 g:1
{"msg":"Request to deactivate this guest","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.947902633Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"c3b92b8f-1217-4240-b3aa-0b4e2d139bb1 set deactivating.","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.947953988Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.948170307Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/7cd46074-355e-4bc3-ab28-d527993e88ad","method":"DELETE","req_id":"d253661b-e826-45b5-a9e2-0c1e7d190a69","remote_addr":"127.0.0.1:43629","local_addr":"127.0.0.1:39234","component":"dropshot","latency_us":2048,"response_code":"204"}
Sep 22 23:14:31.949 DEBG [0] Read AckReady 1001, : downstairs
Sep 22 23:14:31.952 DEBG [1] Read already AckReady 1001, : downstairs
test test::test_pantry_validate_fail ... ok
Sep 22 23:14:31.954 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:31.955 INFO Created new region file "/tmp/downstairs-hyRADvf6/region.json"
Sep 22 23:14:31.955 DEBG [2] Read already AckReady 1001, : downstairs
Sep 22 23:14:31.955 DEBG up_ds_listen was notified
Sep 22 23:14:31.955 DEBG up_ds_listen process 1001
Sep 22 23:14:31.955 DEBG [A] ack job 1001:2, : downstairs
Sep 22 23:14:31.955 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:31.956 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:31.956 INFO Created new region file "/tmp/downstairs-d7nPPm5v/region.json"
Sep 22 23:14:31.958 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:31.958 INFO Opened existing region file "/tmp/downstairs-hyRADvf6/region.json"
Sep 22 23:14:31.958 INFO Database read version 1
Sep 22 23:14:31.958 INFO Database write version 1
Sep 22 23:14:31.959 INFO UUID: 81a6906e-92f4-4b89-a16f-ad205aeec853
Sep 22 23:14:31.959 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:31.959 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:31.959 INFO Opened existing region file "/tmp/downstairs-d7nPPm5v/region.json"
Sep 22 23:14:31.959 INFO Database read version 1
Sep 22 23:14:31.959 INFO Database write version 1
Sep 22 23:14:31.959 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:31.959 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:31.959 INFO Using address: 127.0.0.1:63206, task: main
Sep 22 23:14:31.960 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:31.960 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:39983, task: repair
Sep 22 23:14:31.960 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:39983, task: repair
Sep 22 23:14:31.960 INFO listening, local_addr: 127.0.0.1:39983, task: repair
Sep 22 23:14:31.960 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:39983, task: repair
Sep 22 23:14:31.960 INFO UUID: 337e6b08-994a-40bb-9f68-b28e57db149f
Sep 22 23:14:31.960 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:31.960 INFO Using repair address: 127.0.0.1:39983, task: main
Sep 22 23:14:31.960 INFO No SSL acceptor configured, task: main
Sep 22 23:14:31.960 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:31.960 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:31.960 INFO Using address: 127.0.0.1:63920, task: main
Sep 22 23:14:31.960 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:31.961 INFO Created new region file "/tmp/downstairs-JqsrYBQV/region.json"
Sep 22 23:14:31.961 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:31.961 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:55595, task: repair
Sep 22 23:14:31.961 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:55595, task: repair
Sep 22 23:14:31.961 INFO listening, local_addr: 127.0.0.1:55595, task: repair
Sep 22 23:14:31.961 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:55595, task: repair
Sep 22 23:14:31.961 INFO Using repair address: 127.0.0.1:55595, task: main
Sep 22 23:14:31.961 INFO No SSL acceptor configured, task: main
Sep 22 23:14:31.961 INFO A New downstairs: 127.0.0.1:63920
Sep 22 23:14:31.961 INFO Old ops target: [127.0.0.1:61980, 127.0.0.1:54419, 127.0.0.1:58078]
Sep 22 23:14:31.961 INFO New ops target: [127.0.0.1:63920, 127.0.0.1:54419, 127.0.0.1:58078]
Sep 22 23:14:31.961 INFO Replace VCR now: Volume { id: db7ee0ac-f829-46eb-9220-5bfcb72a4e78, block_size: 512, sub_volumes: [Region { block_size: 512, blocks_per_extent: 5, extent_count: 2, opts: CrucibleOpts { id: 68728c39-2b9d-474c-a894-338e842ff7ac, target: [127.0.0.1:63920, 127.0.0.1:54419, 127.0.0.1:58078], lossy: false, flush_timeout: None, key: Some("sW8AMOMPttO5lP6RWUN7Sd/rpjhwgoA0UyujUtsb1EI="), cert_pem: None, key_pem: None, root_cert_pem: None, control: None, read_only: false }, gen: 3 }], read_only_parent: None }
Sep 22 23:14:31.962 INFO Volume db7ee0ac-f829-46eb-9220-5bfcb72a4e78, OK to replace: 127.0.0.1:61980 with 127.0.0.1:63920
Sep 22 23:14:31.962 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:31.962 WARN db7ee0ac-f829-46eb-9220-5bfcb72a4e78 request to replace downstairs 127.0.0.1:61980 with 127.0.0.1:63920
Sep 22 23:14:31.962 INFO db7ee0ac-f829-46eb-9220-5bfcb72a4e78 found old target: 127.0.0.1:61980 at 0
Sep 22 23:14:31.962 INFO db7ee0ac-f829-46eb-9220-5bfcb72a4e78 replacing old: 127.0.0.1:61980 at 0
Sep 22 23:14:31.962 INFO [0] client skip 2 in process jobs because fault, : downstairs
Sep 22 23:14:31.962 INFO [0] changed 0 jobs to fault skipped, : downstairs
Sep 22 23:14:31.962 INFO [0] 68728c39-2b9d-474c-a894-338e842ff7ac (b1e7d83f-1c6e-44a2-a060-99d0bf74c596) Active Active Active ds_transition to Replacing
Sep 22 23:14:31.962 INFO [0] Transition from Active to Replacing
Sep 22 23:14:31.962 INFO Replace downstairs underway for db7ee0ac-f829-46eb-9220-5bfcb72a4e78
Sep 22 23:14:31.962 INFO send read now
Sep 22 23:14:31.962 DEBG IO Read  1002 has deps [JobId(1000)]
Sep 22 23:14:31.963 DEBG Read      :1002 deps:[JobId(1000)] res:true
Sep 22 23:14:31.964 DEBG Read      :1002 deps:[JobId(1000)] res:true
Sep 22 23:14:31.964 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:31.964 INFO Opened existing region file "/tmp/downstairs-JqsrYBQV/region.json"
Sep 22 23:14:31.964 INFO Database read version 1
Sep 22 23:14:31.964 INFO Database write version 1
Sep 22 23:14:31.965 INFO UUID: b3079a2e-be41-4aab-ab6a-2a7b8c6e517f
Sep 22 23:14:31.965 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:31.965 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:31.965 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:31.965 INFO Using address: 127.0.0.1:55744, task: main
Sep 22 23:14:31.965 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:31.965 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:49915, task: repair
Sep 22 23:14:31.965 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:49915, task: repair
Sep 22 23:14:31.965 INFO listening, local_addr: 127.0.0.1:49915, task: repair
Sep 22 23:14:31.966 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:49915, task: repair
Sep 22 23:14:31.966 INFO Using repair address: 127.0.0.1:49915, task: main
Sep 22 23:14:31.966 INFO No SSL acceptor configured, task: main
Sep 22 23:14:31.966 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:31.966 INFO Created new region file "/tmp/downstairs-94qyRc9J/region.json"
Sep 22 23:14:31.968 DEBG [1] Read AckReady 1002, : downstairs
Sep 22 23:14:31.969 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:31.969 INFO Opened existing region file "/tmp/downstairs-94qyRc9J/region.json"
Sep 22 23:14:31.969 INFO Database read version 1
Sep 22 23:14:31.969 INFO Database write version 1
Sep 22 23:14:31.970 INFO UUID: 2d772065-2982-4cf2-ae0e-3cc9afca219b
Sep 22 23:14:31.970 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:14:31.970 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:31.970 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:31.970 INFO Using address: 127.0.0.1:59203, task: main
Sep 22 23:14:31.970 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:31.970 DEBG [2] Read already AckReady 1002, : downstairs
Sep 22 23:14:31.970 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:52811, task: repair
Sep 22 23:14:31.970 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:52811, task: repair
Sep 22 23:14:31.970 DEBG up_ds_listen was notified
Sep 22 23:14:31.970 INFO listening, local_addr: 127.0.0.1:52811, task: repair
Sep 22 23:14:31.971 DEBG up_ds_listen process 1002
Sep 22 23:14:31.971 DEBG [A] ack job 1002:3, : downstairs
Sep 22 23:14:31.971 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:52811, task: repair
Sep 22 23:14:31.971 INFO Using repair address: 127.0.0.1:52811, task: main
Sep 22 23:14:31.971 INFO No SSL acceptor configured, task: main
Sep 22 23:14:31.971 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:31.971 INFO Upstairs starts
Sep 22 23:14:31.971 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:14:31.971 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:14:31.971 INFO Crucible stats registered with UUID: d6abcc00-47da-425e-a424-9934c8a21974
Sep 22 23:14:31.971 INFO Crucible d6abcc00-47da-425e-a424-9934c8a21974 has session id: 62be0b39-953d-4d8c-883d-97dac094cc50
Sep 22 23:14:31.971 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:31.971 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:31.971 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:31.972 INFO [0] connecting to 127.0.0.1:63206, looper: 0
Sep 22 23:14:31.972 INFO [1] connecting to 127.0.0.1:55744, looper: 1
Sep 22 23:14:31.972 INFO [2] connecting to 127.0.0.1:59203, looper: 2
Sep 22 23:14:31.972 INFO up_listen starts, task: up_listen
Sep 22 23:14:31.972 INFO Wait for all three downstairs to come online
Sep 22 23:14:31.972 INFO Flush timeout: 0.5
Sep 22 23:14:31.972 INFO [1] d6abcc00-47da-425e-a424-9934c8a21974 looper connected, looper: 1
Sep 22 23:14:31.972 INFO [1] Proc runs for 127.0.0.1:55744 in state New
Sep 22 23:14:31.972 INFO [2] d6abcc00-47da-425e-a424-9934c8a21974 looper connected, looper: 2
Sep 22 23:14:31.972 INFO [2] Proc runs for 127.0.0.1:59203 in state New
Sep 22 23:14:31.972 INFO [0] d6abcc00-47da-425e-a424-9934c8a21974 looper connected, looper: 0
Sep 22 23:14:31.973 INFO [0] Proc runs for 127.0.0.1:63206 in state New
Sep 22 23:14:31.973 INFO accepted connection from 127.0.0.1:42562, task: main
Sep 22 23:14:31.973 INFO accepted connection from 127.0.0.1:62245, task: main
Sep 22 23:14:31.973 INFO accepted connection from 127.0.0.1:58254, task: main
Sep 22 23:14:31.973 INFO Connection request from d6abcc00-47da-425e-a424-9934c8a21974 with version 4, task: proc
Sep 22 23:14:31.973 INFO upstairs UpstairsConnection { upstairs_id: d6abcc00-47da-425e-a424-9934c8a21974, session_id: 0ddebd5a-0043-49cf-93bb-961f6f1d3dfa, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:31.973 INFO Connection request from d6abcc00-47da-425e-a424-9934c8a21974 with version 4, task: proc
Sep 22 23:14:31.973 INFO upstairs UpstairsConnection { upstairs_id: d6abcc00-47da-425e-a424-9934c8a21974, session_id: 0ddebd5a-0043-49cf-93bb-961f6f1d3dfa, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:31.973 INFO Connection request from d6abcc00-47da-425e-a424-9934c8a21974 with version 4, task: proc
Sep 22 23:14:31.973 INFO upstairs UpstairsConnection { upstairs_id: d6abcc00-47da-425e-a424-9934c8a21974, session_id: 0ddebd5a-0043-49cf-93bb-961f6f1d3dfa, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:31.974 INFO [1] d6abcc00-47da-425e-a424-9934c8a21974 (0ddebd5a-0043-49cf-93bb-961f6f1d3dfa) New New New ds_transition to WaitActive
Sep 22 23:14:31.974 INFO [1] Transition from New to WaitActive
Sep 22 23:14:31.974 INFO [2] d6abcc00-47da-425e-a424-9934c8a21974 (0ddebd5a-0043-49cf-93bb-961f6f1d3dfa) New WaitActive New ds_transition to WaitActive
Sep 22 23:14:31.974 INFO [2] Transition from New to WaitActive
Sep 22 23:14:31.974 INFO [0] d6abcc00-47da-425e-a424-9934c8a21974 (0ddebd5a-0043-49cf-93bb-961f6f1d3dfa) New WaitActive WaitActive ds_transition to WaitActive
Sep 22 23:14:31.974 INFO [0] Transition from New to WaitActive
The guest has requested activation
Sep 22 23:14:31.974 INFO d6abcc00-47da-425e-a424-9934c8a21974 active request set
Sep 22 23:14:31.974 INFO [0] received activate with gen 1
Sep 22 23:14:31.974 INFO [0] client got ds_active_rx, promote! session 0ddebd5a-0043-49cf-93bb-961f6f1d3dfa
Sep 22 23:14:31.974 INFO [1] received activate with gen 1
Sep 22 23:14:31.974 INFO [1] client got ds_active_rx, promote! session 0ddebd5a-0043-49cf-93bb-961f6f1d3dfa
Sep 22 23:14:31.975 INFO [2] received activate with gen 1
Sep 22 23:14:31.975 INFO [2] client got ds_active_rx, promote! session 0ddebd5a-0043-49cf-93bb-961f6f1d3dfa
Sep 22 23:14:31.975 INFO UpstairsConnection { upstairs_id: d6abcc00-47da-425e-a424-9934c8a21974, session_id: 0ddebd5a-0043-49cf-93bb-961f6f1d3dfa, gen: 1 } is now active (read-write)
Sep 22 23:14:31.975 INFO UpstairsConnection { upstairs_id: d6abcc00-47da-425e-a424-9934c8a21974, session_id: 0ddebd5a-0043-49cf-93bb-961f6f1d3dfa, gen: 1 } is now active (read-write)
Sep 22 23:14:31.975 INFO UpstairsConnection { upstairs_id: d6abcc00-47da-425e-a424-9934c8a21974, session_id: 0ddebd5a-0043-49cf-93bb-961f6f1d3dfa, gen: 1 } is now active (read-write)
test test::test_volume_replace_vcr ... ok
Sep 22 23:14:31.975 INFO [1] downstairs client at 127.0.0.1:55744 has UUID b3079a2e-be41-4aab-ab6a-2a7b8c6e517f
Sep 22 23:14:31.975 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: b3079a2e-be41-4aab-ab6a-2a7b8c6e517f, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:31.975 INFO d6abcc00-47da-425e-a424-9934c8a21974 WaitActive WaitActive WaitActive
Sep 22 23:14:31.975 INFO [2] downstairs client at 127.0.0.1:59203 has UUID 2d772065-2982-4cf2-ae0e-3cc9afca219b
Sep 22 23:14:31.975 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 2d772065-2982-4cf2-ae0e-3cc9afca219b, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:31.976 INFO d6abcc00-47da-425e-a424-9934c8a21974 WaitActive WaitActive WaitActive
Sep 22 23:14:31.976 INFO [0] downstairs client at 127.0.0.1:63206 has UUID 81a6906e-92f4-4b89-a16f-ad205aeec853
Sep 22 23:14:31.976 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 81a6906e-92f4-4b89-a16f-ad205aeec853, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:31.976 INFO d6abcc00-47da-425e-a424-9934c8a21974 WaitActive WaitActive WaitActive
Sep 22 23:14:31.976 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:31.976 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:31.976 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:31.976 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:31.976 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:31.976 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:31.977 INFO [1] d6abcc00-47da-425e-a424-9934c8a21974 (0ddebd5a-0043-49cf-93bb-961f6f1d3dfa) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:31.977 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:14:31.977 WARN [1] new RM replaced this: None
Sep 22 23:14:31.977 INFO [1] Starts reconcile loop
Sep 22 23:14:31.977 INFO [2] d6abcc00-47da-425e-a424-9934c8a21974 (0ddebd5a-0043-49cf-93bb-961f6f1d3dfa) WaitActive WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:14:31.977 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:14:31.977 WARN [2] new RM replaced this: None
Sep 22 23:14:31.977 INFO [2] Starts reconcile loop
Sep 22 23:14:31.977 INFO [0] d6abcc00-47da-425e-a424-9934c8a21974 (0ddebd5a-0043-49cf-93bb-961f6f1d3dfa) WaitActive WaitQuorum WaitQuorum ds_transition to WaitQuorum
Sep 22 23:14:31.977 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:14:31.977 WARN [0] new RM replaced this: None
Sep 22 23:14:31.977 INFO [0] Starts reconcile loop
Sep 22 23:14:31.977 INFO [1] 127.0.0.1:55744 task reports connection:true
Sep 22 23:14:31.977 INFO d6abcc00-47da-425e-a424-9934c8a21974 WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:14:31.977 INFO [0]R  flush_numbers: [0, 0]
Sep 22 23:14:31.977 INFO [0]R  generation: [0, 0]
Sep 22 23:14:31.977 INFO [0]R  dirty: [false, false]
Sep 22 23:14:31.977 INFO [1]R  flush_numbers: [0, 0]
Sep 22 23:14:31.977 INFO [1]R  generation: [0, 0]
Sep 22 23:14:31.977 INFO [1]R  dirty: [false, false]
Sep 22 23:14:31.977 INFO [2]R  flush_numbers: [0, 0]
Sep 22 23:14:31.977 INFO [2]R  generation: [0, 0]
Sep 22 23:14:31.977 INFO [2]R  dirty: [false, false]
Sep 22 23:14:31.977 INFO Max found gen is 1
Sep 22 23:14:31.977 INFO Generation requested: 1 >= found:1
Sep 22 23:14:31.977 INFO Next flush: 1
Sep 22 23:14:31.977 INFO All extents match
Sep 22 23:14:31.977 INFO No downstairs repair required
Sep 22 23:14:31.977 INFO No initial repair work was required
Sep 22 23:14:31.977 INFO Set Downstairs and Upstairs active
Sep 22 23:14:31.977 INFO d6abcc00-47da-425e-a424-9934c8a21974 is now active with session: 0ddebd5a-0043-49cf-93bb-961f6f1d3dfa
Sep 22 23:14:31.977 INFO d6abcc00-47da-425e-a424-9934c8a21974 Set Active after no repair
Sep 22 23:14:31.977 INFO Notify all downstairs, region set compare is done.
Sep 22 23:14:31.977 INFO Set check for repair
Sep 22 23:14:31.977 INFO [2] 127.0.0.1:59203 task reports connection:true
Sep 22 23:14:31.977 INFO d6abcc00-47da-425e-a424-9934c8a21974 Active Active Active
Sep 22 23:14:31.977 INFO Set check for repair
Sep 22 23:14:31.977 INFO [0] 127.0.0.1:63206 task reports connection:true
Sep 22 23:14:31.977 INFO d6abcc00-47da-425e-a424-9934c8a21974 Active Active Active
Sep 22 23:14:31.977 INFO Set check for repair
Sep 22 23:14:31.977 INFO [0] received reconcile message
Sep 22 23:14:31.977 INFO [0] All repairs completed, exit
Sep 22 23:14:31.977 INFO [0] Starts cmd_loop
Sep 22 23:14:31.977 INFO [1] received reconcile message
Sep 22 23:14:31.977 INFO [1] All repairs completed, exit
Sep 22 23:14:31.977 INFO [1] Starts cmd_loop
Sep 22 23:14:31.977 INFO [2] received reconcile message
Sep 22 23:14:31.977 INFO [2] All repairs completed, exit
Sep 22 23:14:31.977 INFO [2] Starts cmd_loop
The guest has finished waiting for activation
test test::volume_zero_length_io ... ok
Sep 22 23:14:31.995 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:31.995 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:31.995 INFO listening on 127.0.0.1:0, task: main
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.995922125Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"local_addr":"127.0.0.1:54057","component":"dropshot","remote_addr":"127.0.0.1:34332"}
{"msg":"no entry exists for volume 7d6e799f-ffff-4301-8a22-abec78155ecf, constructing...","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.996471039Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Upstairs starts","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.996623389Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Crucible Version: BuildInfo {\n    version: \"0.0.1\",\n    git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n    git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n    git_branch: \"main\",\n    rustc_semver: \"1.70.0\",\n    rustc_channel: \"stable\",\n    rustc_host_triple: \"x86_64-unknown-illumos\",\n    rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n    cargo_triple: \"x86_64-unknown-illumos\",\n    debug: true,\n    opt_level: 0,\n}","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.996672965Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.996697152Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Crucible stats registered with UUID: 4444e318-0130-42a4-85b4-7dec4633fa58","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.996744922Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Crucible 4444e318-0130-42a4-85b4-7dec4633fa58 has session id: 5ec47966-8d02-4373-9261-df6674769b26","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.99677209Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] connecting to 127.0.0.1:62247","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.996794557Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"0","component":"datafile"}
{"msg":"[1] connecting to 127.0.0.1:39430","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.996924703Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"1","component":"datafile"}
{"msg":"[2] connecting to 127.0.0.1:44923","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.996985216Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"2","component":"datafile"}
{"msg":"up_listen starts","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.997100595Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile","task":"up_listen"}
{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.99714859Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Flush timeout: 0.5","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.997184138Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"volume 7d6e799f-ffff-4301-8a22-abec78155ecf constructed ok","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.997307762Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
The guest has requested activation
{"msg":"4444e318-0130-42a4-85b4-7dec4633fa58 active request set","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.997500872Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] 4444e318-0130-42a4-85b4-7dec4633fa58 looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.997563975Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"0","component":"datafile"}
{"msg":"[0] Proc runs for 127.0.0.1:62247 in state New","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.997604034Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] 4444e318-0130-42a4-85b4-7dec4633fa58 looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.99764002Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"1","component":"datafile"}
{"msg":"[1] Proc runs for 127.0.0.1:39430 in state New","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.997677243Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:31.997 INFO accepted connection from 127.0.0.1:56510, task: main
{"msg":"[2] 4444e318-0130-42a4-85b4-7dec4633fa58 looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.997712122Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"2","component":"datafile"}
{"msg":"[2] Proc runs for 127.0.0.1:44923 in state New","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.997790617Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:31.997 INFO accepted connection from 127.0.0.1:38856, task: main
Sep 22 23:14:31.997 INFO accepted connection from 127.0.0.1:57737, task: main
Sep 22 23:14:31.998 INFO Connection request from 4444e318-0130-42a4-85b4-7dec4633fa58 with version 4, task: proc
Sep 22 23:14:31.998 INFO upstairs UpstairsConnection { upstairs_id: 4444e318-0130-42a4-85b4-7dec4633fa58, session_id: 93f845b8-2585-4545-9c84-03a1fb12d486, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:31.998 INFO Connection request from 4444e318-0130-42a4-85b4-7dec4633fa58 with version 4, task: proc
Sep 22 23:14:31.998 INFO upstairs UpstairsConnection { upstairs_id: 4444e318-0130-42a4-85b4-7dec4633fa58, session_id: 93f845b8-2585-4545-9c84-03a1fb12d486, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:31.998 INFO Connection request from 4444e318-0130-42a4-85b4-7dec4633fa58 with version 4, task: proc
Sep 22 23:14:31.998 INFO upstairs UpstairsConnection { upstairs_id: 4444e318-0130-42a4-85b4-7dec4633fa58, session_id: 93f845b8-2585-4545-9c84-03a1fb12d486, gen: 1 } connected, version 4, task: proc
{"msg":"[0] 4444e318-0130-42a4-85b4-7dec4633fa58 (93f845b8-2585-4545-9c84-03a1fb12d486) New New New ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.998384133Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.998432275Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] client is_active_req TRUE, promote! session 93f845b8-2585-4545-9c84-03a1fb12d486","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.998470761Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] 4444e318-0130-42a4-85b4-7dec4633fa58 (93f845b8-2585-4545-9c84-03a1fb12d486) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.998509915Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.998544684Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:31.998 INFO UpstairsConnection { upstairs_id: 4444e318-0130-42a4-85b4-7dec4633fa58, session_id: 93f845b8-2585-4545-9c84-03a1fb12d486, gen: 1 } is now active (read-write)
{"msg":"[1] client is_active_req TRUE, promote! session 93f845b8-2585-4545-9c84-03a1fb12d486","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.998583085Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] 4444e318-0130-42a4-85b4-7dec4633fa58 (93f845b8-2585-4545-9c84-03a1fb12d486) WaitActive WaitActive New ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.998626928Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:31.998 INFO UpstairsConnection { upstairs_id: 4444e318-0130-42a4-85b4-7dec4633fa58, session_id: 93f845b8-2585-4545-9c84-03a1fb12d486, gen: 1 } is now active (read-write)
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.99866277Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] client is_active_req TRUE, promote! session 93f845b8-2585-4545-9c84-03a1fb12d486","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.998700161Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:31.998 INFO UpstairsConnection { upstairs_id: 4444e318-0130-42a4-85b4-7dec4633fa58, session_id: 93f845b8-2585-4545-9c84-03a1fb12d486, gen: 1 } is now active (read-write)
{"msg":"[0] downstairs client at 127.0.0.1:62247 has UUID fc04a5c8-8424-4fbf-8cbf-45e4299fa04d","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.999109395Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: fc04a5c8-8424-4fbf-8cbf-45e4299fa04d, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.999171624Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"4444e318-0130-42a4-85b4-7dec4633fa58 WaitActive WaitActive WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.999209901Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] downstairs client at 127.0.0.1:39430 has UUID 64eb55ec-79b8-4aa0-81d6-70fa12931989","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.999252437Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 64eb55ec-79b8-4aa0-81d6-70fa12931989, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.999298054Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"4444e318-0130-42a4-85b4-7dec4633fa58 WaitActive WaitActive WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.999334445Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] downstairs client at 127.0.0.1:44923 has UUID 6e47f0ac-72f2-4910-8085-d56269d7e5b1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.999371453Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:31.999 INFO Current flush_numbers [0..12]: [0, 0]
{"msg":"[2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 6e47f0ac-72f2-4910-8085-d56269d7e5b1, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.999415182Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"4444e318-0130-42a4-85b4-7dec4633fa58 WaitActive WaitActive WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:31.99945171Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:31.999 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:31.999 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:31.999 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:31.999 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:14:32.000 INFO Downstairs has completed Negotiation, task: proc
{"msg":"[0] 4444e318-0130-42a4-85b4-7dec4633fa58 (93f845b8-2585-4545-9c84-03a1fb12d486) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.000288667Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.000337506Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:14:32.000373948Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.000412098Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] 4444e318-0130-42a4-85b4-7dec4633fa58 (93f845b8-2585-4545-9c84-03a1fb12d486) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.000455778Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.000498137Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:14:32.000534258Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
The guest has finished waiting for activation
{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.000569021Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] 4444e318-0130-42a4-85b4-7dec4633fa58 (93f845b8-2585-4545-9c84-03a1fb12d486) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.000608986Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.000644523Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:14:32.000680825Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.000719227Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] 127.0.0.1:62247 task reports connection:true","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.000757559Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"4444e318-0130-42a4-85b4-7dec4633fa58 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.000791824Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0]R  flush_numbers: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.000814591Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0]R  generation: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.000847925Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0]R  dirty: [false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.000886431Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1]R  flush_numbers: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.000924061Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1]R  generation: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.000959084Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1]R  dirty: [false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.000993652Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2]R  flush_numbers: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.001028249Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2]R  generation: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.001062853Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2]R  dirty: [false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.001102941Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Max found gen is 1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.001139367Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.001179033Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Next flush: 1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.00121361Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"All extents match","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.001247971Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"No downstairs repair required","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.001282656Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"No initial repair work was required","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.001317576Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.001352547Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"4444e318-0130-42a4-85b4-7dec4633fa58 is now active with session: 93f845b8-2585-4545-9c84-03a1fb12d486","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.00139022Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"4444e318-0130-42a4-85b4-7dec4633fa58 Set Active after no repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.001429657Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.001468535Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Set check for repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.001503032Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] 127.0.0.1:39430 task reports connection:true","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.00153842Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"4444e318-0130-42a4-85b4-7dec4633fa58 Active Active Active","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.001574191Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Set check for repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.001608551Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] 127.0.0.1:44923 task reports connection:true","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.00164391Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"4444e318-0130-42a4-85b4-7dec4633fa58 Active Active Active","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.001679964Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Set check for repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.001719137Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] received reconcile message","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.001758812Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.001799772Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.001835094Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] received reconcile message","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.001869742Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.001904582Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.001938853Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] received reconcile message","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.001973378Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.002012646Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.002049208Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"volume 7d6e799f-ffff-4301-8a22-abec78155ecf activated ok","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.002085315Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"volume 7d6e799f-ffff-4301-8a22-abec78155ecf constructed and inserted ok","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.002122014Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.002156285Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/7d6e799f-ffff-4301-8a22-abec78155ecf","method":"POST","req_id":"c001bbe6-4785-43eb-915e-b9567e602310","remote_addr":"127.0.0.1:34332","local_addr":"127.0.0.1:54057","component":"dropshot","latency_us":4669,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.00478651Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"local_addr":"127.0.0.1:54057","component":"dropshot","remote_addr":"127.0.0.1:62949"}
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.004846597Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"local_addr":"127.0.0.1:54057","component":"dropshot","remote_addr":"127.0.0.1:53307"}
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.004892955Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"local_addr":"127.0.0.1:54057","component":"dropshot","remote_addr":"127.0.0.1:35509"}
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.004936076Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"local_addr":"127.0.0.1:54057","component":"dropshot","remote_addr":"127.0.0.1:42885"}
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.004978655Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"local_addr":"127.0.0.1:54057","component":"dropshot","remote_addr":"127.0.0.1:65096"}
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.005021144Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"local_addr":"127.0.0.1:54057","component":"dropshot","remote_addr":"127.0.0.1:59818"}
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.005071457Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"local_addr":"127.0.0.1:54057","component":"dropshot","remote_addr":"127.0.0.1:56187"}
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.005121182Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"local_addr":"127.0.0.1:54057","component":"dropshot","remote_addr":"127.0.0.1:56761"}
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.005165584Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"local_addr":"127.0.0.1:54057","component":"dropshot","remote_addr":"127.0.0.1:61150"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.006376526Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/7d6e799f-ffff-4301-8a22-abec78155ecf/bulk_write","method":"POST","req_id":"30056ddb-6ed0-4f18-bd1e-44768cf7bcd4","remote_addr":"127.0.0.1:34332","local_addr":"127.0.0.1:54057","component":"dropshot","latency_us":2404,"response_code":"204"}
Sep 22 23:14:32.009 DEBG Write     :1000 deps:[] res:true
Sep 22 23:14:32.009 DEBG Write     :1000 deps:[] res:true
Sep 22 23:14:32.009 DEBG Write     :1000 deps:[] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.014841988Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/7d6e799f-ffff-4301-8a22-abec78155ecf/bulk_write","method":"POST","req_id":"126feb0c-69ef-46d5-acd7-60170937aef3","remote_addr":"127.0.0.1:62949","local_addr":"127.0.0.1:54057","component":"dropshot","latency_us":8036,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.014917981Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/7d6e799f-ffff-4301-8a22-abec78155ecf/bulk_write","method":"POST","req_id":"cfe6fab7-8bc9-4f9d-bcac-0b4d09e3f00c","remote_addr":"127.0.0.1:53307","local_addr":"127.0.0.1:54057","component":"dropshot","latency_us":7971,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.014980197Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/7d6e799f-ffff-4301-8a22-abec78155ecf/bulk_write","method":"POST","req_id":"49ea016f-8a0c-4f2e-a30a-1d346bf924a8","remote_addr":"127.0.0.1:35509","local_addr":"127.0.0.1:54057","component":"dropshot","latency_us":7870,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.015041491Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/7d6e799f-ffff-4301-8a22-abec78155ecf/bulk_write","method":"POST","req_id":"4a77812b-dcb9-491a-892f-62729e36fc4c","remote_addr":"127.0.0.1:42885","local_addr":"127.0.0.1:54057","component":"dropshot","latency_us":7768,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.015108206Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/7d6e799f-ffff-4301-8a22-abec78155ecf/bulk_write","method":"POST","req_id":"0b76f18e-13f9-4866-bac8-2583255a6088","remote_addr":"127.0.0.1:65096","local_addr":"127.0.0.1:54057","component":"dropshot","latency_us":7660,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.015169773Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/7d6e799f-ffff-4301-8a22-abec78155ecf/bulk_write","method":"POST","req_id":"0c8c4ab0-856a-4b10-b259-3efcbe1b4380","remote_addr":"127.0.0.1:59818","local_addr":"127.0.0.1:54057","component":"dropshot","latency_us":7558,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.015236227Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/7d6e799f-ffff-4301-8a22-abec78155ecf/bulk_write","method":"POST","req_id":"a3429450-5aa8-48f5-92b2-585248ff823e","remote_addr":"127.0.0.1:56187","local_addr":"127.0.0.1:54057","component":"dropshot","latency_us":7458,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.01529676Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/7d6e799f-ffff-4301-8a22-abec78155ecf/bulk_write","method":"POST","req_id":"8554dbf9-0c26-48d0-8026-5a8aa2aed1f6","remote_addr":"127.0.0.1:56761","local_addr":"127.0.0.1:54057","component":"dropshot","latency_us":7359,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.015360171Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/7d6e799f-ffff-4301-8a22-abec78155ecf/bulk_write","method":"POST","req_id":"61ebdd06-77f5-49ab-9634-b6200427a6c8","remote_addr":"127.0.0.1:61150","local_addr":"127.0.0.1:54057","component":"dropshot","latency_us":7253,"response_code":"204"}
Sep 22 23:14:32.015 DEBG Write     :1001 deps:[] res:true
Sep 22 23:14:32.015 DEBG Write     :1002 deps:[] res:true
Sep 22 23:14:32.015 DEBG Write     :1003 deps:[] res:true
Sep 22 23:14:32.015 DEBG Write     :1004 deps:[] res:true
Sep 22 23:14:32.016 DEBG Write     :1005 deps:[] res:true
Sep 22 23:14:32.016 DEBG Write     :1006 deps:[] res:true
Sep 22 23:14:32.016 DEBG Write     :1007 deps:[] res:true
Sep 22 23:14:32.016 DEBG Write     :1008 deps:[] res:true
Sep 22 23:14:32.016 DEBG Write     :1009 deps:[] res:true
Sep 22 23:14:32.018 DEBG Write     :1001 deps:[] res:true
Sep 22 23:14:32.019 DEBG Write     :1002 deps:[] res:true
Sep 22 23:14:32.019 DEBG Write     :1003 deps:[] res:true
Sep 22 23:14:32.019 DEBG Write     :1004 deps:[] res:true
Sep 22 23:14:32.019 DEBG Write     :1005 deps:[] res:true
Sep 22 23:14:32.019 DEBG Write     :1006 deps:[] res:true
Sep 22 23:14:32.019 DEBG Write     :1007 deps:[] res:true
Sep 22 23:14:32.020 DEBG Write     :1008 deps:[] res:true
Sep 22 23:14:32.020 DEBG Write     :1009 deps:[] res:true
Sep 22 23:14:32.020 DEBG Write     :1001 deps:[] res:true
Sep 22 23:14:32.020 DEBG Write     :1002 deps:[] res:true
Sep 22 23:14:32.020 DEBG Write     :1003 deps:[] res:true
Sep 22 23:14:32.020 DEBG Write     :1004 deps:[] res:true
Sep 22 23:14:32.021 DEBG Write     :1005 deps:[] res:true
Sep 22 23:14:32.021 DEBG Write     :1006 deps:[] res:true
Sep 22 23:14:32.021 DEBG Write     :1007 deps:[] res:true
Sep 22 23:14:32.021 DEBG Write     :1008 deps:[] res:true
Sep 22 23:14:32.021 DEBG Write     :1009 deps:[] res:true
Sep 22 23:14:32.023 DEBG Read      :1010 deps:[JobId(1000)] res:true
Sep 22 23:14:32.023 DEBG Read      :1010 deps:[JobId(1000)] res:true
Sep 22 23:14:32.023 DEBG Read      :1010 deps:[JobId(1000)] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.025061451Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/7d6e799f-ffff-4301-8a22-abec78155ecf/bulk_read","method":"POST","req_id":"463ed387-467f-4392-b935-b464d1c1ed56","remote_addr":"127.0.0.1:61150","local_addr":"127.0.0.1:54057","component":"dropshot","latency_us":2493,"response_code":"200"}
Sep 22 23:14:32.026 DEBG Read      :1011 deps:[JobId(1001)] res:true
Sep 22 23:14:32.026 DEBG Read      :1011 deps:[JobId(1001)] res:true
Sep 22 23:14:32.026 DEBG Read      :1011 deps:[JobId(1001)] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.027992771Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/7d6e799f-ffff-4301-8a22-abec78155ecf/bulk_read","method":"POST","req_id":"b761941d-8d3f-49fd-abe7-fb468dee286c","remote_addr":"127.0.0.1:61150","local_addr":"127.0.0.1:54057","component":"dropshot","latency_us":2374,"response_code":"200"}
Sep 22 23:14:32.029 DEBG Read      :1012 deps:[JobId(1002)] res:true
Sep 22 23:14:32.029 DEBG Read      :1012 deps:[JobId(1002)] res:true
Sep 22 23:14:32.029 DEBG Read      :1012 deps:[JobId(1002)] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.031039848Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/7d6e799f-ffff-4301-8a22-abec78155ecf/bulk_read","method":"POST","req_id":"6ca67021-c10a-4252-babd-8c45426e87e9","remote_addr":"127.0.0.1:61150","local_addr":"127.0.0.1:54057","component":"dropshot","latency_us":2474,"response_code":"200"}
Sep 22 23:14:32.032 DEBG Read      :1013 deps:[JobId(1003)] res:true
Sep 22 23:14:32.032 DEBG Read      :1013 deps:[JobId(1003)] res:true
Sep 22 23:14:32.032 DEBG Read      :1013 deps:[JobId(1003)] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.033954737Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/7d6e799f-ffff-4301-8a22-abec78155ecf/bulk_read","method":"POST","req_id":"03ca5732-067f-4810-b794-2a823ce721ab","remote_addr":"127.0.0.1:61150","local_addr":"127.0.0.1:54057","component":"dropshot","latency_us":2368,"response_code":"200"}
Sep 22 23:14:32.035 DEBG Read      :1014 deps:[JobId(1004)] res:true
Sep 22 23:14:32.035 DEBG Read      :1014 deps:[JobId(1004)] res:true
Sep 22 23:14:32.035 DEBG Read      :1014 deps:[JobId(1004)] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.036904892Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/7d6e799f-ffff-4301-8a22-abec78155ecf/bulk_read","method":"POST","req_id":"e09b1a8e-df46-4bb0-ae5e-387c98b0f721","remote_addr":"127.0.0.1:61150","local_addr":"127.0.0.1:54057","component":"dropshot","latency_us":2400,"response_code":"200"}
Sep 22 23:14:32.038 DEBG Read      :1015 deps:[JobId(1005)] res:true
Sep 22 23:14:32.038 DEBG Read      :1015 deps:[JobId(1005)] res:true
Sep 22 23:14:32.038 DEBG Read      :1015 deps:[JobId(1005)] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.039906393Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/7d6e799f-ffff-4301-8a22-abec78155ecf/bulk_read","method":"POST","req_id":"0a359e8e-7efc-4eaf-b5c9-3bb12b087927","remote_addr":"127.0.0.1:61150","local_addr":"127.0.0.1:54057","component":"dropshot","latency_us":2454,"response_code":"200"}
Sep 22 23:14:32.041 DEBG Read      :1016 deps:[JobId(1006)] res:true
Sep 22 23:14:32.041 DEBG Read      :1016 deps:[JobId(1006)] res:true
Sep 22 23:14:32.041 DEBG Read      :1016 deps:[JobId(1006)] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.042835199Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/7d6e799f-ffff-4301-8a22-abec78155ecf/bulk_read","method":"POST","req_id":"723329f0-fcfe-4502-a291-8b3b9c9bbfd7","remote_addr":"127.0.0.1:61150","local_addr":"127.0.0.1:54057","component":"dropshot","latency_us":2377,"response_code":"200"}
Sep 22 23:14:32.044 DEBG Read      :1017 deps:[JobId(1007)] res:true
Sep 22 23:14:32.044 DEBG Read      :1017 deps:[JobId(1007)] res:true
Sep 22 23:14:32.044 DEBG Read      :1017 deps:[JobId(1007)] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.045755217Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/7d6e799f-ffff-4301-8a22-abec78155ecf/bulk_read","method":"POST","req_id":"a20da959-a5a4-4930-9193-4d269b85c2bb","remote_addr":"127.0.0.1:61150","local_addr":"127.0.0.1:54057","component":"dropshot","latency_us":2375,"response_code":"200"}
Sep 22 23:14:32.047 DEBG Read      :1018 deps:[JobId(1008)] res:true
Sep 22 23:14:32.047 DEBG Read      :1018 deps:[JobId(1008)] res:true
Sep 22 23:14:32.047 DEBG Read      :1018 deps:[JobId(1008)] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.048676643Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/7d6e799f-ffff-4301-8a22-abec78155ecf/bulk_read","method":"POST","req_id":"42f8088b-8d87-41a6-9d9e-70a39064ae09","remote_addr":"127.0.0.1:61150","local_addr":"127.0.0.1:54057","component":"dropshot","latency_us":2370,"response_code":"200"}
Sep 22 23:14:32.049 DEBG Read      :1019 deps:[JobId(1009)] res:true
Sep 22 23:14:32.050 DEBG Read      :1019 deps:[JobId(1009)] res:true
Sep 22 23:14:32.050 DEBG Read      :1019 deps:[JobId(1009)] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.051599019Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/7d6e799f-ffff-4301-8a22-abec78155ecf/bulk_read","method":"POST","req_id":"7798cda2-8112-401a-acde-792a99af81b1","remote_addr":"127.0.0.1:61150","local_addr":"127.0.0.1:54057","component":"dropshot","latency_us":2370,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.05261166Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/7d6e799f-ffff-4301-8a22-abec78155ecf/validate","method":"POST","req_id":"cf097eee-de48-420d-bb5c-76c7894f5ab6","remote_addr":"127.0.0.1:61150","local_addr":"127.0.0.1:54057","component":"dropshot","latency_us":236,"response_code":"200"}
Sep 22 23:14:32.053 DEBG Read      :1020 deps:[JobId(1004), JobId(1003), JobId(1002), JobId(1001), JobId(1000)] res:true
Sep 22 23:14:32.053 DEBG Read      :1020 deps:[JobId(1004), JobId(1003), JobId(1002), JobId(1001), JobId(1000)] res:true
Sep 22 23:14:32.053 DEBG Read      :1020 deps:[JobId(1004), JobId(1003), JobId(1002), JobId(1001), JobId(1000)] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.058226326Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/job/e11c43d1-a447-4be1-8ab4-f569126c52b2/is_finished","method":"GET","req_id":"ab161ce6-c776-4d86-9679-71c8e134b9ed","remote_addr":"127.0.0.1:61150","local_addr":"127.0.0.1:54057","component":"dropshot","latency_us":162,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.058881015Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/job/e11c43d1-a447-4be1-8ab4-f569126c52b2/ok","method":"GET","req_id":"ff02ad82-f6f7-4d75-8a94-7903d5fccc55","remote_addr":"127.0.0.1:61150","local_addr":"127.0.0.1:54057","component":"dropshot","latency_us":170,"response_code":"200"}
{"msg":"detach removing entry for volume 7d6e799f-ffff-4301-8a22-abec78155ecf","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.05945453Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"detaching volume 7d6e799f-ffff-4301-8a22-abec78155ecf","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.05950367Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:32.060 DEBG Flush     :1021 extent_limit None deps:[JobId(1020), JobId(1019), JobId(1018), JobId(1017), JobId(1016), JobId(1015), JobId(1014), JobId(1013), JobId(1012), JobId(1011), JobId(1010), JobId(1009), JobId(1008), JobId(1007), JobId(1006), JobId(1005), JobId(1004), JobId(1003), JobId(1002), JobId(1001), JobId(1000)] res:true f:1 g:1
Sep 22 23:14:32.060 DEBG Flush     :1021 extent_limit None deps:[JobId(1020), JobId(1019), JobId(1018), JobId(1017), JobId(1016), JobId(1015), JobId(1014), JobId(1013), JobId(1012), JobId(1011), JobId(1010), JobId(1009), JobId(1008), JobId(1007), JobId(1006), JobId(1005), JobId(1004), JobId(1003), JobId(1002), JobId(1001), JobId(1000)] res:true f:1 g:1
Sep 22 23:14:32.061 DEBG Flush     :1021 extent_limit None deps:[JobId(1020), JobId(1019), JobId(1018), JobId(1017), JobId(1016), JobId(1015), JobId(1014), JobId(1013), JobId(1012), JobId(1011), JobId(1010), JobId(1009), JobId(1008), JobId(1007), JobId(1006), JobId(1005), JobId(1004), JobId(1003), JobId(1002), JobId(1001), JobId(1000)] res:true f:1 g:1
{"msg":"Request to deactivate this guest","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.061599595Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"4444e318-0130-42a4-85b4-7dec4633fa58 set deactivating.","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.061649141Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.061805925Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/7d6e799f-ffff-4301-8a22-abec78155ecf","method":"DELETE","req_id":"4692e680-54f2-48b2-8f9b-4e71bf64d59f","remote_addr":"127.0.0.1:61150","local_addr":"127.0.0.1:54057","component":"dropshot","latency_us":2449,"response_code":"204"}
test test::test_pantry_validate_subset ... ok
Sep 22 23:14:32.093 DEBG IO Write 1000 has deps []
Sep 22 23:14:32.094 INFO Checking if live repair is needed
Sep 22 23:14:32.094 INFO No Live Repair required at this time
Sep 22 23:14:32.094 DEBG up_ds_listen was notified
Sep 22 23:14:32.094 DEBG up_ds_listen process 1000
Sep 22 23:14:32.094 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:14:32.094 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:32.257 DEBG Write     :1000 deps:[] res:true
Sep 22 23:14:32.272 DEBG Write     :1000 deps:[] res:true
Sep 22 23:14:32.288 DEBG Write     :1000 deps:[] res:true
Sep 22 23:14:32.293 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:1 g:1
Sep 22 23:14:32.293 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:1 g:1
Sep 22 23:14:32.293 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:1 g:1
Sep 22 23:14:32.485 DEBG Write     :1000 deps:[] res:true
Sep 22 23:14:32.500 DEBG Write     :1000 deps:[] res:true
Sep 22 23:14:32.502 DEBG IO Write 1001 has deps []
Sep 22 23:14:32.502 DEBG up_ds_listen was notified
Sep 22 23:14:32.502 DEBG up_ds_listen process 1001
Sep 22 23:14:32.502 DEBG [A] ack job 1001:2, : downstairs
Sep 22 23:14:32.503 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:32.503 DEBG IO Flush 1002 has deps [JobId(1001), JobId(1000)]
Sep 22 23:14:32.516 DEBG Write     :1000 deps:[] res:true
Sep 22 23:14:32.522 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:1 g:1
Sep 22 23:14:32.522 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:1 g:1
Sep 22 23:14:32.522 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:1 g:1
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.628306655Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/job/a52bd47d-76c9-4928-9712-218eb57bfeb7/is_finished","method":"GET","req_id":"fd9b3ce4-b994-4876-be17-fd28ec18bca6","remote_addr":"127.0.0.1:49478","local_addr":"127.0.0.1:33566","component":"dropshot","latency_us":273,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:32.642871437Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/job/de4c6089-f970-4d99-9581-6bd61244a7f8/is_finished","method":"GET","req_id":"841983c7-4468-458c-bf79-c64d38599663","remote_addr":"127.0.0.1:58570","local_addr":"127.0.0.1:61933","component":"dropshot","latency_us":256,"response_code":"200"}
Sep 22 23:14:32.842 DEBG IO Write 1003 has deps [JobId(1002)]
Sep 22 23:14:32.842 DEBG up_ds_listen was notified
Sep 22 23:14:32.842 DEBG up_ds_listen process 1003
Sep 22 23:14:32.842 DEBG [A] ack job 1003:4, : downstairs
Sep 22 23:14:32.842 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:33.018 DEBG Write     :1002 deps:[] res:true
Sep 22 23:14:33.033 DEBG Write     :1002 deps:[] res:true
Sep 22 23:14:33.048 DEBG Write     :1002 deps:[] res:true
Sep 22 23:14:33.101 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:33.101 INFO Created new region file "/tmp/downstairs-Xh4OLaSh/region.json"
Sep 22 23:14:33.183 DEBG Flush     :1003 extent_limit None deps:[JobId(1002)] res:true f:2 g:1
Sep 22 23:14:33.183 DEBG Flush     :1003 extent_limit None deps:[JobId(1002)] res:true f:2 g:1
Sep 22 23:14:33.183 DEBG Flush     :1003 extent_limit None deps:[JobId(1002)] res:true f:2 g:1
Sep 22 23:14:33.197 DEBG IO Write 1004 has deps [JobId(1002)]
Sep 22 23:14:33.197 DEBG up_ds_listen was notified
Sep 22 23:14:33.197 DEBG up_ds_listen process 1004
Sep 22 23:14:33.197 DEBG [A] ack job 1004:5, : downstairs
Sep 22 23:14:33.197 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:33.197 DEBG IO Flush 1005 has deps [JobId(1004), JobId(1003), JobId(1002)]
Sep 22 23:14:33.203 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:33.203 INFO Opened existing region file "/tmp/downstairs-Xh4OLaSh/region.json"
Sep 22 23:14:33.203 INFO Database read version 1
Sep 22 23:14:33.203 INFO Database write version 1
Sep 22 23:14:33.251 INFO UUID: bafba53d-f7d4-4132-95f6-92b6a45487a9
Sep 22 23:14:33.251 INFO Blocks per extent:512 Total Extents: 188
Sep 22 23:14:33.251 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:33.251 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:33.251 INFO Using address: 127.0.0.1:63639, task: main
Sep 22 23:14:33.252 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:33.252 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:35667, task: repair
Sep 22 23:14:33.252 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:35667, task: repair
Sep 22 23:14:33.252 INFO listening, local_addr: 127.0.0.1:35667, task: repair
Sep 22 23:14:33.252 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:35667, task: repair
Sep 22 23:14:33.252 INFO Using repair address: 127.0.0.1:35667, task: main
Sep 22 23:14:33.252 INFO No SSL acceptor configured, task: main
Sep 22 23:14:33.259 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:33.259 INFO Created new region file "/tmp/downstairs-x3lx8d7J/region.json"
Sep 22 23:14:33.357 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:33.357 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:33.357 INFO Opened existing region file "/tmp/downstairs-x3lx8d7J/region.json"
Sep 22 23:14:33.357 INFO Database read version 1
Sep 22 23:14:33.357 INFO Database write version 1
Sep 22 23:14:33.378 DEBG Write     :1002 deps:[] res:true
Sep 22 23:14:33.393 DEBG Write     :1002 deps:[] res:true
Sep 22 23:14:33.407 INFO UUID: 1e7a075d-e51e-4361-bdf9-32f916f89647
Sep 22 23:14:33.407 INFO Blocks per extent:512 Total Extents: 188
Sep 22 23:14:33.407 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:33.407 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:33.407 INFO Using address: 127.0.0.1:43949, task: main
Sep 22 23:14:33.408 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:33.408 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:46314, task: repair
Sep 22 23:14:33.408 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:46314, task: repair
Sep 22 23:14:33.408 INFO listening, local_addr: 127.0.0.1:46314, task: repair
Sep 22 23:14:33.408 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:46314, task: repair
Sep 22 23:14:33.408 INFO Using repair address: 127.0.0.1:46314, task: main
Sep 22 23:14:33.408 INFO No SSL acceptor configured, task: main
Sep 22 23:14:33.409 DEBG Write     :1002 deps:[] res:true
Sep 22 23:14:33.415 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:33.415 INFO Created new region file "/tmp/downstairs-qle3gqaw/region.json"
Sep 22 23:14:33.518 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:33.518 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:33.518 INFO Opened existing region file "/tmp/downstairs-qle3gqaw/region.json"
Sep 22 23:14:33.518 INFO Database read version 1
Sep 22 23:14:33.518 INFO Database write version 1
Sep 22 23:14:33.527 DEBG Flush     :1003 extent_limit None deps:[JobId(1002)] res:true f:2 g:1
Sep 22 23:14:33.527 DEBG Flush     :1003 extent_limit None deps:[JobId(1002)] res:true f:2 g:1
Sep 22 23:14:33.527 DEBG Flush     :1003 extent_limit None deps:[JobId(1002)] res:true f:2 g:1
Sep 22 23:14:33.571 INFO UUID: b89ddae9-a405-4d4a-865d-e95f1fd0a715
Sep 22 23:14:33.571 INFO Blocks per extent:512 Total Extents: 188
Sep 22 23:14:33.571 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:33.571 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:33.571 INFO Using address: 127.0.0.1:63175, task: main
Sep 22 23:14:33.572 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:33.572 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:47187, task: repair
Sep 22 23:14:33.572 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:47187, task: repair
Sep 22 23:14:33.572 INFO listening, local_addr: 127.0.0.1:47187, task: repair
Sep 22 23:14:33.572 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:47187, task: repair
Sep 22 23:14:33.572 INFO Using repair address: 127.0.0.1:47187, task: main
Sep 22 23:14:33.572 INFO No SSL acceptor configured, task: main
Sep 22 23:14:33.579 INFO Upstairs starts
Sep 22 23:14:33.579 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:14:33.579 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:14:33.579 INFO Crucible stats registered with UUID: 521f2004-29de-4d41-9494-447a49569d29
Sep 22 23:14:33.579 INFO Crucible 521f2004-29de-4d41-9494-447a49569d29 has session id: 2b3ff256-dd3e-46b0-a6b3-756709d17c55
Sep 22 23:14:33.579 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:33.579 INFO [0] connecting to 127.0.0.1:63639, looper: 0
Sep 22 23:14:33.580 INFO [1] connecting to 127.0.0.1:43949, looper: 1
Sep 22 23:14:33.580 INFO [2] connecting to 127.0.0.1:63175, looper: 2
Sep 22 23:14:33.580 INFO up_listen starts, task: up_listen
Sep 22 23:14:33.580 INFO Wait for all three downstairs to come online
Sep 22 23:14:33.580 INFO Flush timeout: 0.5
Sep 22 23:14:33.580 INFO [0] 521f2004-29de-4d41-9494-447a49569d29 looper connected, looper: 0
Sep 22 23:14:33.580 INFO [0] Proc runs for 127.0.0.1:63639 in state New
Sep 22 23:14:33.580 INFO [1] 521f2004-29de-4d41-9494-447a49569d29 looper connected, looper: 1
Sep 22 23:14:33.580 INFO [1] Proc runs for 127.0.0.1:43949 in state New
Sep 22 23:14:33.580 INFO [2] 521f2004-29de-4d41-9494-447a49569d29 looper connected, looper: 2
Sep 22 23:14:33.580 INFO [2] Proc runs for 127.0.0.1:63175 in state New
Sep 22 23:14:33.580 INFO accepted connection from 127.0.0.1:52189, task: main
Sep 22 23:14:33.580 INFO accepted connection from 127.0.0.1:54318, task: main
Sep 22 23:14:33.580 INFO accepted connection from 127.0.0.1:37634, task: main
Sep 22 23:14:33.581 DEBG IO Write 1006 has deps [JobId(1005), JobId(1002)]
Sep 22 23:14:33.581 DEBG up_ds_listen was notified
Sep 22 23:14:33.581 DEBG up_ds_listen process 1006
Sep 22 23:14:33.581 DEBG [A] ack job 1006:7, : downstairs
Sep 22 23:14:33.581 DEBG up_ds_listen checked 1 jobs, back to waiting
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:33.630965Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/job/a52bd47d-76c9-4928-9712-218eb57bfeb7/is_finished","method":"GET","req_id":"099b8a41-2c32-421f-aa4a-808d3528e54b","remote_addr":"127.0.0.1:49478","local_addr":"127.0.0.1:33566","component":"dropshot","latency_us":216,"response_code":"200"}
Sep 22 23:14:33.671 INFO Connection request from 521f2004-29de-4d41-9494-447a49569d29 with version 4, task: proc
Sep 22 23:14:33.671 INFO upstairs UpstairsConnection { upstairs_id: 521f2004-29de-4d41-9494-447a49569d29, session_id: ca92c31b-eb28-41e5-843d-26b8bd24de02, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:33.671 INFO Connection request from 521f2004-29de-4d41-9494-447a49569d29 with version 4, task: proc
Sep 22 23:14:33.671 INFO upstairs UpstairsConnection { upstairs_id: 521f2004-29de-4d41-9494-447a49569d29, session_id: ca92c31b-eb28-41e5-843d-26b8bd24de02, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:33.671 INFO Connection request from 521f2004-29de-4d41-9494-447a49569d29 with version 4, task: proc
Sep 22 23:14:33.671 INFO upstairs UpstairsConnection { upstairs_id: 521f2004-29de-4d41-9494-447a49569d29, session_id: ca92c31b-eb28-41e5-843d-26b8bd24de02, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:33.672 INFO [0] 521f2004-29de-4d41-9494-447a49569d29 (ca92c31b-eb28-41e5-843d-26b8bd24de02) New New New ds_transition to WaitActive
Sep 22 23:14:33.672 INFO [0] Transition from New to WaitActive
Sep 22 23:14:33.672 INFO [1] 521f2004-29de-4d41-9494-447a49569d29 (ca92c31b-eb28-41e5-843d-26b8bd24de02) WaitActive New New ds_transition to WaitActive
Sep 22 23:14:33.672 INFO [1] Transition from New to WaitActive
Sep 22 23:14:33.672 INFO [2] 521f2004-29de-4d41-9494-447a49569d29 (ca92c31b-eb28-41e5-843d-26b8bd24de02) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:14:33.672 INFO [2] Transition from New to WaitActive
Sep 22 23:14:33.703 DEBG Write     :1004 deps:[] res:true
Sep 22 23:14:33.719 DEBG Write     :1004 deps:[] res:true
Sep 22 23:14:33.734 DEBG Write     :1004 deps:[] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:33.73744675Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/job/de4c6089-f970-4d99-9581-6bd61244a7f8/is_finished","method":"GET","req_id":"f88f8002-cb7f-4c17-a71d-06fed6786627","remote_addr":"127.0.0.1:58570","local_addr":"127.0.0.1:61933","component":"dropshot","latency_us":373,"response_code":"200"}
Sep 22 23:14:33.742 DEBG Flush     :1005 extent_limit None deps:[JobId(1004)] res:true f:3 g:1
Sep 22 23:14:33.742 DEBG Flush     :1005 extent_limit None deps:[JobId(1004)] res:true f:3 g:1
Sep 22 23:14:33.742 DEBG Flush     :1005 extent_limit None deps:[JobId(1004)] res:true f:3 g:1
Sep 22 23:14:33.917 DEBG IO Write 1007 has deps [JobId(1005), JobId(1002)]
Sep 22 23:14:33.917 DEBG up_ds_listen was notified
Sep 22 23:14:33.917 DEBG up_ds_listen process 1007
Sep 22 23:14:33.917 DEBG [A] ack job 1007:8, : downstairs
Sep 22 23:14:33.917 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:33.918 DEBG IO Flush 1008 has deps [JobId(1007), JobId(1006), JobId(1005)]
The guest has requested activation
Sep 22 23:14:33.959 INFO 521f2004-29de-4d41-9494-447a49569d29 active request set
Sep 22 23:14:33.959 INFO [0] received activate with gen 1
Sep 22 23:14:33.959 INFO [0] client got ds_active_rx, promote! session ca92c31b-eb28-41e5-843d-26b8bd24de02
Sep 22 23:14:33.959 INFO [1] received activate with gen 1
Sep 22 23:14:33.959 INFO [1] client got ds_active_rx, promote! session ca92c31b-eb28-41e5-843d-26b8bd24de02
Sep 22 23:14:33.959 INFO [2] received activate with gen 1
Sep 22 23:14:33.959 INFO [2] client got ds_active_rx, promote! session ca92c31b-eb28-41e5-843d-26b8bd24de02
Sep 22 23:14:33.960 INFO UpstairsConnection { upstairs_id: 521f2004-29de-4d41-9494-447a49569d29, session_id: ca92c31b-eb28-41e5-843d-26b8bd24de02, gen: 1 } is now active (read-write)
Sep 22 23:14:33.960 INFO UpstairsConnection { upstairs_id: 521f2004-29de-4d41-9494-447a49569d29, session_id: ca92c31b-eb28-41e5-843d-26b8bd24de02, gen: 1 } is now active (read-write)
Sep 22 23:14:33.960 INFO UpstairsConnection { upstairs_id: 521f2004-29de-4d41-9494-447a49569d29, session_id: ca92c31b-eb28-41e5-843d-26b8bd24de02, gen: 1 } is now active (read-write)
Sep 22 23:14:33.961 INFO [0] downstairs client at 127.0.0.1:63639 has UUID bafba53d-f7d4-4132-95f6-92b6a45487a9
Sep 22 23:14:33.961 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 512, shift: 9 }, extent_count: 188, uuid: bafba53d-f7d4-4132-95f6-92b6a45487a9, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:33.961 INFO 521f2004-29de-4d41-9494-447a49569d29 WaitActive WaitActive WaitActive
Sep 22 23:14:33.961 INFO [1] downstairs client at 127.0.0.1:43949 has UUID 1e7a075d-e51e-4361-bdf9-32f916f89647
Sep 22 23:14:33.961 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 512, shift: 9 }, extent_count: 188, uuid: 1e7a075d-e51e-4361-bdf9-32f916f89647, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:33.961 INFO 521f2004-29de-4d41-9494-447a49569d29 WaitActive WaitActive WaitActive
Sep 22 23:14:33.961 INFO [2] downstairs client at 127.0.0.1:63175 has UUID b89ddae9-a405-4d4a-865d-e95f1fd0a715
Sep 22 23:14:33.961 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 512, shift: 9 }, extent_count: 188, uuid: b89ddae9-a405-4d4a-865d-e95f1fd0a715, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:33.961 INFO 521f2004-29de-4d41-9494-447a49569d29 WaitActive WaitActive WaitActive
Sep 22 23:14:33.979 INFO Current flush_numbers [0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:14:33.982 INFO Current flush_numbers [0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:14:33.985 INFO Current flush_numbers [0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:14:34.011 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:34.012 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:34.013 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:34.013 INFO [0] 521f2004-29de-4d41-9494-447a49569d29 (ca92c31b-eb28-41e5-843d-26b8bd24de02) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:34.013 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:14:34.013 WARN [0] new RM replaced this: None
Sep 22 23:14:34.013 INFO [0] Starts reconcile loop
Sep 22 23:14:34.013 INFO [1] 521f2004-29de-4d41-9494-447a49569d29 (ca92c31b-eb28-41e5-843d-26b8bd24de02) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:34.013 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:14:34.013 WARN [1] new RM replaced this: None
Sep 22 23:14:34.013 INFO [1] Starts reconcile loop
Sep 22 23:14:34.014 INFO [2] 521f2004-29de-4d41-9494-447a49569d29 (ca92c31b-eb28-41e5-843d-26b8bd24de02) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:14:34.014 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:14:34.014 WARN [2] new RM replaced this: None
Sep 22 23:14:34.014 INFO [2] Starts reconcile loop
Sep 22 23:14:34.014 INFO [0] 127.0.0.1:63639 task reports connection:true
Sep 22 23:14:34.014 INFO 521f2004-29de-4d41-9494-447a49569d29 WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:14:34.014 INFO [0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:14:34.014 INFO [0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:14:34.014 INFO [0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]
Sep 22 23:14:34.014 INFO [1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:14:34.014 INFO [1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:14:34.014 INFO [1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]
Sep 22 23:14:34.014 INFO [2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:14:34.014 INFO [2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:14:34.014 INFO [2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]
Sep 22 23:14:34.014 INFO Max found gen is 1
Sep 22 23:14:34.014 INFO Generation requested: 1 >= found:1
Sep 22 23:14:34.014 INFO Next flush: 1
Sep 22 23:14:34.014 INFO All extents match
Sep 22 23:14:34.014 INFO No downstairs repair required
Sep 22 23:14:34.014 INFO No initial repair work was required
Sep 22 23:14:34.014 INFO Set Downstairs and Upstairs active
Sep 22 23:14:34.014 INFO 521f2004-29de-4d41-9494-447a49569d29 is now active with session: ca92c31b-eb28-41e5-843d-26b8bd24de02
Sep 22 23:14:34.014 INFO 521f2004-29de-4d41-9494-447a49569d29 Set Active after no repair
Sep 22 23:14:34.014 INFO Notify all downstairs, region set compare is done.
Sep 22 23:14:34.014 INFO Set check for repair
Sep 22 23:14:34.014 INFO [1] 127.0.0.1:43949 task reports connection:true
Sep 22 23:14:34.014 INFO 521f2004-29de-4d41-9494-447a49569d29 Active Active Active
Sep 22 23:14:34.014 INFO Set check for repair
Sep 22 23:14:34.014 INFO [2] 127.0.0.1:63175 task reports connection:true
Sep 22 23:14:34.014 INFO 521f2004-29de-4d41-9494-447a49569d29 Active Active Active
Sep 22 23:14:34.014 INFO Set check for repair
Sep 22 23:14:34.014 INFO [0] received reconcile message
Sep 22 23:14:34.014 INFO [0] All repairs completed, exit
Sep 22 23:14:34.014 INFO [0] Starts cmd_loop
Sep 22 23:14:34.014 INFO [1] received reconcile message
Sep 22 23:14:34.014 INFO [1] All repairs completed, exit
Sep 22 23:14:34.014 INFO [1] Starts cmd_loop
Sep 22 23:14:34.014 INFO [2] received reconcile message
Sep 22 23:14:34.014 INFO [2] All repairs completed, exit
Sep 22 23:14:34.014 INFO [2] Starts cmd_loop
The guest has finished waiting for activation
Sep 22 23:14:34.028 DEBG IO Read  1000 has deps []
Sep 22 23:14:34.065 DEBG Read      :1000 deps:[] res:true
Sep 22 23:14:34.089 DEBG Read      :1000 deps:[] res:true
Sep 22 23:14:34.126 DEBG Read      :1000 deps:[] res:true
Sep 22 23:14:34.147 DEBG IO Flush 1001 has deps [JobId(1000)]
Sep 22 23:14:34.150 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:1 g:1
Sep 22 23:14:34.151 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:1 g:1
Sep 22 23:14:34.152 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:1 g:1
Sep 22 23:14:34.232 DEBG Write     :1006 deps:[] res:true
Sep 22 23:14:34.237 DEBG Write     :1004 deps:[] res:true
Sep 22 23:14:34.252 DEBG Write     :1006 deps:[] res:true
Sep 22 23:14:34.262 DEBG Write     :1004 deps:[] res:true
Sep 22 23:14:34.271 DEBG Write     :1006 deps:[] res:true
Sep 22 23:14:34.273 DEBG IO Write 1009 has deps [JobId(1008), JobId(1005), JobId(1002)]
Sep 22 23:14:34.273 DEBG up_ds_listen was notified
Sep 22 23:14:34.273 DEBG up_ds_listen process 1009
Sep 22 23:14:34.273 DEBG [A] ack job 1009:10, : downstairs
Sep 22 23:14:34.273 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:34.279 DEBG Flush     :1007 extent_limit None deps:[JobId(1006)] res:true f:4 g:1
Sep 22 23:14:34.279 DEBG Flush     :1007 extent_limit None deps:[JobId(1006)] res:true f:4 g:1
Sep 22 23:14:34.279 DEBG Flush     :1007 extent_limit None deps:[JobId(1006)] res:true f:4 g:1
Sep 22 23:14:34.286 DEBG Write     :1004 deps:[] res:true
Sep 22 23:14:34.291 DEBG Flush     :1005 extent_limit None deps:[JobId(1004)] res:true f:3 g:1
Sep 22 23:14:34.291 DEBG Flush     :1005 extent_limit None deps:[JobId(1004)] res:true f:3 g:1
Sep 22 23:14:34.291 DEBG Flush     :1005 extent_limit None deps:[JobId(1004)] res:true f:3 g:1
Sep 22 23:14:34.344 DEBG [0] Read AckReady 1000, : downstairs
Sep 22 23:14:34.381 DEBG [1] Read already AckReady 1000, : downstairs
Sep 22 23:14:34.418 DEBG [2] Read already AckReady 1000, : downstairs
Sep 22 23:14:34.420 DEBG up_ds_listen was notified
Sep 22 23:14:34.420 DEBG up_ds_listen process 1000
Sep 22 23:14:34.420 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:14:34.518 DEBG up_ds_listen process 1001
Sep 22 23:14:34.518 DEBG [A] ack job 1001:2, : downstairs
Sep 22 23:14:34.518 DEBG [rc] retire 1001 clears [JobId(1000), JobId(1001)], : downstairs
Sep 22 23:14:34.518 DEBG up_ds_listen checked 2 jobs, back to waiting
Sep 22 23:14:34.518 DEBG up_ds_listen was notified
Sep 22 23:14:34.518 DEBG up_ds_listen checked 0 jobs, back to waiting
Sep 22 23:14:34.607 DEBG IO Write 1010 has deps [JobId(1008), JobId(1005), JobId(1002)]
Sep 22 23:14:34.607 DEBG up_ds_listen was notified
Sep 22 23:14:34.607 DEBG up_ds_listen process 1010
Sep 22 23:14:34.607 DEBG [A] ack job 1010:11, : downstairs
Sep 22 23:14:34.607 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:34.607 DEBG IO Flush 1011 has deps [JobId(1010), JobId(1009), JobId(1008)]
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:34.645984254Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/job/a52bd47d-76c9-4928-9712-218eb57bfeb7/is_finished","method":"GET","req_id":"dfb7be5b-209a-4151-83e5-bdabb62bc93b","remote_addr":"127.0.0.1:49478","local_addr":"127.0.0.1:33566","component":"dropshot","latency_us":314,"response_code":"200"}
Sep 22 23:14:34.711 DEBG Write     :1006 deps:[] res:true
Sep 22 23:14:34.723 DEBG Write     :1006 deps:[] res:true
Sep 22 23:14:34.735 DEBG Write     :1006 deps:[] res:true
Sep 22 23:14:34.739 DEBG Flush     :1007 extent_limit None deps:[JobId(1006)] res:true f:4 g:1
Sep 22 23:14:34.739 DEBG Flush     :1007 extent_limit None deps:[JobId(1006)] res:true f:4 g:1
Sep 22 23:14:34.739 DEBG Flush     :1007 extent_limit None deps:[JobId(1006)] res:true f:4 g:1
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:34.745003209Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/job/de4c6089-f970-4d99-9581-6bd61244a7f8/is_finished","method":"GET","req_id":"6cfed45a-e874-4b8e-b7ba-11ec76783bdb","remote_addr":"127.0.0.1:58570","local_addr":"127.0.0.1:61933","component":"dropshot","latency_us":175,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:34.745658149Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/job/de4c6089-f970-4d99-9581-6bd61244a7f8/ok","method":"GET","req_id":"1fb6ef58-1139-4f08-98ee-924b8eb1e7b2","remote_addr":"127.0.0.1:58570","local_addr":"127.0.0.1:61933","component":"dropshot","latency_us":172,"response_code":"200"}
{"msg":"detach removing entry for volume d6736968-99ba-4ebf-b4ff-15afb0904fed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:34.746235669Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"detaching volume d6736968-99ba-4ebf-b4ff-15afb0904fed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:34.746267559Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:34.746 DEBG Flush     :1008 extent_limit None deps:[] res:true f:5 g:1
Sep 22 23:14:34.746 DEBG Flush     :1008 extent_limit None deps:[] res:true f:5 g:1
Sep 22 23:14:34.746 DEBG Flush     :1008 extent_limit None deps:[] res:true f:5 g:1
{"msg":"Request to deactivate this guest","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:34.747281043Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"0d1229ab-bd36-49b6-95e1-d6287586b840 set deactivating.","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:34.747311857Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:34.747483772Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/d6736968-99ba-4ebf-b4ff-15afb0904fed","method":"DELETE","req_id":"b4571eef-b144-43ac-ae81-3e5e28b550cb","remote_addr":"127.0.0.1:58570","local_addr":"127.0.0.1:61933","component":"dropshot","latency_us":1348,"response_code":"204"}
Sep 22 23:14:34.938 DEBG IO Write 1012 has deps [JobId(1011), JobId(1008), JobId(1005), JobId(1002)]
Sep 22 23:14:34.938 DEBG up_ds_listen was notified
Sep 22 23:14:34.938 DEBG up_ds_listen process 1012
Sep 22 23:14:34.938 DEBG [A] ack job 1012:13, : downstairs
Sep 22 23:14:34.938 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:35.015 INFO Checking if live repair is needed
Sep 22 23:14:35.015 INFO No Live Repair required at this time
Sep 22 23:14:35.331 DEBG IO Write 1013 has deps [JobId(1011), JobId(1008), JobId(1005), JobId(1002)]
Sep 22 23:14:35.331 DEBG up_ds_listen was notified
Sep 22 23:14:35.331 DEBG up_ds_listen process 1013
Sep 22 23:14:35.331 DEBG [A] ack job 1013:14, : downstairs
Sep 22 23:14:35.331 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:35.477 DEBG IO Flush 1014 has deps [JobId(1013), JobId(1012), JobId(1011)]
Sep 22 23:14:35.478 INFO [lossy] sleeping 1 second
Sep 22 23:14:35.509 DEBG Write     :1000 deps:[] res:true
Sep 22 23:14:35.510 WARN returning error on write!
Sep 22 23:14:35.510 DEBG Write     :1000 deps:[] res:false
Sep 22 23:14:35.540 DEBG Write     :1000 deps:[] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:35.648745981Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/job/a52bd47d-76c9-4928-9712-218eb57bfeb7/is_finished","method":"GET","req_id":"3e70b85a-273e-4e67-9d0c-8ea791124386","remote_addr":"127.0.0.1:49478","local_addr":"127.0.0.1:33566","component":"dropshot","latency_us":252,"response_code":"200"}
{"msg":"job a52bd47d-76c9-4928-9712-218eb57bfeb7 failed with sha256 digest mismatch! expected 00000000000000000000000000000000000000000000000000000000f5b32221, saw 319d678f093c43502ca360911d52b475dea7fa6dcd962150c84fff18f5b32221","v":0,"name":"crucible-pantry","level":50,"time":"2023-09-22T23:14:35.649415394Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:35.64950897Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/job/a52bd47d-76c9-4928-9712-218eb57bfeb7/ok","method":"GET","req_id":"81f6f686-93b7-4ef4-bd40-9057ebcdbb21","remote_addr":"127.0.0.1:49478","local_addr":"127.0.0.1:33566","component":"dropshot","latency_us":225,"response_code":"200"}
{"msg":"detach removing entry for volume 1e088c74-6c1b-4846-90a3-1bfe70bc4f7f","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:35.650108372Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"detaching volume 1e088c74-6c1b-4846-90a3-1bfe70bc4f7f","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:35.650139028Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:35.650 DEBG Flush     :1008 extent_limit None deps:[] res:true f:5 g:1
Sep 22 23:14:35.650 DEBG Flush     :1008 extent_limit None deps:[] res:true f:5 g:1
Sep 22 23:14:35.650 DEBG Flush     :1008 extent_limit None deps:[] res:true f:5 g:1
{"msg":"Request to deactivate this guest","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:35.651264435Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"982ad9a6-37b1-4b89-8804-8ba0c1300cac set deactivating.","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:35.651300129Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:35.651474491Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/1e088c74-6c1b-4846-90a3-1bfe70bc4f7f","method":"DELETE","req_id":"cccaebf3-3153-477b-8d2b-af0f03a4a07c","remote_addr":"127.0.0.1:49478","local_addr":"127.0.0.1:33566","component":"dropshot","latency_us":1472,"response_code":"204"}
test test::test_pantry_import_from_url_ovmf_bad_digest ... ok
Sep 22 23:14:35.882 DEBG IO Write 1015 has deps [JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)]
Sep 22 23:14:35.882 DEBG up_ds_listen was notified
Sep 22 23:14:35.882 DEBG up_ds_listen process 1015
Sep 22 23:14:35.883 DEBG [A] ack job 1015:16, : downstairs
Sep 22 23:14:35.883 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:35.939 INFO Request to deactivate this guest
Sep 22 23:14:35.939 INFO 521f2004-29de-4d41-9494-447a49569d29 set deactivating.
Sep 22 23:14:35.939 DEBG No work, no need to flush, return OK
note: configured to log to "/dev/stdout"
{"msg":"listening","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:35.941359138Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"local_addr":"127.0.0.1:49411","component":"dropshot"}
{"msg":"listen IP: 127.0.0.1:49411","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:35.941519821Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.007235648Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"local_addr":"127.0.0.1:49411","component":"dropshot","remote_addr":"127.0.0.1:43906"}
{"msg":"no entry exists for volume fc5b0c26-5d3b-433c-b382-b2a002415eea, constructing...","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.007875991Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Upstairs starts","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.008069919Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Crucible Version: BuildInfo {\n    version: \"0.0.1\",\n    git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n    git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n    git_branch: \"main\",\n    rustc_semver: \"1.70.0\",\n    rustc_channel: \"stable\",\n    rustc_host_triple: \"x86_64-unknown-illumos\",\n    rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n    cargo_triple: \"x86_64-unknown-illumos\",\n    debug: true,\n    opt_level: 0,\n}","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.008118626Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.008143362Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Crucible stats registered with UUID: 521f2004-29de-4d41-9494-447a49569d29","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.008167201Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Crucible 521f2004-29de-4d41-9494-447a49569d29 has session id: aa90d023-19be-44f8-b8a2-74eb98aeb980","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.008191638Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] connecting to 127.0.0.1:63639","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.008293116Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"0","component":"datafile"}
{"msg":"[1] connecting to 127.0.0.1:43949","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.00839748Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"1","component":"datafile"}
{"msg":"[2] connecting to 127.0.0.1:63175","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.008483221Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"2","component":"datafile"}
{"msg":"up_listen starts","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.00852785Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile","task":"up_listen"}
{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.008553869Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Flush timeout: 0.5","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.00857591Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] 521f2004-29de-4d41-9494-447a49569d29 looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.073559912Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"0","component":"datafile"}
{"msg":"[0] Proc runs for 127.0.0.1:63639 in state New","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.073597872Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] 521f2004-29de-4d41-9494-447a49569d29 looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.073623702Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"1","component":"datafile"}
{"msg":"[1] Proc runs for 127.0.0.1:43949 in state New","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.073647762Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] 521f2004-29de-4d41-9494-447a49569d29 looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.073670604Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"2","component":"datafile"}
{"msg":"[2] Proc runs for 127.0.0.1:63175 in state New","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.073694056Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:36.073 INFO accepted connection from 127.0.0.1:33714, task: main
Sep 22 23:14:36.073 INFO accepted connection from 127.0.0.1:36045, task: main
Sep 22 23:14:36.073 INFO accepted connection from 127.0.0.1:62745, task: main
Sep 22 23:14:36.074 INFO Connection request from 521f2004-29de-4d41-9494-447a49569d29 with version 4, task: proc
Sep 22 23:14:36.074 INFO upstairs UpstairsConnection { upstairs_id: 521f2004-29de-4d41-9494-447a49569d29, session_id: 1e7e774e-f305-4cf5-a9d4-7b0b96e564fc, gen: 2 } connected, version 4, task: proc
Sep 22 23:14:36.074 INFO Connection request from 521f2004-29de-4d41-9494-447a49569d29 with version 4, task: proc
Sep 22 23:14:36.074 INFO upstairs UpstairsConnection { upstairs_id: 521f2004-29de-4d41-9494-447a49569d29, session_id: 1e7e774e-f305-4cf5-a9d4-7b0b96e564fc, gen: 2 } connected, version 4, task: proc
Sep 22 23:14:36.074 INFO Connection request from 521f2004-29de-4d41-9494-447a49569d29 with version 4, task: proc
Sep 22 23:14:36.074 INFO upstairs UpstairsConnection { upstairs_id: 521f2004-29de-4d41-9494-447a49569d29, session_id: 1e7e774e-f305-4cf5-a9d4-7b0b96e564fc, gen: 2 } connected, version 4, task: proc
{"msg":"[0] 521f2004-29de-4d41-9494-447a49569d29 (1e7e774e-f305-4cf5-a9d4-7b0b96e564fc) New New New ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.074382912Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.074419589Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] 521f2004-29de-4d41-9494-447a49569d29 (1e7e774e-f305-4cf5-a9d4-7b0b96e564fc) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.074446313Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.074468837Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] 521f2004-29de-4d41-9494-447a49569d29 (1e7e774e-f305-4cf5-a9d4-7b0b96e564fc) WaitActive WaitActive New ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.074493324Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.074515645Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:36.127 INFO Upstairs starts
Sep 22 23:14:36.127 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:14:36.128 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:14:36.128 INFO Crucible stats registered with UUID: 0d1229ab-bd36-49b6-95e1-d6287586b840
Sep 22 23:14:36.128 INFO Crucible 0d1229ab-bd36-49b6-95e1-d6287586b840 has session id: b6e522d3-fd64-4f28-884d-5ed051ece380
Sep 22 23:14:36.128 INFO [0] connecting to 127.0.0.1:38920, looper: 0
Sep 22 23:14:36.128 INFO [1] connecting to 127.0.0.1:44435, looper: 1
Sep 22 23:14:36.128 INFO [2] connecting to 127.0.0.1:51108, looper: 2
Sep 22 23:14:36.128 INFO up_listen starts, task: up_listen
Sep 22 23:14:36.128 INFO Wait for all three downstairs to come online
Sep 22 23:14:36.128 INFO Flush timeout: 0.5
Sep 22 23:14:36.128 INFO [0] 0d1229ab-bd36-49b6-95e1-d6287586b840 looper connected, looper: 0
Sep 22 23:14:36.128 INFO [0] Proc runs for 127.0.0.1:38920 in state New
Sep 22 23:14:36.128 INFO [1] 0d1229ab-bd36-49b6-95e1-d6287586b840 looper connected, looper: 1
Sep 22 23:14:36.128 INFO [1] Proc runs for 127.0.0.1:44435 in state New
Sep 22 23:14:36.128 INFO [2] 0d1229ab-bd36-49b6-95e1-d6287586b840 looper connected, looper: 2
Sep 22 23:14:36.128 INFO [2] Proc runs for 127.0.0.1:51108 in state New
Sep 22 23:14:36.128 INFO accepted connection from 127.0.0.1:33103, task: main
Sep 22 23:14:36.128 INFO accepted connection from 127.0.0.1:50432, task: main
Sep 22 23:14:36.128 INFO accepted connection from 127.0.0.1:35052, task: main
Sep 22 23:14:36.129 INFO Connection request from 0d1229ab-bd36-49b6-95e1-d6287586b840 with version 4, task: proc
Sep 22 23:14:36.129 INFO upstairs UpstairsConnection { upstairs_id: 0d1229ab-bd36-49b6-95e1-d6287586b840, session_id: 950a06ad-9a63-4ae1-a8e0-14de8b7aaece, gen: 2 } connected, version 4, task: proc
Sep 22 23:14:36.129 INFO Connection request from 0d1229ab-bd36-49b6-95e1-d6287586b840 with version 4, task: proc
Sep 22 23:14:36.129 INFO upstairs UpstairsConnection { upstairs_id: 0d1229ab-bd36-49b6-95e1-d6287586b840, session_id: 950a06ad-9a63-4ae1-a8e0-14de8b7aaece, gen: 2 } connected, version 4, task: proc
Sep 22 23:14:36.129 INFO Connection request from 0d1229ab-bd36-49b6-95e1-d6287586b840 with version 4, task: proc
Sep 22 23:14:36.129 INFO upstairs UpstairsConnection { upstairs_id: 0d1229ab-bd36-49b6-95e1-d6287586b840, session_id: 950a06ad-9a63-4ae1-a8e0-14de8b7aaece, gen: 2 } connected, version 4, task: proc
Sep 22 23:14:36.129 INFO [0] 0d1229ab-bd36-49b6-95e1-d6287586b840 (950a06ad-9a63-4ae1-a8e0-14de8b7aaece) New New New ds_transition to WaitActive
Sep 22 23:14:36.129 INFO [0] Transition from New to WaitActive
Sep 22 23:14:36.129 INFO [1] 0d1229ab-bd36-49b6-95e1-d6287586b840 (950a06ad-9a63-4ae1-a8e0-14de8b7aaece) WaitActive New New ds_transition to WaitActive
Sep 22 23:14:36.129 INFO [1] Transition from New to WaitActive
Sep 22 23:14:36.129 INFO [2] 0d1229ab-bd36-49b6-95e1-d6287586b840 (950a06ad-9a63-4ae1-a8e0-14de8b7aaece) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:14:36.129 INFO [2] Transition from New to WaitActive
The guest has requested activation
Sep 22 23:14:36.129 INFO 0d1229ab-bd36-49b6-95e1-d6287586b840 active request set
Sep 22 23:14:36.129 INFO [0] received activate with gen 2
Sep 22 23:14:36.129 INFO [0] client got ds_active_rx, promote! session 950a06ad-9a63-4ae1-a8e0-14de8b7aaece
Sep 22 23:14:36.130 INFO [1] received activate with gen 2
Sep 22 23:14:36.130 INFO [1] client got ds_active_rx, promote! session 950a06ad-9a63-4ae1-a8e0-14de8b7aaece
Sep 22 23:14:36.130 INFO [2] received activate with gen 2
Sep 22 23:14:36.130 INFO [2] client got ds_active_rx, promote! session 950a06ad-9a63-4ae1-a8e0-14de8b7aaece
Sep 22 23:14:36.130 WARN Attempting RW takeover from UpstairsConnection { upstairs_id: 0d1229ab-bd36-49b6-95e1-d6287586b840, session_id: 16060f7a-9986-4db1-95c8-74c4346b949d, gen: 1 } to UpstairsConnection { upstairs_id: 0d1229ab-bd36-49b6-95e1-d6287586b840, session_id: 950a06ad-9a63-4ae1-a8e0-14de8b7aaece, gen: 2 }
Sep 22 23:14:36.130 WARN Signaling to UpstairsConnection { upstairs_id: 0d1229ab-bd36-49b6-95e1-d6287586b840, session_id: 16060f7a-9986-4db1-95c8-74c4346b949d, gen: 1 } thread that UpstairsConnection { upstairs_id: 0d1229ab-bd36-49b6-95e1-d6287586b840, session_id: 950a06ad-9a63-4ae1-a8e0-14de8b7aaece, gen: 2 } is being promoted (read-write)
Sep 22 23:14:36.130 WARN Attempting RW takeover from UpstairsConnection { upstairs_id: 0d1229ab-bd36-49b6-95e1-d6287586b840, session_id: 16060f7a-9986-4db1-95c8-74c4346b949d, gen: 1 } to UpstairsConnection { upstairs_id: 0d1229ab-bd36-49b6-95e1-d6287586b840, session_id: 950a06ad-9a63-4ae1-a8e0-14de8b7aaece, gen: 2 }
Sep 22 23:14:36.130 WARN Signaling to UpstairsConnection { upstairs_id: 0d1229ab-bd36-49b6-95e1-d6287586b840, session_id: 16060f7a-9986-4db1-95c8-74c4346b949d, gen: 1 } thread that UpstairsConnection { upstairs_id: 0d1229ab-bd36-49b6-95e1-d6287586b840, session_id: 950a06ad-9a63-4ae1-a8e0-14de8b7aaece, gen: 2 } is being promoted (read-write)
Sep 22 23:14:36.130 WARN Attempting RW takeover from UpstairsConnection { upstairs_id: 0d1229ab-bd36-49b6-95e1-d6287586b840, session_id: 16060f7a-9986-4db1-95c8-74c4346b949d, gen: 1 } to UpstairsConnection { upstairs_id: 0d1229ab-bd36-49b6-95e1-d6287586b840, session_id: 950a06ad-9a63-4ae1-a8e0-14de8b7aaece, gen: 2 }
Sep 22 23:14:36.130 WARN Signaling to UpstairsConnection { upstairs_id: 0d1229ab-bd36-49b6-95e1-d6287586b840, session_id: 16060f7a-9986-4db1-95c8-74c4346b949d, gen: 1 } thread that UpstairsConnection { upstairs_id: 0d1229ab-bd36-49b6-95e1-d6287586b840, session_id: 950a06ad-9a63-4ae1-a8e0-14de8b7aaece, gen: 2 } is being promoted (read-write)
Sep 22 23:14:36.130 WARN Another upstairs UpstairsConnection { upstairs_id: 0d1229ab-bd36-49b6-95e1-d6287586b840, session_id: 950a06ad-9a63-4ae1-a8e0-14de8b7aaece, gen: 2 } promoted to active, shutting down connection for UpstairsConnection { upstairs_id: 0d1229ab-bd36-49b6-95e1-d6287586b840, session_id: 16060f7a-9986-4db1-95c8-74c4346b949d, gen: 1 }, task: main
Sep 22 23:14:36.130 INFO UpstairsConnection { upstairs_id: 0d1229ab-bd36-49b6-95e1-d6287586b840, session_id: 950a06ad-9a63-4ae1-a8e0-14de8b7aaece, gen: 2 } is now active (read-write)
Sep 22 23:14:36.130 WARN Another upstairs UpstairsConnection { upstairs_id: 0d1229ab-bd36-49b6-95e1-d6287586b840, session_id: 950a06ad-9a63-4ae1-a8e0-14de8b7aaece, gen: 2 } promoted to active, shutting down connection for UpstairsConnection { upstairs_id: 0d1229ab-bd36-49b6-95e1-d6287586b840, session_id: 16060f7a-9986-4db1-95c8-74c4346b949d, gen: 1 }, task: main
Sep 22 23:14:36.131 INFO UpstairsConnection { upstairs_id: 0d1229ab-bd36-49b6-95e1-d6287586b840, session_id: 950a06ad-9a63-4ae1-a8e0-14de8b7aaece, gen: 2 } is now active (read-write)
Sep 22 23:14:36.131 WARN Another upstairs UpstairsConnection { upstairs_id: 0d1229ab-bd36-49b6-95e1-d6287586b840, session_id: 950a06ad-9a63-4ae1-a8e0-14de8b7aaece, gen: 2 } promoted to active, shutting down connection for UpstairsConnection { upstairs_id: 0d1229ab-bd36-49b6-95e1-d6287586b840, session_id: 16060f7a-9986-4db1-95c8-74c4346b949d, gen: 1 }, task: main
Sep 22 23:14:36.131 INFO UpstairsConnection { upstairs_id: 0d1229ab-bd36-49b6-95e1-d6287586b840, session_id: 950a06ad-9a63-4ae1-a8e0-14de8b7aaece, gen: 2 } is now active (read-write)
Sep 22 23:14:36.131 INFO connection (127.0.0.1:59183): all done
Sep 22 23:14:36.131 INFO connection (127.0.0.1:49569): all done
Sep 22 23:14:36.131 INFO connection (127.0.0.1:64659): all done
{"msg":"[0] 0d1229ab-bd36-49b6-95e1-d6287586b840 (16060f7a-9986-4db1-95c8-74c4346b949d) cmd_loop saw YouAreNoLongerActive 0d1229ab-bd36-49b6-95e1-d6287586b840 950a06ad-9a63-4ae1-a8e0-14de8b7aaece 2","v":0,"name":"crucible-pantry","level":50,"time":"2023-09-22T23:14:36.131792703Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] 0d1229ab-bd36-49b6-95e1-d6287586b840 (16060f7a-9986-4db1-95c8-74c4346b949d) Active Active Active ds_transition to Disabled","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.131837212Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] Transition from Active to Disabled","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.131862195Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"0d1229ab-bd36-49b6-95e1-d6287586b840 set inactive, session 16060f7a-9986-4db1-95c8-74c4346b949d","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.131886229Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"127.0.0.1:38920: proc: Generation number is too low: saw YouAreNoLongerActive with larger gen 2 than ours 1","v":0,"name":"crucible-pantry","level":50,"time":"2023-09-22T23:14:36.13191005Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"0","component":"datafile"}
{"msg":"[0] 0d1229ab-bd36-49b6-95e1-d6287586b840 Gone missing, transition from Disabled to Disconnected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.131935304Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] 0d1229ab-bd36-49b6-95e1-d6287586b840 connection to 127.0.0.1:38920 closed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.131958207Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"0","component":"datafile"}
{"msg":"[1] 0d1229ab-bd36-49b6-95e1-d6287586b840 (16060f7a-9986-4db1-95c8-74c4346b949d) cmd_loop saw YouAreNoLongerActive 0d1229ab-bd36-49b6-95e1-d6287586b840 950a06ad-9a63-4ae1-a8e0-14de8b7aaece 2","v":0,"name":"crucible-pantry","level":50,"time":"2023-09-22T23:14:36.13198491Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] 0d1229ab-bd36-49b6-95e1-d6287586b840 (16060f7a-9986-4db1-95c8-74c4346b949d) Disconnected Active Active ds_transition to Disabled","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.132009248Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] Transition from Active to Disabled","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.132030823Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"0d1229ab-bd36-49b6-95e1-d6287586b840 set inactive, session 16060f7a-9986-4db1-95c8-74c4346b949d","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.132059692Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"127.0.0.1:44435: proc: Generation number is too low: saw YouAreNoLongerActive with larger gen 2 than ours 1","v":0,"name":"crucible-pantry","level":50,"time":"2023-09-22T23:14:36.132093496Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"1","component":"datafile"}
{"msg":"[1] 0d1229ab-bd36-49b6-95e1-d6287586b840 Gone missing, transition from Disabled to Disconnected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.132121605Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] 0d1229ab-bd36-49b6-95e1-d6287586b840 connection to 127.0.0.1:44435 closed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.132144874Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"1","component":"datafile"}
{"msg":"[2] 0d1229ab-bd36-49b6-95e1-d6287586b840 (16060f7a-9986-4db1-95c8-74c4346b949d) cmd_loop saw YouAreNoLongerActive 0d1229ab-bd36-49b6-95e1-d6287586b840 950a06ad-9a63-4ae1-a8e0-14de8b7aaece 2","v":0,"name":"crucible-pantry","level":50,"time":"2023-09-22T23:14:36.132171555Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] 0d1229ab-bd36-49b6-95e1-d6287586b840 (16060f7a-9986-4db1-95c8-74c4346b949d) Disconnected Disconnected Active ds_transition to Disabled","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.132195928Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] Transition from Active to Disabled","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.132217825Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"0d1229ab-bd36-49b6-95e1-d6287586b840 set inactive, session 16060f7a-9986-4db1-95c8-74c4346b949d","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.132241061Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"127.0.0.1:51108: proc: Generation number is too low: saw YouAreNoLongerActive with larger gen 2 than ours 1","v":0,"name":"crucible-pantry","level":50,"time":"2023-09-22T23:14:36.132264468Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"2","component":"datafile"}
{"msg":"[2] 0d1229ab-bd36-49b6-95e1-d6287586b840 Gone missing, transition from Disabled to Disconnected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.132288608Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] 0d1229ab-bd36-49b6-95e1-d6287586b840 connection to 127.0.0.1:51108 closed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.132311149Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"2","component":"datafile"}
{"msg":"[0] pm_task rx.recv() is None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:14:36.132333621Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] 127.0.0.1:38920 task reports connection:false","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.132355651Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"0d1229ab-bd36-49b6-95e1-d6287586b840 Disconnected Disconnected Disconnected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.132378224Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] 127.0.0.1:38920 task reports offline","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.132399957Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:36.132 INFO [0] downstairs client at 127.0.0.1:38920 has UUID b38dafac-2da7-4558-aae1-959c564193b2
{"msg":"[1] 127.0.0.1:44435 task reports connection:false","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.132425982Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:36.132 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 512, shift: 9 }, extent_count: 188, uuid: b38dafac-2da7-4558-aae1-959c564193b2, encrypted: true, database_read_version: 1, database_write_version: 1 }
{"msg":"0d1229ab-bd36-49b6-95e1-d6287586b840 Disconnected Disconnected Disconnected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.132449675Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] 127.0.0.1:44435 task reports offline","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.132471671Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:36.132 INFO 0d1229ab-bd36-49b6-95e1-d6287586b840 WaitActive WaitActive WaitActive
{"msg":"[2] 127.0.0.1:51108 task reports connection:false","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.132503632Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"0d1229ab-bd36-49b6-95e1-d6287586b840 Disconnected Disconnected Disconnected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.132531853Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:36.132 INFO [1] downstairs client at 127.0.0.1:44435 has UUID 14399b3b-6e6e-4995-afe0-74c1441f0833
Sep 22 23:14:36.132 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 512, shift: 9 }, extent_count: 188, uuid: 14399b3b-6e6e-4995-afe0-74c1441f0833, encrypted: true, database_read_version: 1, database_write_version: 1 }
{"msg":"[2] 127.0.0.1:51108 task reports offline","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.132553992Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] pm_task rx.recv() is None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:14:36.13257571Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:36.132 INFO 0d1229ab-bd36-49b6-95e1-d6287586b840 WaitActive WaitActive WaitActive
{"msg":"[2] pm_task rx.recv() is None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:14:36.132597057Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:36.132 INFO [2] downstairs client at 127.0.0.1:51108 has UUID 9da41dd6-4cdf-4b03-8df1-d13ee59384a3
Sep 22 23:14:36.132 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 512, shift: 9 }, extent_count: 188, uuid: 9da41dd6-4cdf-4b03-8df1-d13ee59384a3, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:14:36.132 INFO 0d1229ab-bd36-49b6-95e1-d6287586b840 WaitActive WaitActive WaitActive
Sep 22 23:14:36.140 INFO Current flush_numbers [0..12]: [1, 1, 2, 2, 3, 3, 4, 4, 0, 0, 0, 0]
Sep 22 23:14:36.141 INFO Current flush_numbers [0..12]: [1, 1, 2, 2, 3, 3, 4, 4, 0, 0, 0, 0]
Sep 22 23:14:36.142 INFO Current flush_numbers [0..12]: [1, 1, 2, 2, 3, 3, 4, 4, 0, 0, 0, 0]
Sep 22 23:14:36.154 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:36.155 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:36.156 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:36.156 INFO [0] 0d1229ab-bd36-49b6-95e1-d6287586b840 (950a06ad-9a63-4ae1-a8e0-14de8b7aaece) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:36.156 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:14:36.156 WARN [0] new RM replaced this: None
Sep 22 23:14:36.157 INFO [0] Starts reconcile loop
Sep 22 23:14:36.157 INFO [1] 0d1229ab-bd36-49b6-95e1-d6287586b840 (950a06ad-9a63-4ae1-a8e0-14de8b7aaece) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:14:36.157 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:14:36.157 WARN [1] new RM replaced this: None
Sep 22 23:14:36.157 INFO [1] Starts reconcile loop
Sep 22 23:14:36.157 INFO [2] 0d1229ab-bd36-49b6-95e1-d6287586b840 (950a06ad-9a63-4ae1-a8e0-14de8b7aaece) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:14:36.157 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:14:36.157 WARN [2] new RM replaced this: None
Sep 22 23:14:36.157 INFO [2] Starts reconcile loop
Sep 22 23:14:36.157 INFO [0] 127.0.0.1:38920 task reports connection:true
Sep 22 23:14:36.157 INFO 0d1229ab-bd36-49b6-95e1-d6287586b840 WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:14:36.157 INFO [0]R flush_numbers[0..12]: [1, 1, 2, 2, 3, 3, 4, 4, 0, 0, 0, 0]
Sep 22 23:14:36.157 INFO [0]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0]
Sep 22 23:14:36.157 INFO [0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]
Sep 22 23:14:36.157 INFO [1]R flush_numbers[0..12]: [1, 1, 2, 2, 3, 3, 4, 4, 0, 0, 0, 0]
Sep 22 23:14:36.157 INFO [1]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0]
Sep 22 23:14:36.157 INFO [1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]
Sep 22 23:14:36.157 INFO [2]R flush_numbers[0..12]: [1, 1, 2, 2, 3, 3, 4, 4, 0, 0, 0, 0]
Sep 22 23:14:36.157 INFO [2]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0]
Sep 22 23:14:36.157 INFO [2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]
Sep 22 23:14:36.157 INFO Max found gen is 2
Sep 22 23:14:36.157 INFO Generation requested: 2 >= found:2
Sep 22 23:14:36.157 INFO Next flush: 5
Sep 22 23:14:36.157 INFO All extents match
Sep 22 23:14:36.157 INFO No downstairs repair required
Sep 22 23:14:36.157 INFO No initial repair work was required
Sep 22 23:14:36.157 INFO Set Downstairs and Upstairs active
Sep 22 23:14:36.157 INFO 0d1229ab-bd36-49b6-95e1-d6287586b840 is now active with session: 950a06ad-9a63-4ae1-a8e0-14de8b7aaece
Sep 22 23:14:36.157 INFO 0d1229ab-bd36-49b6-95e1-d6287586b840 Set Active after no repair
Sep 22 23:14:36.157 INFO Notify all downstairs, region set compare is done.
Sep 22 23:14:36.157 INFO Set check for repair
Sep 22 23:14:36.157 INFO [1] 127.0.0.1:44435 task reports connection:true
Sep 22 23:14:36.158 INFO 0d1229ab-bd36-49b6-95e1-d6287586b840 Active Active Active
Sep 22 23:14:36.158 INFO Set check for repair
Sep 22 23:14:36.158 INFO [2] 127.0.0.1:51108 task reports connection:true
Sep 22 23:14:36.158 INFO 0d1229ab-bd36-49b6-95e1-d6287586b840 Active Active Active
Sep 22 23:14:36.158 INFO Set check for repair
Sep 22 23:14:36.158 INFO [0] received reconcile message
Sep 22 23:14:36.158 INFO [0] All repairs completed, exit
Sep 22 23:14:36.158 INFO [0] Starts cmd_loop
Sep 22 23:14:36.158 INFO [1] received reconcile message
Sep 22 23:14:36.158 INFO [1] All repairs completed, exit
Sep 22 23:14:36.158 INFO [1] Starts cmd_loop
Sep 22 23:14:36.158 INFO [2] received reconcile message
Sep 22 23:14:36.158 INFO [2] All repairs completed, exit
Sep 22 23:14:36.158 INFO [2] Starts cmd_loop
The guest has finished waiting for activation
Sep 22 23:14:36.160 DEBG IO Read  1000 has deps []
Sep 22 23:14:36.197 DEBG Read      :1000 deps:[] res:true
Sep 22 23:14:36.228 DEBG IO Write 1016 has deps [JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)]
Sep 22 23:14:36.228 DEBG up_ds_listen was notified
Sep 22 23:14:36.228 DEBG up_ds_listen process 1016
Sep 22 23:14:36.229 DEBG [A] ack job 1016:17, : downstairs
Sep 22 23:14:36.229 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:36.229 DEBG IO Flush 1017 has deps [JobId(1016), JobId(1015), JobId(1014)]
Sep 22 23:14:36.253 DEBG Read      :1000 deps:[] res:true
Sep 22 23:14:36.312 DEBG Read      :1000 deps:[] res:true
{"msg":"volume fc5b0c26-5d3b-433c-b382-b2a002415eea constructed ok","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.349141779Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
The guest has requested activation
{"msg":"521f2004-29de-4d41-9494-447a49569d29 active request set","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.349325626Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] received activate with gen 2","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.349374757Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] client got ds_active_rx, promote! session 1e7e774e-f305-4cf5-a9d4-7b0b96e564fc","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.349419001Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] received activate with gen 2","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.349456216Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] client got ds_active_rx, promote! session 1e7e774e-f305-4cf5-a9d4-7b0b96e564fc","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.349493438Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:36.349 WARN Attempting RW takeover from UpstairsConnection { upstairs_id: 521f2004-29de-4d41-9494-447a49569d29, session_id: ca92c31b-eb28-41e5-843d-26b8bd24de02, gen: 1 } to UpstairsConnection { upstairs_id: 521f2004-29de-4d41-9494-447a49569d29, session_id: 1e7e774e-f305-4cf5-a9d4-7b0b96e564fc, gen: 2 }
{"msg":"[2] received activate with gen 2","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.349528734Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:36.349 WARN Signaling to UpstairsConnection { upstairs_id: 521f2004-29de-4d41-9494-447a49569d29, session_id: ca92c31b-eb28-41e5-843d-26b8bd24de02, gen: 1 } thread that UpstairsConnection { upstairs_id: 521f2004-29de-4d41-9494-447a49569d29, session_id: 1e7e774e-f305-4cf5-a9d4-7b0b96e564fc, gen: 2 } is being promoted (read-write)
{"msg":"[2] client got ds_active_rx, promote! session 1e7e774e-f305-4cf5-a9d4-7b0b96e564fc","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.349566805Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:36.349 WARN Attempting RW takeover from UpstairsConnection { upstairs_id: 521f2004-29de-4d41-9494-447a49569d29, session_id: ca92c31b-eb28-41e5-843d-26b8bd24de02, gen: 1 } to UpstairsConnection { upstairs_id: 521f2004-29de-4d41-9494-447a49569d29, session_id: 1e7e774e-f305-4cf5-a9d4-7b0b96e564fc, gen: 2 }
Sep 22 23:14:36.349 WARN Signaling to UpstairsConnection { upstairs_id: 521f2004-29de-4d41-9494-447a49569d29, session_id: ca92c31b-eb28-41e5-843d-26b8bd24de02, gen: 1 } thread that UpstairsConnection { upstairs_id: 521f2004-29de-4d41-9494-447a49569d29, session_id: 1e7e774e-f305-4cf5-a9d4-7b0b96e564fc, gen: 2 } is being promoted (read-write)
Sep 22 23:14:36.349 WARN Attempting RW takeover from UpstairsConnection { upstairs_id: 521f2004-29de-4d41-9494-447a49569d29, session_id: ca92c31b-eb28-41e5-843d-26b8bd24de02, gen: 1 } to UpstairsConnection { upstairs_id: 521f2004-29de-4d41-9494-447a49569d29, session_id: 1e7e774e-f305-4cf5-a9d4-7b0b96e564fc, gen: 2 }
Sep 22 23:14:36.349 WARN Signaling to UpstairsConnection { upstairs_id: 521f2004-29de-4d41-9494-447a49569d29, session_id: ca92c31b-eb28-41e5-843d-26b8bd24de02, gen: 1 } thread that UpstairsConnection { upstairs_id: 521f2004-29de-4d41-9494-447a49569d29, session_id: 1e7e774e-f305-4cf5-a9d4-7b0b96e564fc, gen: 2 } is being promoted (read-write)
Sep 22 23:14:36.350 WARN Another upstairs UpstairsConnection { upstairs_id: 521f2004-29de-4d41-9494-447a49569d29, session_id: 1e7e774e-f305-4cf5-a9d4-7b0b96e564fc, gen: 2 } promoted to active, shutting down connection for UpstairsConnection { upstairs_id: 521f2004-29de-4d41-9494-447a49569d29, session_id: ca92c31b-eb28-41e5-843d-26b8bd24de02, gen: 1 }, task: main
Sep 22 23:14:36.350 INFO UpstairsConnection { upstairs_id: 521f2004-29de-4d41-9494-447a49569d29, session_id: 1e7e774e-f305-4cf5-a9d4-7b0b96e564fc, gen: 2 } is now active (read-write)
Sep 22 23:14:36.350 WARN Another upstairs UpstairsConnection { upstairs_id: 521f2004-29de-4d41-9494-447a49569d29, session_id: 1e7e774e-f305-4cf5-a9d4-7b0b96e564fc, gen: 2 } promoted to active, shutting down connection for UpstairsConnection { upstairs_id: 521f2004-29de-4d41-9494-447a49569d29, session_id: ca92c31b-eb28-41e5-843d-26b8bd24de02, gen: 1 }, task: main
Sep 22 23:14:36.350 INFO UpstairsConnection { upstairs_id: 521f2004-29de-4d41-9494-447a49569d29, session_id: 1e7e774e-f305-4cf5-a9d4-7b0b96e564fc, gen: 2 } is now active (read-write)
Sep 22 23:14:36.350 WARN Another upstairs UpstairsConnection { upstairs_id: 521f2004-29de-4d41-9494-447a49569d29, session_id: 1e7e774e-f305-4cf5-a9d4-7b0b96e564fc, gen: 2 } promoted to active, shutting down connection for UpstairsConnection { upstairs_id: 521f2004-29de-4d41-9494-447a49569d29, session_id: ca92c31b-eb28-41e5-843d-26b8bd24de02, gen: 1 }, task: main
Sep 22 23:14:36.350 INFO UpstairsConnection { upstairs_id: 521f2004-29de-4d41-9494-447a49569d29, session_id: 1e7e774e-f305-4cf5-a9d4-7b0b96e564fc, gen: 2 } is now active (read-write)
Sep 22 23:14:36.350 INFO connection (127.0.0.1:52189): all done
Sep 22 23:14:36.350 INFO connection (127.0.0.1:54318): all done
Sep 22 23:14:36.350 INFO connection (127.0.0.1:37634): all done
Sep 22 23:14:36.350 ERRO [0] 521f2004-29de-4d41-9494-447a49569d29 (ca92c31b-eb28-41e5-843d-26b8bd24de02) cmd_loop saw YouAreNoLongerActive 521f2004-29de-4d41-9494-447a49569d29 1e7e774e-f305-4cf5-a9d4-7b0b96e564fc 2
Sep 22 23:14:36.350 INFO [0] 521f2004-29de-4d41-9494-447a49569d29 (ca92c31b-eb28-41e5-843d-26b8bd24de02) Active Active Active ds_transition to Disabled
Sep 22 23:14:36.351 INFO [0] Transition from Active to Disabled
Sep 22 23:14:36.351 INFO 521f2004-29de-4d41-9494-447a49569d29 set inactive, session ca92c31b-eb28-41e5-843d-26b8bd24de02
Sep 22 23:14:36.351 ERRO 127.0.0.1:63639: proc: Generation number is too low: saw YouAreNoLongerActive with larger gen 2 than ours 1, looper: 0
Sep 22 23:14:36.351 INFO [0] 521f2004-29de-4d41-9494-447a49569d29 Gone missing, transition from Disabled to Disconnected
Sep 22 23:14:36.351 INFO [0] 521f2004-29de-4d41-9494-447a49569d29 connection to 127.0.0.1:63639 closed, looper: 0
Sep 22 23:14:36.351 ERRO [1] 521f2004-29de-4d41-9494-447a49569d29 (ca92c31b-eb28-41e5-843d-26b8bd24de02) cmd_loop saw YouAreNoLongerActive 521f2004-29de-4d41-9494-447a49569d29 1e7e774e-f305-4cf5-a9d4-7b0b96e564fc 2
Sep 22 23:14:36.351 INFO [1] 521f2004-29de-4d41-9494-447a49569d29 (ca92c31b-eb28-41e5-843d-26b8bd24de02) Disconnected Active Active ds_transition to Disabled
Sep 22 23:14:36.351 INFO [1] Transition from Active to Disabled
Sep 22 23:14:36.351 INFO 521f2004-29de-4d41-9494-447a49569d29 set inactive, session ca92c31b-eb28-41e5-843d-26b8bd24de02
Sep 22 23:14:36.351 ERRO 127.0.0.1:43949: proc: Generation number is too low: saw YouAreNoLongerActive with larger gen 2 than ours 1, looper: 1
Sep 22 23:14:36.351 INFO [1] 521f2004-29de-4d41-9494-447a49569d29 Gone missing, transition from Disabled to Disconnected
Sep 22 23:14:36.351 INFO [1] 521f2004-29de-4d41-9494-447a49569d29 connection to 127.0.0.1:43949 closed, looper: 1
Sep 22 23:14:36.351 ERRO [2] 521f2004-29de-4d41-9494-447a49569d29 (ca92c31b-eb28-41e5-843d-26b8bd24de02) cmd_loop saw YouAreNoLongerActive 521f2004-29de-4d41-9494-447a49569d29 1e7e774e-f305-4cf5-a9d4-7b0b96e564fc 2
Sep 22 23:14:36.351 INFO [2] 521f2004-29de-4d41-9494-447a49569d29 (ca92c31b-eb28-41e5-843d-26b8bd24de02) Disconnected Disconnected Active ds_transition to Disabled
Sep 22 23:14:36.351 INFO [2] Transition from Active to Disabled
Sep 22 23:14:36.351 INFO 521f2004-29de-4d41-9494-447a49569d29 set inactive, session ca92c31b-eb28-41e5-843d-26b8bd24de02
Sep 22 23:14:36.351 ERRO 127.0.0.1:63175: proc: Generation number is too low: saw YouAreNoLongerActive with larger gen 2 than ours 1, looper: 2
Sep 22 23:14:36.351 INFO [2] 521f2004-29de-4d41-9494-447a49569d29 Gone missing, transition from Disabled to Disconnected
Sep 22 23:14:36.351 INFO [2] 521f2004-29de-4d41-9494-447a49569d29 connection to 127.0.0.1:63175 closed, looper: 2
Sep 22 23:14:36.351 WARN [0] pm_task rx.recv() is None
Sep 22 23:14:36.351 INFO [0] 127.0.0.1:63639 task reports connection:false
Sep 22 23:14:36.351 INFO 521f2004-29de-4d41-9494-447a49569d29 Disconnected Disconnected Disconnected
Sep 22 23:14:36.351 INFO [0] 127.0.0.1:63639 task reports offline
Sep 22 23:14:36.351 INFO [1] 127.0.0.1:43949 task reports connection:false
Sep 22 23:14:36.351 INFO 521f2004-29de-4d41-9494-447a49569d29 Disconnected Disconnected Disconnected
Sep 22 23:14:36.351 INFO [1] 127.0.0.1:43949 task reports offline
Sep 22 23:14:36.351 INFO [2] 127.0.0.1:63175 task reports connection:false
Sep 22 23:14:36.351 INFO 521f2004-29de-4d41-9494-447a49569d29 Disconnected Disconnected Disconnected
Sep 22 23:14:36.351 INFO [2] 127.0.0.1:63175 task reports offline
Sep 22 23:14:36.351 WARN [1] pm_task rx.recv() is None
Sep 22 23:14:36.351 WARN [2] pm_task rx.recv() is None
{"msg":"[0] downstairs client at 127.0.0.1:63639 has UUID bafba53d-f7d4-4132-95f6-92b6a45487a9","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.352242605Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 512, shift: 9 }, extent_count: 188, uuid: bafba53d-f7d4-4132-95f6-92b6a45487a9, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.35230223Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"521f2004-29de-4d41-9494-447a49569d29 WaitActive WaitActive WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.352341789Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] downstairs client at 127.0.0.1:43949 has UUID 1e7a075d-e51e-4361-bdf9-32f916f89647","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.352385908Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 512, shift: 9 }, extent_count: 188, uuid: 1e7a075d-e51e-4361-bdf9-32f916f89647, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.352432286Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"521f2004-29de-4d41-9494-447a49569d29 WaitActive WaitActive WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.352469137Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] downstairs client at 127.0.0.1:63175 has UUID b89ddae9-a405-4d4a-865d-e95f1fd0a715","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.352506539Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 512, shift: 9 }, extent_count: 188, uuid: b89ddae9-a405-4d4a-865d-e95f1fd0a715, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.352550255Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"521f2004-29de-4d41-9494-447a49569d29 WaitActive WaitActive WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.35258698Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:36.360 INFO Current flush_numbers [0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:14:36.361 INFO Current flush_numbers [0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:14:36.362 INFO Current flush_numbers [0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:14:36.375 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:36.376 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:36.376 INFO Downstairs has completed Negotiation, task: proc
{"msg":"[0] 521f2004-29de-4d41-9494-447a49569d29 (1e7e774e-f305-4cf5-a9d4-7b0b96e564fc) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.377295404Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.377351793Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:14:36.37739217Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.37743117Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] 521f2004-29de-4d41-9494-447a49569d29 (1e7e774e-f305-4cf5-a9d4-7b0b96e564fc) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.377557271Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.377603685Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:14:36.377641937Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.377681719Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] 521f2004-29de-4d41-9494-447a49569d29 (1e7e774e-f305-4cf5-a9d4-7b0b96e564fc) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.377772336Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.37781688Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:14:36.37785286Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.377887367Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] 127.0.0.1:63639 task reports connection:true","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.377926614Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"521f2004-29de-4d41-9494-447a49569d29 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.377972127Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.378009859Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.378046686Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
The guest has finished waiting for activation
{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.378095467Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.37813417Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.37817048Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.378214316Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.378253794Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.37829054Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.378328451Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Max found gen is 1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.378362395Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Generation requested: 2 >= found:1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.378397123Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Next flush: 1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.378431237Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"All extents match","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.378467122Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"No downstairs repair required","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.378506487Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"No initial repair work was required","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.378542202Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.378576938Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"521f2004-29de-4d41-9494-447a49569d29 is now active with session: 1e7e774e-f305-4cf5-a9d4-7b0b96e564fc","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.378614733Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"521f2004-29de-4d41-9494-447a49569d29 Set Active after no repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.37865103Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.378686319Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Set check for repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.378725954Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] 127.0.0.1:43949 task reports connection:true","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.378767422Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"521f2004-29de-4d41-9494-447a49569d29 Active Active Active","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.378805812Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Set check for repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.378840149Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] 127.0.0.1:63175 task reports connection:true","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.37887569Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"521f2004-29de-4d41-9494-447a49569d29 Active Active Active","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.378911526Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Set check for repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.378945633Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] received reconcile message","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.378979903Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.379017118Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.379055914Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] received reconcile message","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.37909169Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.379126109Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.379160332Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] received reconcile message","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.379194717Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.379229096Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.379263501Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"volume fc5b0c26-5d3b-433c-b382-b2a002415eea activated ok","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.379301968Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"volume fc5b0c26-5d3b-433c-b382-b2a002415eea constructed and inserted ok","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.379342363Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.379382743Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/fc5b0c26-5d3b-433c-b382-b2a002415eea","method":"POST","req_id":"6e1960e8-b6da-4b37-898f-cb0ed7d8b107","remote_addr":"127.0.0.1:43906","local_addr":"127.0.0.1:49411","component":"dropshot","latency_us":370911,"response_code":"200"}
{"msg":"Scrub check for fc5b0c26-5d3b-433c-b382-b2a002415eea","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.379456635Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Scrub for fc5b0c26-5d3b-433c-b382-b2a002415eea begins","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.379494658Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Scrub with total_size:1966080 block_size:512","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.379530254Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Scrubs from block 0 to 3840 in (256) 131072 size IOs pm:0","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.379567832Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.379606725Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/fc5b0c26-5d3b-433c-b382-b2a002415eea/scrub","method":"POST","req_id":"86c01c82-da45-4c1a-ba27-6bcbb1a7137e","remote_addr":"127.0.0.1:43906","local_addr":"127.0.0.1:49411","component":"dropshot","latency_us":491,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.380320518Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/job/9582738c-4762-4d3d-9423-661cc66a6ac0/is_finished","method":"GET","req_id":"0fcc339e-b13c-47ab-b5fc-4d74dab524ed","remote_addr":"127.0.0.1:43906","local_addr":"127.0.0.1:49411","component":"dropshot","latency_us":166,"response_code":"200"}
Sep 22 23:14:36.564 DEBG IO Write 1018 has deps [JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)]
Sep 22 23:14:36.564 DEBG up_ds_listen was notified
Sep 22 23:14:36.564 DEBG up_ds_listen process 1018
Sep 22 23:14:36.564 DEBG [A] ack job 1018:19, : downstairs
Sep 22 23:14:36.564 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:36.596 DEBG Write     :1000 deps:[] res:true
{"msg":"Scrub at offset 256/3840 sp:256","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.671089403Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Scrub at offset 512/3840 sp:512","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:36.900818585Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:36.930 DEBG IO Write 1019 has deps [JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)]
Sep 22 23:14:36.930 DEBG up_ds_listen was notified
Sep 22 23:14:36.930 DEBG up_ds_listen process 1019
Sep 22 23:14:36.930 DEBG [A] ack job 1019:20, : downstairs
Sep 22 23:14:36.930 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:36.930 DEBG IO Flush 1020 has deps [JobId(1019), JobId(1018), JobId(1017)]
Sep 22 23:14:37.015 DEBG Flush     :1002 extent_limit None deps:[JobId(1001), JobId(1000)] res:true f:1 g:2
Sep 22 23:14:37.016 DEBG Flush     :1002 extent_limit None deps:[JobId(1001), JobId(1000)] res:true f:1 g:2
Sep 22 23:14:37.016 DEBG Flush     :1002 extent_limit None deps:[JobId(1001), JobId(1000)] res:true f:1 g:2
{"msg":"Scrub at offset 768/3840 sp:768","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:37.129720528Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:37.267 DEBG [0] Read AckReady 1000, : downstairs
Sep 22 23:14:37.271 DEBG IO Write 1021 has deps [JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)]
Sep 22 23:14:37.271 DEBG up_ds_listen was notified
Sep 22 23:14:37.271 DEBG up_ds_listen process 1021
Sep 22 23:14:37.271 DEBG [A] ack job 1021:22, : downstairs
Sep 22 23:14:37.271 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:37.362 INFO [0] 521f2004-29de-4d41-9494-447a49569d29 looper connected, looper: 0
{"msg":"Scrub at offset 1024/3840 sp:1024","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:37.362951926Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:37.363 INFO [0] Proc runs for 127.0.0.1:63639 in state Disconnected
Sep 22 23:14:37.363 INFO [1] 521f2004-29de-4d41-9494-447a49569d29 looper connected, looper: 1
Sep 22 23:14:37.363 INFO [1] Proc runs for 127.0.0.1:43949 in state Disconnected
Sep 22 23:14:37.363 INFO [2] 521f2004-29de-4d41-9494-447a49569d29 looper connected, looper: 2
Sep 22 23:14:37.363 INFO [2] Proc runs for 127.0.0.1:63175 in state Disconnected
Sep 22 23:14:37.363 INFO accepted connection from 127.0.0.1:57238, task: main
Sep 22 23:14:37.363 INFO accepted connection from 127.0.0.1:59756, task: main
Sep 22 23:14:37.363 INFO accepted connection from 127.0.0.1:38472, task: main
Sep 22 23:14:37.363 INFO Connection request from 521f2004-29de-4d41-9494-447a49569d29 with version 4, task: proc
Sep 22 23:14:37.363 INFO upstairs UpstairsConnection { upstairs_id: 521f2004-29de-4d41-9494-447a49569d29, session_id: ca92c31b-eb28-41e5-843d-26b8bd24de02, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:37.363 INFO Connection request from 521f2004-29de-4d41-9494-447a49569d29 with version 4, task: proc
Sep 22 23:14:37.363 INFO upstairs UpstairsConnection { upstairs_id: 521f2004-29de-4d41-9494-447a49569d29, session_id: ca92c31b-eb28-41e5-843d-26b8bd24de02, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:37.363 INFO Connection request from 521f2004-29de-4d41-9494-447a49569d29 with version 4, task: proc
Sep 22 23:14:37.363 INFO upstairs UpstairsConnection { upstairs_id: 521f2004-29de-4d41-9494-447a49569d29, session_id: ca92c31b-eb28-41e5-843d-26b8bd24de02, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:37.363 INFO [0] 521f2004-29de-4d41-9494-447a49569d29 (ca92c31b-eb28-41e5-843d-26b8bd24de02) Disconnected Disconnected Disconnected ds_transition to WaitActive
Sep 22 23:14:37.363 INFO [0] Transition from Disconnected to WaitActive
Sep 22 23:14:37.363 INFO [1] 521f2004-29de-4d41-9494-447a49569d29 (ca92c31b-eb28-41e5-843d-26b8bd24de02) WaitActive Disconnected Disconnected ds_transition to WaitActive
Sep 22 23:14:37.363 INFO [1] Transition from Disconnected to WaitActive
Sep 22 23:14:37.363 INFO [2] 521f2004-29de-4d41-9494-447a49569d29 (ca92c31b-eb28-41e5-843d-26b8bd24de02) WaitActive WaitActive Disconnected ds_transition to WaitActive
Sep 22 23:14:37.363 INFO [2] Transition from Disconnected to WaitActive
{"msg":"Checking if live repair is needed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:37.379533456Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"No Live Repair required at this time","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:37.379572659Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:37.382186544Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/job/9582738c-4762-4d3d-9423-661cc66a6ac0/is_finished","method":"GET","req_id":"dd4bfebc-ba03-4174-b9cf-3f9167bfc939","remote_addr":"127.0.0.1:43906","local_addr":"127.0.0.1:49411","component":"dropshot","latency_us":215,"response_code":"200"}
Sep 22 23:14:37.516 DEBG Flush     :1005 extent_limit None deps:[JobId(1004), JobId(1003)] res:true f:2 g:2
Sep 22 23:14:37.516 DEBG Flush     :1005 extent_limit None deps:[JobId(1004), JobId(1003)] res:true f:2 g:2
Sep 22 23:14:37.517 DEBG Flush     :1005 extent_limit None deps:[JobId(1004), JobId(1003)] res:true f:2 g:2
{"msg":"Scrub at offset 1280/3840 sp:1280","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:37.614777454Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:37.620 DEBG IO Write 1022 has deps [JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)]
Sep 22 23:14:37.620 DEBG up_ds_listen was notified
Sep 22 23:14:37.620 DEBG up_ds_listen process 1022
Sep 22 23:14:37.620 DEBG [A] ack job 1022:23, : downstairs
Sep 22 23:14:37.620 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:37.621 DEBG IO Flush 1023 has deps [JobId(1022), JobId(1021), JobId(1020)]
{"msg":"Scrub at offset 1536/3840 sp:1536","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:37.84345624Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:37.900 DEBG [1] Read already AckReady 1000, : downstairs
Sep 22 23:14:37.961 DEBG IO Write 1024 has deps [JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)]
Sep 22 23:14:37.961 DEBG up_ds_listen was notified
Sep 22 23:14:37.961 DEBG up_ds_listen process 1024
Sep 22 23:14:37.961 DEBG [A] ack job 1024:25, : downstairs
Sep 22 23:14:37.961 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:38.089 DEBG Flush     :1009 extent_limit None deps:[JobId(1008), JobId(1007), JobId(1006)] res:true f:3 g:2
Sep 22 23:14:38.089 DEBG Flush     :1009 extent_limit None deps:[JobId(1008), JobId(1007), JobId(1006)] res:true f:3 g:2
Sep 22 23:14:38.089 DEBG Flush     :1009 extent_limit None deps:[JobId(1008), JobId(1007), JobId(1006)] res:true f:3 g:2
{"msg":"Scrub at offset 1792/3840 sp:1792","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:38.090347219Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:38.304 DEBG IO Write 1025 has deps [JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)]
Sep 22 23:14:38.304 DEBG up_ds_listen was notified
Sep 22 23:14:38.304 DEBG up_ds_listen process 1025
Sep 22 23:14:38.304 DEBG [A] ack job 1025:26, : downstairs
Sep 22 23:14:38.304 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:38.304 DEBG IO Flush 1026 has deps [JobId(1025), JobId(1024), JobId(1023)]
{"msg":"Scrub at offset 2048/3840 sp:2048","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:38.31696777Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:38.38429948Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/job/9582738c-4762-4d3d-9423-661cc66a6ac0/is_finished","method":"GET","req_id":"81dcca62-e53d-4efa-866c-1484734c6836","remote_addr":"127.0.0.1:43906","local_addr":"127.0.0.1:49411","component":"dropshot","latency_us":234,"response_code":"200"}
Sep 22 23:14:38.522 DEBG [2] Read already AckReady 1000, : downstairs
Sep 22 23:14:38.527 DEBG up_ds_listen was notified
Sep 22 23:14:38.527 DEBG up_ds_listen process 1000
Sep 22 23:14:38.527 DEBG [A] ack job 1000:1, : downstairs
{"msg":"Scrub at offset 2304/3840 sp:2304","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:38.54535Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:38.559 DEBG Flush     :1012 extent_limit None deps:[JobId(1011), JobId(1010)] res:true f:4 g:2
Sep 22 23:14:38.559 DEBG Flush     :1012 extent_limit None deps:[JobId(1011), JobId(1010)] res:true f:4 g:2
Sep 22 23:14:38.559 DEBG Flush     :1012 extent_limit None deps:[JobId(1011), JobId(1010)] res:true f:4 g:2
Sep 22 23:14:38.627 DEBG up_ds_listen checked 1 jobs, back to waiting
0 512 ok
512 1024 ok
1024 1536 ok
1536 2048 ok
2048 2560 ok
2560 3072 ok
3072 3584 ok
3584 4096 ok
4096 4608 ok
4608 5120 ok
5120 5632 ok
5632 6144 ok
6144 6656 ok
6656 7168 ok
7168 7680 ok
7680 8192 ok
8192 8704 ok
8704 9216 ok
9216 9728 ok
9728 10240 ok
10240 10752 ok
10752 11264 ok
11264 11776 ok
11776 12288 ok
12288 12800 ok
12800 13312 ok
13312 13824 ok
13824 14336 ok
14336 14848 ok
14848 15360 ok
15360 15872 ok
15872 16384 ok
16384 16896 ok
16896 17408 ok
17408 17920 ok
17920 18432 ok
18432 18944 ok
18944 19456 ok
19456 19968 ok
19968 20480 ok
20480 20992 ok
20992 21504 ok
21504 22016 ok
22016 22528 ok
22528 23040 ok
23040 23552 ok
23552 24064 ok
24064 24576 ok
24576 25088 ok
25088 25600 ok
25600 26112 ok
26112 26624 ok
26624 27136 ok
27136 27648 ok
27648 28160 ok
28160 28672 ok
28672 29184 ok
29184 29696 ok
29696 30208 ok
30208 30720 ok
30720 31232 ok
31232 31744 ok
31744 32256 ok
32256 32768 ok
32768 33280 ok
33280 33792 ok
33792 34304 ok
34304 34816 ok
34816 35328 ok
35328 35840 ok
35840 36352 ok
36352 36864 ok
36864 37376 ok
37376 37888 ok
37888 38400 ok
38400 38912 ok
38912 39424 ok
39424 39936 ok
39936 40448 ok
40448 40960 ok
40960 41472 ok
41472 41984 ok
41984 42496 ok
42496 43008 ok
43008 43520 ok
43520 44032 ok
44032 44544 ok
44544 45056 ok
45056 45568 ok
45568 46080 ok
46080 46592 ok
46592 47104 ok
47104 47616 ok
47616 48128 ok
48128 48640 ok
48640 49152 ok
49152 49664 ok
49664 50176 ok
50176 50688 ok
50688 51200 ok
51200 51712 ok
51712 52224 ok
52224 52736 ok
52736 53248 ok
53248 53760 ok
53760 54272 ok
54272 54784 ok
54784 55296 ok
55296 55808 ok
55808 56320 ok
56320 56832 ok
56832 57344 ok
57344 57856 ok
57856 58368 ok
58368 58880 ok
58880 59392 ok
59392 59904 ok
59904 60416 ok
60416 60928 ok
Sep 22 23:14:38.630 INFO Checking if live repair is needed
Sep 22 23:14:38.630 INFO No Live Repair required at this time
Sep 22 23:14:38.630 DEBG IO Flush 1001 has deps [JobId(1000)]
60928 61440 ok
61440 61952 ok
61952 62464 ok
62464 62976 ok
62976 63488 ok
63488 64000 ok
64000 64512 ok
64512 65024 ok
65024 65536 ok
65536 66048 ok
66048 66560 ok
66560 67072 ok
67072 67584 ok
67584 68096 ok
68096 68608 ok
68608 69120 ok
69120 69632 ok
69632 70144 ok
70144 70656 ok
70656 71168 ok
71168 71680 ok
71680 72192 ok
72192 72704 ok
72704 73216 ok
73216 73728 ok
73728 74240 ok
74240 74752 ok
74752 75264 ok
75264 75776 ok
75776 76288 ok
76288 76800 ok
76800 77312 ok
77312 77824 ok
77824 78336 ok
78336 78848 ok
78848 79360 ok
79360 79872 ok
79872 80384 ok
80384 80896 ok
80896 81408 ok
81408 81920 ok
81920 82432 ok
82432 82944 ok
82944 83456 ok
83456 83968 ok
83968 84480 ok
84480 84992 ok
84992 85504 ok
85504 86016 ok
86016 86528 ok
86528 87040 ok
87040 87552 ok
87552 88064 ok
88064 88576 ok
88576 89088 ok
89088 89600 ok
89600 90112 ok
90112 90624 ok
90624 91136 ok
91136 91648 ok
91648 92160 ok
92160 92672 ok
92672 93184 ok
93184 93696 ok
93696 94208 ok
94208 94720 ok
94720 95232 ok
95232 95744 ok
95744 96256 ok
96256 96768 ok
96768 97280 ok
97280 97792 ok
97792 98304 ok
98304 98816 ok
98816 99328 ok
99328 99840 ok
99840 100352 ok
100352 100864 ok
100864 101376 ok
101376 101888 ok
101888 102400 ok
102400 102912 ok
102912 103424 ok
103424 103936 ok
103936 104448 ok
104448 104960 ok
104960 105472 ok
105472 105984 ok
105984 106496 ok
106496 107008 ok
107008 107520 ok
107520 108032 ok
108032 108544 ok
108544 109056 ok
109056 109568 ok
109568 110080 ok
110080 110592 ok
110592 111104 ok
111104 111616 ok
111616 112128 ok
112128 112640 ok
112640 113152 ok
113152 113664 ok
113664 114176 ok
114176 114688 ok
114688 115200 ok
115200 115712 ok
115712 116224 ok
116224 116736 ok
116736 117248 ok
117248 117760 ok
117760 118272 ok
118272 118784 ok
118784 119296 ok
119296 119808 ok
119808 120320 ok
120320 120832 ok
120832 121344 ok
121344 121856 ok
121856 122368 ok
122368 122880 ok
122880 123392 ok
123392 123904 ok
123904 124416 ok
124416 124928 ok
124928 125440 ok
125440 125952 ok
125952 126464 ok
{"msg":"[0] 0d1229ab-bd36-49b6-95e1-d6287586b840 looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:38.631436286Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"0","component":"datafile"}
{"msg":"[0] Proc runs for 127.0.0.1:38920 in state Disconnected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:38.631514479Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] 0d1229ab-bd36-49b6-95e1-d6287586b840 looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:38.631556048Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"1","component":"datafile"}
Sep 22 23:14:38.631 INFO accepted connection from 127.0.0.1:42804, task: main
{"msg":"[1] Proc runs for 127.0.0.1:44435 in state Disconnected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:38.63159419Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:38.631 INFO accepted connection from 127.0.0.1:34565, task: main
{"msg":"[2] 0d1229ab-bd36-49b6-95e1-d6287586b840 looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:38.631631194Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"2","component":"datafile"}
{"msg":"[2] Proc runs for 127.0.0.1:51108 in state Disconnected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:38.631669152Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:38.631 INFO accepted connection from 127.0.0.1:42753, task: main
Sep 22 23:14:38.631 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:5 g:2
Sep 22 23:14:38.631 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:5 g:2
Sep 22 23:14:38.632 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:5 g:2
126464 126976 ok
126976 127488 ok
127488 128000 ok
128000 128512 ok
128512 129024 ok
129024 129536 ok
129536 130048 ok
130048 130560 ok
130560 131072 ok
131072 131584 ok
131584 132096 ok
132096 132608 ok
132608 133120 ok
133120 133632 ok
133632 134144 ok
134144 134656 ok
134656 135168 ok
135168 135680 ok
135680 136192 ok
136192 136704 ok
136704 137216 ok
137216 137728 ok
137728 138240 ok
138240 138752 ok
138752 139264 ok
139264 139776 ok
139776 140288 ok
140288 140800 ok
140800 141312 ok
141312 141824 ok
141824 142336 ok
142336 142848 ok
142848 143360 ok
143360 143872 ok
143872 144384 ok
144384 144896 ok
144896 145408 ok
145408 145920 ok
145920 146432 ok
146432 146944 ok
146944 147456 ok
147456 147968 ok
147968 148480 ok
148480 148992 ok
148992 149504 ok
149504 150016 ok
150016 150528 ok
150528 151040 ok
151040 151552 ok
151552 152064 ok
152064 152576 ok
152576 153088 ok
153088 153600 ok
153600 154112 ok
154112 154624 ok
154624 155136 ok
155136 155648 ok
155648 156160 ok
156160 156672 ok
156672 157184 ok
157184 157696 ok
157696 158208 ok
158208 158720 ok
158720 159232 ok
159232 159744 ok
159744 160256 ok
160256 160768 ok
160768 161280 ok
161280 161792 ok
161792 162304 ok
162304 162816 ok
162816 163328 ok
163328 163840 ok
163840 164352 ok
164352 164864 ok
164864 165376 ok
165376 165888 ok
165888 166400 ok
166400 166912 ok
166912 167424 ok
167424 167936 ok
167936 168448 ok
168448 168960 ok
168960 169472 ok
169472 169984 ok
169984 170496 ok
170496 171008 ok
171008 171520 ok
171520 172032 ok
172032 172544 ok
172544 173056 ok
173056 173568 ok
173568 174080 ok
174080 174592 ok
174592 175104 ok
175104 175616 ok
175616 176128 ok
176128 176640 ok
176640 177152 ok
177152 177664 ok
177664 178176 ok
178176 178688 ok
178688 179200 ok
179200 179712 ok
179712 180224 ok
180224 180736 ok
180736 181248 ok
181248 181760 ok
181760 182272 ok
182272 182784 ok
182784 183296 ok
183296 183808 ok
183808 184320 ok
184320 184832 ok
184832 185344 ok
185344 185856 ok
185856 186368 ok
186368 186880 ok
186880 187392 ok
187392 187904 ok
187904 188416 ok
188416 188928 ok
188928 189440 ok
189440 189952 ok
189952 190464 ok
190464 190976 ok
190976 191488 ok
191488 192000 ok
Sep 22 23:14:38.632 INFO Connection request from 0d1229ab-bd36-49b6-95e1-d6287586b840 with version 4, task: proc
Sep 22 23:14:38.632 INFO upstairs UpstairsConnection { upstairs_id: 0d1229ab-bd36-49b6-95e1-d6287586b840, session_id: 16060f7a-9986-4db1-95c8-74c4346b949d, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:38.632 INFO Connection request from 0d1229ab-bd36-49b6-95e1-d6287586b840 with version 4, task: proc
Sep 22 23:14:38.632 INFO upstairs UpstairsConnection { upstairs_id: 0d1229ab-bd36-49b6-95e1-d6287586b840, session_id: 16060f7a-9986-4db1-95c8-74c4346b949d, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:38.632 INFO Connection request from 0d1229ab-bd36-49b6-95e1-d6287586b840 with version 4, task: proc
Sep 22 23:14:38.633 INFO upstairs UpstairsConnection { upstairs_id: 0d1229ab-bd36-49b6-95e1-d6287586b840, session_id: 16060f7a-9986-4db1-95c8-74c4346b949d, gen: 1 } connected, version 4, task: proc
Sep 22 23:14:38.633 DEBG up_ds_listen was notified
Sep 22 23:14:38.633 DEBG up_ds_listen process 1001
Sep 22 23:14:38.633 DEBG [A] ack job 1001:2, : downstairs
Sep 22 23:14:38.633 DEBG [rc] retire 1001 clears [JobId(1000), JobId(1001)], : downstairs
Sep 22 23:14:38.633 DEBG up_ds_listen checked 1 jobs, back to waiting
192000 192512 ok
192512 193024 ok
193024 193536 ok
193536 194048 ok
194048 194560 ok
194560 195072 ok
195072 195584 ok
195584 196096 ok
196096 196608 ok
196608 197120 ok
197120 197632 ok
197632 198144 ok
198144 198656 ok
198656 199168 ok
199168 199680 ok
199680 200192 ok
200192 200704 ok
200704 201216 ok
201216 201728 ok
201728 202240 ok
202240 202752 ok
202752 203264 ok
203264 203776 ok
203776 204288 ok
204288 204800 ok
204800 205312 ok
205312 205824 ok
205824 206336 ok
206336 206848 ok
206848 207360 ok
207360 207872 ok
207872 208384 ok
208384 208896 ok
208896 209408 ok
209408 209920 ok
209920 210432 ok
210432 210944 ok
210944 211456 ok
211456 211968 ok
211968 212480 ok
212480 212992 ok
212992 213504 ok
213504 214016 ok
214016 214528 ok
214528 215040 ok
215040 215552 ok
215552 216064 ok
216064 216576 ok
216576 217088 ok
217088 217600 ok
217600 218112 ok
218112 218624 ok
218624 219136 ok
219136 219648 ok
219648 220160 ok
220160 220672 ok
220672 221184 ok
221184 221696 ok
221696 222208 ok
222208 222720 ok
222720 223232 ok
223232 223744 ok
223744 224256 ok
224256 224768 ok
224768 225280 ok
225280 225792 ok
225792 226304 ok
226304 226816 ok
226816 227328 ok
227328 227840 ok
227840 228352 ok
228352 228864 ok
228864 229376 ok
229376 229888 ok
229888 230400 ok
230400 230912 ok
230912 231424 ok
231424 231936 ok
231936 232448 ok
232448 232960 ok
232960 233472 ok
233472 233984 ok
233984 234496 ok
234496 235008 ok
235008 235520 ok
235520 236032 ok
236032 236544 ok
236544 237056 ok
237056 237568 ok
237568 238080 ok
238080 238592 ok
238592 239104 ok
239104 239616 ok
239616 240128 ok
240128 240640 ok
240640 241152 ok
241152 241664 ok
241664 242176 ok
242176 242688 ok
242688 243200 ok
243200 243712 ok
243712 244224 ok
244224 244736 ok
244736 245248 ok
245248 245760 ok
245760 246272 ok
246272 246784 ok
246784 247296 ok
247296 247808 ok
247808 248320 ok
248320 248832 ok
248832 249344 ok
249344 249856 ok
249856 250368 ok
250368 250880 ok
250880 251392 ok
251392 251904 ok
251904 252416 ok
252416 252928 ok
252928 253440 ok
253440 253952 ok
253952 254464 ok
254464 254976 ok
254976 255488 ok
255488 256000 ok
256000 256512 ok
256512 257024 ok
257024 257536 ok
{"msg":"[0] 0d1229ab-bd36-49b6-95e1-d6287586b840 (16060f7a-9986-4db1-95c8-74c4346b949d) Disconnected Disconnected Disconnected ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:38.633962684Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] Transition from Disconnected to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:38.634005606Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] 0d1229ab-bd36-49b6-95e1-d6287586b840 (16060f7a-9986-4db1-95c8-74c4346b949d) WaitActive Disconnected Disconnected ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:38.634033777Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
257536 258048 ok
258048 258560 ok
258560 259072 ok
259072{"msg":"[1] Transition from Disconnected to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:38.634062843Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
 259584 ok
259584 260096 ok
260096 260608 ok
260608 261120 ok
{"msg":"[2] 0d1229ab-bd36-49b6-95e1-d6287586b840 (16060f7a-9986-4db1-95c8-74c4346b949d) WaitActive WaitActive Disconnected ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:38.634117393Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
261120 261632 ok
261632 262144 ok
262144 262656 ok
262656 263168 ok
263168 263680 ok
263680 264192{"msg":"[2] Transition from Disconnected to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:38.634161966Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
 ok
264192 264704 ok
264704 265216 ok
265216 265728 ok
265728 266240 ok
266240 266752 ok
266752 267264 ok
267264 267776 ok
267776 268288 ok
268288 268800 ok
268800 269312 ok
269312 269824 ok
269824 270336 ok
270336 270848 ok
270848 271360 ok
271360 271872 ok
271872 272384 ok
272384 272896 ok
272896 273408 ok
273408 273920 ok
273920 274432 ok
274432 274944 ok
274944 275456 ok
275456 275968 ok
275968 276480 ok
276480 276992 ok
276992 277504 ok
277504 278016 ok
278016 278528 ok
278528 279040 ok
279040 279552 ok
279552 280064 ok
280064 280576 ok
280576 281088 ok
281088 281600 ok
281600 282112 ok
282112 282624 ok
282624 283136 ok
283136 283648 ok
283648 284160 ok
284160 284672 ok
284672 285184 ok
285184 285696 ok
285696 286208 ok
286208 286720 ok
286720 287232 ok
287232 287744 ok
287744 288256 ok
288256 288768 ok
288768 289280 ok
289280 289792 ok
289792 290304 ok
290304 290816 ok
290816 291328 ok
291328 291840 ok
291840 292352 ok
292352 292864 ok
292864 293376 ok
293376 293888 ok
293888 294400 ok
294400 294912 ok
294912 295424 ok
295424 295936 ok
295936 296448 ok
296448 296960 ok
296960 297472 ok
297472 297984 ok
297984 298496 ok
298496 299008 ok
299008 299520 ok
299520 300032 ok
300032 300544 ok
300544 301056 ok
301056 301568 ok
301568 302080 ok
302080 302592 ok
302592 303104 ok
303104 303616 ok
303616 304128 ok
304128 304640 ok
304640 305152 ok
305152 305664 ok
305664 306176 ok
306176 306688 ok
306688 307200 ok
307200 307712 ok
307712 308224 ok
308224 308736 ok
308736 309248 ok
309248 309760 ok
309760 310272 ok
310272 310784 ok
310784 311296 ok
311296 311808 ok
311808 312320 ok
312320 312832 ok
312832 313344 ok
313344 313856 ok
313856 314368 ok
314368 314880 ok
314880 315392 ok
315392 315904 ok
315904 316416 ok
316416 316928 ok
316928 317440 ok
317440 317952 ok
317952 318464 ok
318464 318976 ok
318976 319488 ok
319488 320000 ok
320000 320512 ok
320512 321024 ok
321024 321536 ok
321536 322048 ok
322048 322560 ok
322560 323072 ok
323072 323584 ok
323584 324096 ok
324096 324608 ok
324608 325120 ok
325120 325632 ok
325632 326144 ok
326144 326656 ok
326656 327168 ok
327168 327680 ok
327680 328192 ok
328192 328704 ok
328704 329216 ok
329216 329728 ok
329728 330240 ok
330240 330752 ok
330752 331264 ok
331264 331776 ok
331776 332288 ok
332288 332800 ok
332800 333312 ok
333312 333824 ok
333824 334336 ok
334336 334848 ok
334848 335360 ok
335360 335872 ok
335872 336384 ok
336384 336896 ok
336896 337408 ok
337408 337920 ok
337920 338432 ok
338432 338944 ok
338944 339456 ok
339456 339968 ok
339968 340480 ok
340480 340992 ok
340992 341504 ok
341504 342016 ok
342016 342528 ok
342528 343040 ok
343040 343552 ok
343552 344064 ok
344064 344576 ok
344576 345088 ok
345088 345600 ok
345600 346112 ok
346112 346624 ok
346624 347136 ok
347136 347648 ok
347648 348160 ok
348160 348672 ok
348672 349184 ok
349184 349696 ok
349696 350208 ok
350208 350720 ok
350720 351232 ok
351232 351744 ok
351744 352256 ok
352256 352768 ok
352768 353280 ok
353280 353792 ok
353792 354304 ok
354304 354816 ok
354816 355328 ok
355328 355840 ok
355840 356352 ok
356352 356864 ok
356864 357376 ok
357376 357888 ok
357888 358400 ok
358400 358912 ok
358912 359424 ok
359424 359936 ok
359936 360448 ok
360448 360960 ok
360960 361472 ok
361472 361984 ok
361984 362496 ok
362496 363008 ok
363008 363520 ok
363520 364032 ok
364032 364544 ok
364544 365056 ok
365056 365568 ok
365568 366080 ok
366080 366592 ok
366592 367104 ok
367104 367616 ok
367616 368128 ok
368128 368640 ok
368640 369152 ok
369152 369664 ok
369664 370176 ok
370176 370688 ok
370688 371200 ok
371200 371712 ok
371712 372224 ok
372224 372736 ok
372736 373248 ok
373248 373760 ok
373760 374272 ok
374272 374784 ok
374784 375296 ok
375296 375808 ok
375808 376320 ok
376320 376832 ok
376832 377344 ok
377344 377856 ok
377856 378368 ok
378368 378880 ok
378880 379392 ok
379392 379904 ok
379904 380416 ok
380416 380928 ok
380928 381440 ok
381440 381952 ok
381952 382464 ok
382464 382976 ok
382976 383488 ok
383488 384000 ok
384000 384512 ok
384512 385024 ok
385024 385536 ok
385536 386048 ok
386048 386560 ok
386560 387072 ok
387072 387584 ok
387584 388096 ok
388096 388608 ok
388608 389120 ok
389120 389632 ok
389632 390144 ok
390144 390656 ok
390656 391168 ok
391168 391680 ok
391680 392192 ok
392192 392704 ok
392704 393216 ok
393216 393728 ok
393728 394240 ok
394240 394752 ok
394752 395264 ok
395264 395776 ok
395776 396288 ok
396288 396800 ok
396800 397312 ok
397312 397824 ok
397824 398336 ok
398336 398848 ok
398848 399360 ok
399360 399872 ok
399872 400384 ok
400384 400896 ok
400896 401408 ok
401408 401920 ok
401920 402432 ok
402432 402944 ok
402944 403456 ok
403456 403968 ok
403968 404480 ok
404480 404992 ok
404992 405504 ok
405504 406016 ok
406016 406528 ok
406528 407040 ok
407040 407552 ok
407552 408064 ok
408064 408576 ok
408576 409088 ok
409088 409600 ok
409600 410112 ok
410112 410624 ok
410624 411136 ok
411136 411648 ok
411648 412160 ok
412160 412672 ok
412672 413184 ok
413184 413696 ok
413696 414208 ok
414208 414720 ok
414720 415232 ok
415232 415744 ok
415744 416256 ok
416256 416768 ok
416768 417280 ok
417280 417792 ok
417792 418304 ok
418304 418816 ok
418816 419328 ok
419328 419840 ok
419840 420352 ok
420352 420864 ok
420864 421376 ok
421376 421888 ok
421888 422400 ok
422400 422912 ok
422912 423424 ok
423424 423936 ok
423936 424448 ok
424448 424960 ok
424960 425472 ok
425472 425984 ok
425984 426496 ok
426496 427008 ok
427008 427520 ok
427520 428032 ok
428032 428544 ok
428544 429056 ok
429056 429568 ok
429568 430080 ok
430080 430592 ok
430592 431104 ok
431104 431616 ok
431616 432128 ok
432128 432640 ok
432640 433152 ok
433152 433664 ok
433664 434176 ok
434176 434688 ok
434688 435200 ok
435200 435712 ok
435712 436224 ok
436224 436736 ok
436736 437248 ok
437248 437760 ok
437760 438272 ok
438272 438784 ok
438784 439296 ok
439296 439808 ok
439808 440320 ok
440320 440832 ok
440832 441344 ok
441344 441856 ok
441856 442368 ok
442368 442880 ok
442880 443392 ok
443392 443904 ok
443904 444416 ok
444416 444928 ok
444928 445440 ok
445440 445952 ok
445952 446464 ok
446464 446976 ok
446976 447488 ok
447488 448000 ok
448000 448512 ok
448512 449024 ok
449024 449536 ok
449536 450048 ok
450048 450560 ok
450560 451072 ok
451072 451584 ok
451584 452096 ok
452096 452608 ok
452608 453120 ok
453120 453632 ok
453632 454144 ok
454144 454656 ok
454656 455168 ok
455168 455680 ok
455680 456192 ok
456192 456704 ok
456704 457216 ok
457216 457728 ok
457728 458240 ok
458240 458752 ok
458752 459264 ok
459264 459776 ok
459776 460288 ok
460288 460800 ok
460800 461312 ok
461312 461824 ok
461824 462336 ok
462336 462848 ok
462848 463360 ok
463360 463872 ok
463872 464384 ok
464384 464896 ok
464896 465408 ok
465408 465920 ok
465920 466432 ok
466432 466944 ok
466944 467456 ok
467456 467968 ok
467968 468480 ok
468480 468992 ok
468992 469504 ok
469504 470016 ok
470016 470528 ok
470528 471040 ok
471040 471552 ok
471552 472064 ok
472064 472576 ok
472576 473088 ok
473088 473600 ok
473600 474112 ok
474112 474624 ok
474624 475136 ok
475136 475648 ok
475648 476160 ok
476160 476672 ok
476672 477184 ok
477184 477696 ok
477696 478208 ok
478208 478720 ok
478720 479232 ok
479232 479744 ok
479744 480256 ok
480256 480768 ok
480768 481280 ok
481280 481792 ok
481792 482304 ok
482304 482816 ok
482816 483328 ok
483328 483840 ok
483840 484352 ok
484352 484864 ok
484864 485376 ok
485376 485888 ok
485888 486400 ok
486400 486912 ok
486912 487424 ok
487424 487936 ok
487936 488448 ok
488448 488960 ok
488960 489472 ok
489472 489984 ok
489984 490496 ok
490496 491008 ok
491008 491520 ok
491520 492032 ok
492032 492544 ok
492544 493056 ok
493056 493568 ok
493568 494080 ok
494080 494592 ok
494592 495104 ok
495104 495616 ok
495616 496128 ok
496128 496640 ok
496640 497152 ok
497152 497664 ok
497664 498176 ok
498176 498688 ok
498688 499200 ok
499200 499712 ok
499712 500224 ok
500224 500736 ok
500736 501248 ok
501248 501760 ok
501760 502272 ok
502272 502784 ok
502784 503296 ok
503296 503808 ok
503808 504320 ok
504320 504832 ok
504832 505344 ok
505344 505856 ok
505856 506368 ok
506368 506880 ok
506880 507392 ok
507392 507904 ok
507904 508416 ok
508416 508928 ok
508928 509440 ok
509440 509952 ok
509952 510464 ok
510464 510976 ok
510976 511488 ok
511488 512000 ok
512000 512512 ok
512512 513024 ok
513024 513536 ok
513536 514048 ok
514048 514560 ok
514560 515072 ok
515072 515584 ok
515584 516096 ok
516096 516608 ok
516608 517120 ok
517120 517632 ok
517632 518144 ok
518144 518656 ok
518656 519168 ok
519168 519680 ok
519680 520192 ok
520192 520704 ok
520704 521216 ok
521216 521728 ok
521728 522240 ok
522240 522752 ok
522752 523264 ok
523264 523776 ok
523776 524288 ok
524288 524800 ok
524800 525312 ok
525312 525824 ok
525824 526336 ok
526336 526848 ok
526848 527360 ok
527360 527872 ok
527872 528384 ok
528384 528896 ok
528896 529408 ok
529408 529920 ok
529920 530432 ok
530432 530944 ok
530944 531456 ok
531456 531968 ok
531968 532480 ok
532480 532992 ok
532992 533504 ok
533504 534016 ok
534016 534528 ok
534528 535040 ok
535040 535552 ok
535552 536064 ok
536064 536576 ok
536576 537088 ok
537088 537600 ok
537600 538112 ok
538112 538624 ok
538624 539136 ok
539136 539648 ok
539648 540160 ok
540160 540672 ok
540672 541184 ok
541184 541696 ok
541696 542208 ok
542208 542720 ok
542720 543232 ok
543232 543744 ok
543744 544256 ok
544256 544768 ok
544768 545280 ok
545280 545792 ok
545792 546304 ok
546304 546816 ok
546816 547328 ok
547328 547840 ok
547840 548352 ok
548352 548864 ok
548864 549376 ok
549376 549888 ok
549888 550400 ok
550400 550912 ok
550912 551424 ok
551424 551936 ok
551936 552448 ok
552448 552960 ok
552960 553472 ok
553472 553984 ok
553984 554496 ok
554496 555008 ok
555008 555520 ok
555520 556032 ok
556032 556544 ok
556544 557056 ok
557056 557568 ok
557568 558080 ok
558080 558592 ok
558592 559104 ok
559104 559616 ok
559616 560128 ok
560128 560640 ok
560640 561152 ok
561152 561664 ok
561664 562176 ok
562176 562688 ok
562688 563200 ok
563200 563712 ok
563712 564224 ok
564224 564736 ok
564736 565248 ok
565248 565760 ok
565760 566272 ok
566272 566784 ok
566784 567296 ok
567296 567808 ok
567808 568320 ok
568320 568832 ok
568832 569344 ok
569344 569856 ok
569856 570368 ok
570368 570880 ok
570880 571392 ok
571392 571904 ok
571904 572416 ok
572416 572928 ok
572928 573440 ok
573440 573952 ok
573952 574464 ok
574464 574976 ok
574976 575488 ok
575488 576000 ok
576000 576512 ok
576512 577024 ok
577024 577536 ok
577536 578048 ok
578048 578560 ok
578560 579072 ok
579072 579584 ok
579584 580096 ok
580096 580608 ok
580608 581120 ok
581120 581632 ok
581632 582144 ok
582144 582656 ok
582656 583168 ok
583168 583680 ok
583680 584192 ok
584192 584704 ok
584704 585216 ok
585216 585728 ok
585728 586240 ok
586240 586752 ok
586752 587264 ok
587264 587776 ok
587776 588288 ok
588288 588800 ok
588800 589312 ok
589312 589824 ok
589824 590336 ok
590336 590848 ok
590848 591360 ok
591360 591872 ok
591872 592384 ok
592384 592896 ok
592896 593408 ok
593408 593920 ok
593920 594432 ok
594432 594944 ok
594944 595456 ok
595456 595968 ok
595968 596480 ok
596480 596992 ok
596992 597504 ok
597504 598016 ok
598016 598528 ok
598528 599040 ok
599040 599552 ok
599552 600064 ok
600064 600576 ok
600576 601088 ok
601088 601600 ok
601600 602112 ok
602112 602624 ok
602624 603136 ok
603136 603648 ok
603648 604160 ok
604160 604672 ok
604672 605184 ok
605184 605696 ok
605696 606208 ok
606208 606720 ok
606720 607232 ok
607232 607744 ok
607744 608256 ok
608256 608768 ok
608768 609280 ok
609280 609792 ok
609792 610304 ok
610304 610816 ok
610816 611328 ok
611328 611840 ok
611840 612352 ok
612352 612864 ok
612864 613376 ok
613376 613888 ok
613888 614400 ok
614400 614912 ok
614912 615424 ok
615424 615936 ok
615936 616448 ok
616448 616960 ok
616960 617472 ok
617472 617984 ok
617984 618496 ok
618496 619008 ok
619008 619520 ok
619520 620032 ok
620032 620544 ok
620544 621056 ok
621056 621568 ok
621568 622080 ok
622080 622592 ok
622592 623104 ok
623104 623616 ok
623616 624128 ok
624128 624640 ok
624640 625152 ok
625152 625664 ok
625664 626176 ok
626176 626688 ok
626688 627200 ok
627200 627712 ok
627712 628224 ok
628224 628736 ok
628736 629248 ok
629248 629760 ok
629760 630272 ok
630272 630784 ok
630784 631296 ok
631296 631808 ok
631808 632320 ok
632320 632832 ok
632832 633344 ok
633344 633856 ok
633856 634368 ok
634368 634880 ok
634880 635392 ok
635392 635904 ok
635904 636416 ok
636416 636928 ok
636928 637440 ok
637440 637952 ok
637952 638464 ok
638464 638976 ok
638976 639488 ok
639488 640000 ok
640000 640512 ok
640512 641024 ok
641024 641536 ok
641536 642048 ok
642048 642560 ok
642560 643072 ok
643072 643584 ok
643584 644096 ok
644096 644608 ok
644608 645120 ok
645120 645632 ok
645632 646144 ok
646144 646656 ok
646656 647168 ok
647168 647680 ok
647680 648192 ok
648192 648704 ok
648704 649216 ok
649216 649728 ok
649728 650240 ok
650240 650752 ok
650752 651264 ok
651264 651776 ok
651776 652288 ok
652288 652800 ok
652800 653312 ok
653312 653824 ok
653824 654336 ok
654336 654848 ok
654848 655360 ok
655360 655872 ok
655872 656384 ok
656384 656896 ok
656896 657408 ok
657408 657920 ok
657920 658432 ok
658432 658944 ok
658944 659456 ok
659456 659968 ok
659968 660480 ok
660480 660992 ok
660992 661504 ok
661504 662016 ok
662016 662528 ok
662528 663040 ok
663040 663552 ok
663552 664064 ok
664064 664576 ok
664576 665088 ok
665088 665600 ok
665600 666112 ok
666112 666624 ok
666624 667136 ok
667136 667648 ok
667648 668160 ok
668160 668672 ok
668672 669184 ok
669184 669696 ok
669696 670208 ok
670208 670720 ok
670720 671232 ok
671232 671744 ok
671744 672256 ok
672256 672768 ok
672768 673280 ok
673280 673792 ok
673792 674304 ok
674304 674816 ok
674816 675328 ok
675328 675840 ok
675840 676352 ok
676352 676864 ok
676864 677376 ok
677376 677888 ok
677888 678400 ok
678400 678912 ok
678912 679424 ok
679424 679936 ok
679936 680448 ok
680448 680960 ok
680960 681472 ok
681472 681984 ok
681984 682496 ok
682496 683008 ok
683008 683520 ok
683520 684032 ok
684032 684544 ok
684544 685056 ok
685056 685568 ok
685568 686080 ok
686080 686592 ok
686592 687104 ok
687104 687616 ok
687616 688128 ok
688128 688640 ok
688640 689152 ok
689152 689664 ok
689664 690176 ok
690176 690688 ok
690688 691200 ok
691200 691712 ok
691712 692224 ok
692224 692736 ok
692736 693248 ok
693248 693760 ok
693760 694272 ok
694272 694784 ok
694784 695296 ok
695296 695808 ok
695808 696320 ok
696320 696832 ok
696832 697344 ok
697344 697856 ok
697856 698368 ok
698368 698880 ok
698880 699392 ok
699392 699904 ok
699904 700416 ok
700416 700928 ok
700928 701440 ok
701440 701952 ok
701952 702464 ok
702464 702976 ok
702976 703488 ok
703488 704000 ok
704000 704512 ok
704512 705024 ok
705024 705536 ok
705536 706048 ok
706048 706560 ok
706560 707072 ok
707072 707584 ok
707584 708096 ok
708096 708608 ok
708608 709120 ok
709120 709632 ok
709632 710144 ok
710144 710656 ok
710656 711168 ok
711168 711680 ok
711680 712192 ok
712192 712704 ok
712704 713216 ok
713216 713728 ok
713728 714240 ok
714240 714752 ok
714752 715264 ok
715264 715776 ok
715776 716288 ok
716288 716800 ok
716800 717312 ok
717312 717824 ok
717824 718336 ok
718336 718848 ok
718848 719360 ok
719360 719872 ok
719872 720384 ok
720384 720896 ok
720896 721408 ok
721408 721920 ok
721920 722432 ok
722432 722944 ok
722944 723456 ok
723456 723968 ok
723968 724480 ok
724480 724992 ok
724992 725504 ok
725504 726016 ok
726016 726528 ok
726528 727040 ok
727040 727552 ok
727552 728064 ok
728064 728576 ok
728576 729088 ok
729088 729600 ok
729600 730112 ok
730112 730624 ok
730624 731136 ok
731136 731648 ok
731648 732160 ok
732160 732672 ok
732672 733184 ok
733184 733696 ok
733696 734208 ok
734208 734720 ok
734720 735232 ok
735232 735744 ok
735744 736256 ok
736256 736768 ok
736768 737280 ok
737280 737792 ok
737792 738304 ok
738304 738816 ok
738816 739328 ok
739328 739840 ok
739840 740352 ok
740352 740864 ok
740864 741376 ok
741376 741888 ok
741888 742400 ok
742400 742912 ok
742912 743424 ok
743424 743936 ok
743936 744448 ok
744448 744960 ok
744960 745472 ok
745472 745984 ok
745984 746496 ok
746496 747008 ok
747008 747520 ok
747520 748032 ok
748032 748544 ok
748544 749056 ok
749056 749568 ok
749568 750080 ok
750080 750592 ok
750592 751104 ok
751104 751616 ok
751616 752128 ok
752128 752640 ok
752640 753152 ok
753152 753664 ok
753664 754176 ok
754176 754688 ok
754688 755200 ok
755200 755712 ok
755712 756224 ok
756224 756736 ok
756736 757248 ok
757248 757760 ok
757760 758272 ok
758272 758784 ok
758784 759296 ok
759296 759808 ok
759808 760320 ok
760320 760832 ok
760832 761344 ok
761344 761856 ok
761856 762368 ok
762368 762880 ok
762880 763392 ok
763392 763904 ok
763904 764416 ok
764416 764928 ok
764928 765440 ok
765440 765952 ok
765952 766464 ok
766464 766976 ok
766976 767488 ok
767488 768000 ok
768000 768512 ok
768512 769024 ok
769024 769536 ok
769536 770048 ok
770048 770560 ok
770560 771072 ok
771072 771584 ok
771584 772096 ok
772096 772608 ok
772608 773120 ok
773120 773632 ok
773632 774144 ok
774144 774656 ok
774656 775168 ok
775168 775680 ok
775680 776192 ok
776192 776704 ok
776704 777216 ok
777216 777728 ok
777728 778240 ok
778240 778752 ok
778752 779264 ok
779264 779776 ok
779776 780288 ok
780288 780800 ok
780800 781312 ok
781312 781824 ok
781824 782336 ok
782336 782848 ok
782848 783360 ok
783360 783872 ok
783872 784384 ok
784384 784896 ok
784896 785408 ok
785408 785920 ok
785920 786432 ok
786432 786944 ok
786944 787456 ok
787456 787968 ok
787968 788480 ok
788480 788992 ok
788992 789504 ok
789504 790016 ok
790016 790528 ok
790528 791040 ok
791040 791552 ok
791552 792064 ok
792064 792576 ok
792576 793088 ok
793088 793600 ok
793600 794112 ok
794112 794624 ok
794624 795136 ok
795136 795648 ok
795648 796160 ok
796160 796672 ok
796672 797184 ok
797184 797696 ok
797696 798208 ok
798208 798720 ok
798720 799232 ok
799232 799744 ok
799744 800256 ok
800256 800768 ok
800768 801280 ok
801280 801792 ok
801792 802304 ok
802304 802816 ok
802816 803328 ok
803328 803840 ok
803840 804352 ok
804352 804864 ok
804864 805376 ok
805376 805888 ok
805888 806400 ok
806400 806912 ok
806912 807424 ok
807424 807936 ok
807936 808448 ok
808448 808960 ok
808960 809472 ok
809472 809984 ok
809984 810496 ok
810496 811008 ok
811008 811520 ok
811520 812032 ok
812032 812544 ok
812544 813056 ok
813056 813568 ok
813568 814080 ok
814080 814592 ok
814592 815104 ok
815104 815616 ok
815616 816128 ok
816128 816640 ok
816640 817152 ok
817152 817664 ok
817664 818176 ok
818176 818688 ok
818688 819200 ok
819200 819712 ok
819712 820224 ok
820224 820736 ok
820736 821248 ok
821248 821760 ok
821760 822272 ok
822272 822784 ok
822784 823296 ok
823296 823808 ok
823808 824320 ok
824320 824832 ok
824832 825344 ok
825344 825856 ok
825856 826368 ok
826368 826880 ok
826880 827392 ok
827392 827904 ok
827904 828416 ok
828416 828928 ok
828928 829440 ok
829440 829952 ok
829952 830464 ok
830464 830976 ok
830976 831488 ok
831488 832000 ok
832000 832512 ok
832512 833024 ok
833024 833536 ok
833536 834048 ok
834048 834560 ok
834560 835072 ok
835072 835584 ok
835584 836096 ok
836096 836608 ok
836608 837120 ok
837120 837632 ok
837632 838144 ok
838144 838656 ok
838656 839168 ok
839168 839680 ok
839680 840192 ok
840192 840704 ok
840704 841216 ok
841216 841728 ok
841728 842240 ok
842240 842752 ok
842752 843264 ok
843264 843776 ok
843776 844288 ok
844288 844800 ok
844800 845312 ok
845312 845824 ok
845824 846336 ok
846336 846848 ok
846848 847360 ok
847360 847872 ok
847872 848384 ok
848384 848896 ok
848896 849408 ok
849408 849920 ok
849920 850432 ok
850432 850944 ok
850944 851456 ok
851456 851968 ok
851968 852480 ok
852480 852992 ok
852992 853504 ok
853504 854016 ok
854016 854528 ok
854528 855040 ok
855040 855552 ok
855552 856064 ok
856064 856576 ok
856576 857088 ok
857088 857600 ok
857600 858112 ok
858112 858624 ok
858624 859136 ok
859136 859648 ok
859648 860160 ok
860160 860672 ok
860672 861184 ok
861184 861696 ok
861696 862208 ok
862208 862720 ok
862720 863232 ok
863232 863744 ok
863744 864256 ok
864256 864768 ok
864768 865280 ok
865280 865792 ok
865792 866304 ok
866304 866816 ok
866816 867328 ok
867328 867840 ok
867840 868352 ok
868352 868864 ok
868864 869376 ok
869376 869888 ok
869888 870400 ok
870400 870912 ok
870912 871424 ok
871424 871936 ok
871936 872448 ok
872448 872960 ok
872960 873472 ok
873472 873984 ok
873984 874496 ok
874496 875008 ok
875008 875520 ok
875520 876032 ok
876032 876544 ok
876544 877056 ok
877056 877568 ok
877568 878080 ok
878080 878592 ok
878592 879104 ok
879104 879616 ok
879616 880128 ok
880128 880640 ok
880640 881152 ok
881152 881664 ok
881664 882176 ok
882176 882688 ok
882688 883200 ok
883200 883712 ok
883712 884224 ok
884224 884736 ok
884736 885248 ok
885248 885760 ok
885760 886272 ok
886272 886784 ok
886784 887296 ok
887296 887808 ok
887808 888320 ok
888320 888832 ok
888832 889344 ok
889344 889856 ok
889856 890368 ok
890368 890880 ok
890880 891392 ok
891392 891904 ok
891904 892416 ok
892416 892928 ok
892928 893440 ok
893440 893952 ok
893952 894464 ok
894464 894976 ok
894976 895488 ok
895488 896000 ok
896000 896512 ok
896512 897024 ok
897024 897536 ok
897536 898048 ok
898048 898560 ok
898560 899072 ok
899072 899584 ok
899584 900096 ok
900096 900608 ok
900608 901120 ok
901120 901632 ok
901632 902144 ok
902144 902656 ok
902656 903168 ok
903168 903680 ok
903680 904192 ok
904192 904704 ok
904704 905216 ok
905216 905728 ok
905728 906240 ok
906240 906752 ok
906752 907264 ok
907264 907776 ok
907776 908288 ok
908288 908800 ok
908800 909312 ok
909312 909824 ok
909824 910336 ok
910336 910848 ok
910848 911360 ok
911360 911872 ok
911872 912384 ok
912384 912896 ok
912896 913408 ok
913408 913920 ok
913920 914432 ok
914432 914944 ok
914944 915456 ok
915456 915968 ok
915968 916480 ok
916480 916992 ok
916992 917504 ok
917504 918016 ok
918016 918528 ok
918528 919040 ok
919040 919552 ok
919552 920064 ok
920064 920576 ok
920576 921088 ok
921088 921600 ok
921600 922112 ok
922112 922624 ok
922624 923136 ok
923136 923648 ok
923648 924160 ok
924160 924672 ok
924672 925184 ok
925184 925696 ok
925696 926208 ok
926208 926720 ok
926720 927232 ok
927232 927744 ok
927744 928256 ok
928256 928768 ok
928768 929280 ok
929280 929792 ok
929792 930304 ok
930304 930816 ok
930816 931328 ok
931328 931840 ok
931840 932352 ok
932352 932864 ok
932864 933376 ok
933376 933888 ok
933888 934400 ok
934400 934912 ok
934912 935424 ok
935424 935936 ok
935936 936448 ok
936448 936960 ok
936960 937472 ok
937472 937984 ok
937984 938496 ok
938496 939008 ok
939008 939520 ok
939520 940032 ok
940032 940544 ok
940544 941056 ok
941056 941568 ok
941568 942080 ok
942080 942592 ok
942592 943104 ok
943104 943616 ok
943616 944128 ok
944128 944640 ok
944640 945152 ok
945152 945664 ok
945664 946176 ok
946176 946688 ok
946688 947200 ok
947200 947712 ok
947712 948224 ok
948224 948736 ok
948736 949248 ok
949248 949760 ok
949760 950272 ok
950272 950784 ok
950784 951296 ok
951296 951808 ok
951808 952320 ok
952320 952832 ok
952832 953344 ok
953344 953856 ok
953856 954368 ok
954368 954880 ok
954880 955392 ok
955392 955904 ok
955904 956416 ok
956416 956928 ok
956928 957440 ok
957440 957952 ok
957952 958464 ok
958464 958976 ok
958976 959488 ok
959488 960000 ok
960000 960512 ok
960512 961024 ok
961024 961536 ok
961536 962048 ok
962048 962560 ok
962560 963072 ok
963072 963584 ok
963584 964096 ok
964096 964608 ok
964608 965120 ok
965120 965632 ok
965632 966144 ok
966144 966656 ok
966656 967168 ok
967168 967680 ok
967680 968192 ok
968192 968704 ok
968704 969216 ok
969216 969728 ok
969728 970240 ok
970240 970752 ok
970752 971264 ok
971264 971776 ok
971776 972288 ok
972288 972800 ok
972800 973312 ok
973312 973824 ok
973824 974336 ok
974336 974848 ok
974848 975360 ok
975360 975872 ok
975872 976384 ok
976384 976896 ok
976896 977408 ok
977408 977920 ok
977920 978432 ok
978432 978944 ok
978944 979456 ok
979456 979968 ok
979968 980480 ok
980480 980992 ok
980992 981504 ok
981504 982016 ok
982016 982528 ok
982528 983040 ok
983040 983552 ok
983552 984064 ok
984064 984576 ok
984576 985088 ok
985088 985600 ok
985600 986112 ok
986112 986624 ok
986624 987136 ok
987136 987648 ok
987648 988160 ok
988160 988672 ok
988672 989184 ok
989184 989696 ok
989696 990208 ok
990208 990720 ok
990720 991232 ok
991232 991744 ok
991744 992256 ok
992256 992768 ok
992768 993280 ok
993280 993792 ok
993792 994304 ok
994304 994816 ok
994816 995328 ok
995328 995840 ok
995840 996352 ok
996352 996864 ok
996864 997376 ok
997376 997888 ok
997888 998400 ok
998400 998912 ok
998912 999424 ok
999424 999936 ok
999936 1000448 ok
1000448 1000960 ok
1000960 1001472 ok
1001472 1001984 ok
1001984 1002496 ok
1002496 1003008 ok
1003008 1003520 ok
1003520 1004032 ok
1004032 1004544 ok
1004544 1005056 ok
1005056 1005568 ok
1005568 1006080 ok
1006080 1006592 ok
1006592 1007104 ok
1007104 1007616 ok
1007616 1008128 ok
1008128 1008640 ok
1008640 1009152 ok
1009152 1009664 ok
1009664 1010176 ok
1010176 1010688 ok
1010688 1011200 ok
1011200 1011712 ok
1011712 1012224 ok
1012224 1012736 ok
1012736 1013248 ok
1013248 1013760 ok
1013760 1014272 ok
1014272 1014784 ok
1014784 1015296 ok
1015296 1015808 ok
1015808 1016320 ok
1016320 1016832 ok
1016832 1017344 ok
1017344 1017856 ok
1017856 1018368 ok
1018368 1018880 ok
1018880 1019392 ok
1019392 1019904 ok
1019904 1020416 ok
1020416 1020928 ok
1020928 1021440 ok
1021440 1021952 ok
1021952 1022464 ok
1022464 1022976 ok
1022976 1023488 ok
1023488 1024000 ok
1024000 1024512 ok
1024512 1025024 ok
1025024 1025536 ok
1025536 1026048 ok
1026048 1026560 ok
1026560 1027072 ok
1027072 1027584 ok
1027584 1028096 ok
1028096 1028608 ok
1028608 1029120 ok
1029120 1029632 ok
1029632 1030144 ok
1030144 1030656 ok
1030656 1031168 ok
1031168 1031680 ok
1031680 1032192 ok
1032192 1032704 ok
1032704 1033216 ok
1033216 1033728 ok
1033728 1034240 ok
1034240 1034752 ok
1034752 1035264 ok
1035264 1035776 ok
1035776 1036288 ok
1036288 1036800 ok
1036800 1037312 ok
1037312 1037824 ok
1037824 1038336 ok
1038336 1038848 ok
1038848 1039360 ok
1039360 1039872 ok
1039872 1040384 ok
1040384 1040896 ok
1040896 1041408 ok
1041408 1041920 ok
1041920 1042432 ok
1042432 1042944 ok
1042944 1043456 ok
1043456 1043968 ok
1043968 1044480 ok
1044480 1044992 ok
1044992 1045504 ok
1045504 1046016 ok
1046016 1046528 ok
1046528 1047040 ok
1047040 1047552 ok
1047552 1048064 ok
1048064 1048576 ok
1048576 1049088 ok
1049088 1049600 ok
1049600 1050112 ok
1050112 1050624 ok
1050624 1051136 ok
1051136 1051648 ok
1051648 1052160 ok
1052160 1052672 ok
1052672 1053184 ok
1053184 1053696 ok
1053696 1054208 ok
1054208 1054720 ok
1054720 1055232 ok
1055232 1055744 ok
1055744 1056256 ok
1056256 1056768 ok
1056768 1057280 ok
1057280 1057792 ok
1057792 1058304 ok
1058304 1058816 ok
1058816 1059328 ok
1059328 1059840 ok
1059840 1060352 ok
1060352 1060864 ok
1060864 1061376 ok
1061376 1061888 ok
1061888 1062400 ok
1062400 1062912 ok
1062912 1063424 ok
1063424 1063936 ok
1063936 1064448 ok
1064448 1064960 ok
1064960 1065472 ok
1065472 1065984 ok
1065984 1066496 ok
1066496 1067008 ok
1067008 1067520 ok
1067520 1068032 ok
1068032 1068544 ok
1068544 1069056 ok
1069056 1069568 ok
1069568 1070080 ok
1070080 1070592 ok
1070592 1071104 ok
1071104 1071616 ok
1071616 1072128 ok
1072128 1072640 ok
1072640 1073152 ok
1073152 1073664 ok
1073664 1074176 ok
1074176 1074688 ok
1074688 1075200 ok
1075200 1075712 ok
1075712 1076224 ok
1076224 1076736 ok
1076736 1077248 ok
1077248 1077760 ok
1077760 1078272 ok
1078272 1078784 ok
1078784 1079296 ok
1079296 1079808 ok
1079808 1080320 ok
1080320 1080832 ok
1080832 1081344 ok
1081344 1081856 ok
1081856 1082368 ok
1082368 1082880 ok
1082880 1083392 ok
1083392 1083904 ok
1083904 1084416 ok
1084416 1084928 ok
1084928 1085440 ok
1085440 1085952 ok
1085952 1086464 ok
1086464 1086976 ok
1086976 1087488 ok
1087488 1088000 ok
1088000 1088512 ok
1088512 1089024 ok
1089024 1089536 ok
1089536 1090048 ok
1090048 1090560 ok
1090560 1091072 ok
1091072 1091584 ok
1091584 1092096 ok
1092096 1092608 ok
1092608 1093120 ok
1093120 1093632 ok
1093632 1094144 ok
1094144 1094656 ok
1094656 1095168 ok
1095168 1095680 ok
1095680 1096192 ok
1096192 1096704 ok
1096704 1097216 ok
1097216 1097728 ok
1097728 1098240 ok
1098240 1098752 ok
1098752 1099264 ok
1099264 1099776 ok
1099776 1100288 ok
1100288 1100800 ok
1100800 1101312 ok
1101312 1101824 ok
1101824 1102336 ok
1102336 1102848 ok
1102848 1103360 ok
1103360 1103872 ok
1103872 1104384 ok
1104384 1104896 ok
1104896 1105408 ok
1105408 1105920 ok
1105920 1106432 ok
1106432 1106944 ok
1106944 1107456 ok
1107456 1107968 ok
1107968 1108480 ok
1108480 1108992 ok
1108992 1109504 ok
1109504 1110016 ok
1110016 1110528 ok
1110528 1111040 ok
1111040 1111552 ok
1111552 1112064 ok
1112064 1112576 ok
1112576 1113088 ok
1113088 1113600 ok
1113600 1114112 ok
1114112 1114624 ok
1114624 1115136 ok
1115136 1115648 ok
1115648 1116160 ok
1116160 1116672 ok
1116672 1117184 ok
1117184 1117696 ok
1117696 1118208 ok
1118208 1118720 ok
1118720 1119232 ok
1119232 1119744 ok
1119744 1120256 ok
1120256 1120768 ok
1120768 1121280 ok
1121280 1121792 ok
1121792 1122304 ok
1122304 1122816 ok
1122816 1123328 ok
1123328 1123840 ok
1123840 1124352 ok
1124352 1124864 ok
1124864 1125376 ok
1125376 1125888 ok
1125888 1126400 ok
1126400 1126912 ok
1126912 1127424 ok
1127424 1127936 ok
1127936 1128448 ok
1128448 1128960 ok
1128960 1129472 ok
1129472 1129984 ok
1129984 1130496 ok
1130496 1131008 ok
1131008 1131520 ok
1131520 1132032 ok
1132032 1132544 ok
1132544 1133056 ok
1133056 1133568 ok
1133568 1134080 ok
1134080 1134592 ok
1134592 1135104 ok
1135104 1135616 ok
1135616 1136128 ok
1136128 1136640 ok
1136640 1137152 ok
1137152 1137664 ok
1137664 1138176 ok
1138176 1138688 ok
1138688 1139200 ok
1139200 1139712 ok
1139712 1140224 ok
1140224 1140736 ok
1140736 1141248 ok
1141248 1141760 ok
1141760 1142272 ok
1142272 1142784 ok
1142784 1143296 ok
1143296 1143808 ok
1143808 1144320 ok
1144320 1144832 ok
1144832 1145344 ok
1145344 1145856 ok
1145856 1146368 ok
1146368 1146880 ok
1146880 1147392 ok
1147392 1147904 ok
1147904 1148416 ok
1148416 1148928 ok
1148928 1149440 ok
1149440 1149952 ok
1149952 1150464 ok
1150464 1150976 ok
1150976 1151488 ok
1151488 1152000 ok
1152000 1152512 ok
1152512 1153024 ok
1153024 1153536 ok
1153536 1154048 ok
1154048 1154560 ok
1154560 1155072 ok
1155072 1155584 ok
1155584 1156096 ok
1156096 1156608 ok
1156608 1157120 ok
1157120 1157632 ok
1157632 1158144 ok
1158144 1158656 ok
1158656 1159168 ok
1159168 1159680 ok
1159680 1160192 ok
1160192 1160704 ok
1160704 1161216 ok
1161216 1161728 ok
1161728 1162240 ok
1162240 1162752 ok
1162752 1163264 ok
1163264 1163776 ok
1163776 1164288 ok
1164288 1164800 ok
1164800 1165312 ok
1165312 1165824 ok
1165824 1166336 ok
1166336 1166848 ok
1166848 1167360 ok
1167360 1167872 ok
1167872 1168384 ok
1168384 1168896 ok
1168896 1169408 ok
1169408 1169920 ok
1169920 1170432 ok
1170432 1170944 ok
1170944 1171456 ok
1171456 1171968 ok
1171968 1172480 ok
1172480 1172992 ok
1172992 1173504 ok
1173504 1174016 ok
1174016 1174528 ok
1174528 1175040 ok
1175040 1175552 ok
1175552 1176064 ok
1176064 1176576 ok
1176576 1177088 ok
1177088 1177600 ok
1177600 1178112 ok
1178112 1178624 ok
1178624 1179136 ok
1179136 1179648 ok
1179648 1180160 ok
1180160 1180672 ok
1180672 1181184 ok
1181184 1181696 ok
1181696 1182208 ok
1182208 1182720 ok
1182720 1183232 ok
1183232 1183744 ok
1183744 1184256 ok
1184256 1184768 ok
1184768 1185280 ok
1185280 1185792 ok
1185792 1186304 ok
1186304 1186816 ok
1186816 1187328 ok
1187328 1187840 ok
1187840 1188352 ok
1188352 1188864 ok
1188864 1189376 ok
1189376 1189888 ok
1189888 1190400 ok
1190400 1190912 ok
1190912 1191424 ok
1191424 1191936 ok
1191936 1192448 ok
1192448 1192960 ok
1192960 1193472 ok
1193472 1193984 ok
1193984 1194496 ok
1194496 1195008 ok
1195008 1195520 ok
1195520 1196032 ok
1196032 1196544 ok
1196544 1197056 ok
1197056 1197568 ok
1197568 1198080 ok
1198080 1198592 ok
1198592 1199104 ok
1199104 1199616 ok
1199616 1200128 ok
1200128 1200640 ok
1200640 1201152 ok
1201152 1201664 ok
1201664 1202176 ok
1202176 1202688 ok
1202688 1203200 ok
1203200 1203712 ok
1203712 1204224 ok
1204224 1204736 ok
1204736 1205248 ok
1205248 1205760 ok
1205760 1206272 ok
1206272 1206784 ok
1206784 1207296 ok
1207296 1207808 ok
1207808 1208320 ok
1208320 1208832 ok
1208832 1209344 ok
1209344 1209856 ok
1209856 1210368 ok
1210368 1210880 ok
1210880 1211392 ok
1211392 1211904 ok
1211904 1212416 ok
1212416 1212928 ok
1212928 1213440 ok
1213440 1213952 ok
1213952 1214464 ok
1214464 1214976 ok
1214976 1215488 ok
1215488 1216000 ok
1216000 1216512 ok
1216512 1217024 ok
1217024 1217536 ok
1217536 1218048 ok
1218048 1218560 ok
1218560 1219072 ok
1219072 1219584 ok
1219584 1220096 ok
1220096 1220608 ok
1220608 1221120 ok
1221120 1221632 ok
1221632 1222144 ok
1222144 1222656 ok
1222656 1223168 ok
1223168 1223680 ok
1223680 1224192 ok
1224192 1224704 ok
1224704 1225216 ok
1225216 1225728 ok
1225728 1226240 ok
1226240 1226752 ok
1226752 1227264 ok
1227264 1227776 ok
1227776 1228288 ok
1228288 1228800 ok
1228800 1229312 ok
1229312 1229824 ok
1229824 1230336 ok
1230336 1230848 ok
1230848 1231360 ok
1231360 1231872 ok
1231872 1232384 ok
1232384 1232896 ok
1232896 1233408 ok
1233408 1233920 ok
1233920 1234432 ok
1234432 1234944 ok
1234944 1235456 ok
1235456 1235968 ok
1235968 1236480 ok
1236480 1236992 ok
1236992 1237504 ok
1237504 1238016 ok
1238016 1238528 ok
1238528 1239040 ok
1239040 1239552 ok
1239552 1240064 ok
1240064 1240576 ok
1240576 1241088 ok
1241088 1241600 ok
1241600 1242112 ok
1242112 1242624 ok
1242624 1243136 ok
1243136 1243648 ok
1243648 1244160 ok
1244160 1244672 ok
1244672 1245184 ok
1245184 1245696 ok
1245696 1246208 ok
1246208 1246720 ok
1246720 1247232 ok
1247232 1247744 ok
1247744 1248256 ok
1248256 1248768 ok
1248768 1249280 ok
1249280 1249792 ok
1249792 1250304 ok
1250304 1250816 ok
1250816 1251328 ok
1251328 1251840 ok
1251840 1252352 ok
1252352 1252864 ok
1252864 1253376 ok
1253376 1253888 ok
1253888 1254400 ok
1254400 1254912 ok
1254912 1255424 ok
1255424 1255936 ok
1255936 1256448 ok
1256448 1256960 ok
1256960 1257472 ok
1257472 1257984 ok
1257984 1258496 ok
1258496 1259008 ok
1259008 1259520 ok
1259520 1260032 ok
1260032 1260544 ok
1260544 1261056 ok
1261056 1261568 ok
1261568 1262080 ok
1262080 1262592 ok
1262592 1263104 ok
1263104 1263616 ok
1263616 1264128 ok
1264128 1264640 ok
1264640 1265152 ok
1265152 1265664 ok
1265664 1266176 ok
1266176 1266688 ok
1266688 1267200 ok
1267200 1267712 ok
1267712 1268224 ok
1268224 1268736 ok
1268736 1269248 ok
1269248 1269760 ok
1269760 1270272 ok
1270272 1270784 ok
1270784 1271296 ok
1271296 1271808 ok
1271808 1272320 ok
1272320 1272832 ok
1272832 1273344 ok
1273344 1273856 ok
1273856 1274368 ok
1274368 1274880 ok
1274880 1275392 ok
1275392 1275904 ok
1275904 1276416 ok
1276416 1276928 ok
1276928 1277440 ok
1277440 1277952 ok
1277952 1278464 ok
1278464 1278976 ok
1278976 1279488 ok
1279488 1280000 ok
1280000 1280512 ok
1280512 1281024 ok
1281024 1281536 ok
1281536 1282048 ok
1282048 1282560 ok
1282560 1283072 ok
1283072 1283584 ok
1283584 1284096 ok
1284096 1284608 ok
1284608 1285120 ok
1285120 1285632 ok
1285632 1286144 ok
1286144 1286656 ok
1286656 1287168 ok
1287168 1287680 ok
1287680 1288192 ok
1288192 1288704 ok
1288704 1289216 ok
1289216 1289728 ok
1289728 1290240 ok
1290240 1290752 ok
1290752 1291264 ok
1291264 1291776 ok
1291776 1292288 ok
1292288 1292800 ok
1292800 1293312 ok
1293312 1293824 ok
1293824 1294336 ok
1294336 1294848 ok
1294848 1295360 ok
1295360 1295872 ok
1295872 1296384 ok
1296384 1296896 ok
1296896 1297408 ok
1297408 1297920 ok
1297920 1298432 ok
1298432 1298944 ok
1298944 1299456 ok
1299456 1299968 ok
1299968 1300480 ok
1300480 1300992 ok
1300992 1301504 ok
1301504 1302016 ok
1302016 1302528 ok
1302528 1303040 ok
1303040 1303552 ok
1303552 1304064 ok
1304064 1304576 ok
1304576 1305088 ok
1305088 1305600 ok
1305600 1306112 ok
1306112 1306624 ok
1306624 1307136 ok
1307136 1307648 ok
1307648 1308160 ok
1308160 1308672 ok
1308672 1309184 ok
1309184 1309696 ok
1309696 1310208 ok
1310208 1310720 ok
1310720 1311232 ok
1311232 1311744 ok
1311744 1312256 ok
1312256 1312768 ok
1312768 1313280 ok
1313280 1313792 ok
1313792 1314304 ok
1314304 1314816 ok
1314816 1315328 ok
1315328 1315840 ok
1315840 1316352 ok
1316352 1316864 ok
1316864 1317376 ok
1317376 1317888 ok
1317888 1318400 ok
1318400 1318912 ok
1318912 1319424 ok
1319424 1319936 ok
1319936 1320448 ok
1320448 1320960 ok
1320960 1321472 ok
1321472 1321984 ok
1321984 1322496 ok
1322496 1323008 ok
1323008 1323520 ok
1323520 1324032 ok
1324032 1324544 ok
1324544 1325056 ok
1325056 1325568 ok
1325568 1326080 ok
1326080 1326592 ok
1326592 1327104 ok
1327104 1327616 ok
1327616 1328128 ok
1328128 1328640 ok
1328640 1329152 ok
1329152 1329664 ok
1329664 1330176 ok
1330176 1330688 ok
1330688 1331200 ok
1331200 1331712 ok
1331712 1332224 ok
1332224 1332736 ok
1332736 1333248 ok
1333248 1333760 ok
1333760 1334272 ok
1334272 1334784 ok
1334784 1335296 ok
1335296 1335808 ok
1335808 1336320 ok
1336320 1336832 ok
1336832 1337344 ok
1337344 1337856 ok
1337856 1338368 ok
1338368 1338880 ok
1338880 1339392 ok
1339392 1339904 ok
1339904 1340416 ok
1340416 1340928 ok
1340928 1341440 ok
1341440 1341952 ok
1341952 1342464 ok
1342464 1342976 ok
1342976 1343488 ok
1343488 1344000 ok
1344000 1344512 ok
1344512 1345024 ok
1345024 1345536 ok
1345536 1346048 ok
1346048 1346560 ok
1346560 1347072 ok
1347072 1347584 ok
1347584 1348096 ok
1348096 1348608 ok
1348608 1349120 ok
1349120 1349632 ok
1349632 1350144 ok
1350144 1350656 ok
1350656 1351168 ok
1351168 1351680 ok
1351680 1352192 ok
1352192 1352704 ok
1352704 1353216 ok
1353216 1353728 ok
1353728 1354240 ok
1354240 1354752 ok
1354752 1355264 ok
1355264 1355776 ok
1355776 1356288 ok
1356288 1356800 ok
1356800 1357312 ok
1357312 1357824 ok
1357824 1358336 ok
1358336 1358848 ok
1358848 1359360 ok
1359360 1359872 ok
1359872 1360384 ok
1360384 1360896 ok
1360896 1361408 ok
1361408 1361920 ok
1361920 1362432 ok
1362432 1362944 ok
1362944 1363456 ok
1363456 1363968 ok
1363968 1364480 ok
1364480 1364992 ok
1364992 1365504 ok
1365504 1366016 ok
1366016 1366528 ok
1366528 1367040 ok
1367040 1367552 ok
1367552 1368064 ok
1368064 1368576 ok
1368576 1369088 ok
1369088 1369600 ok
1369600 1370112 ok
1370112 1370624 ok
1370624 1371136 ok
1371136 1371648 ok
1371648 1372160 ok
1372160 1372672 ok
1372672 1373184 ok
1373184 1373696 ok
1373696 1374208 ok
1374208 1374720 ok
1374720 1375232 ok
1375232 1375744 ok
1375744 1376256 ok
1376256 1376768 ok
1376768 1377280 ok
1377280 1377792 ok
1377792 1378304 ok
1378304 1378816 ok
1378816 1379328 ok
1379328 1379840 ok
1379840 1380352 ok
1380352 1380864 ok
1380864 1381376 ok
1381376 1381888 ok
1381888 1382400 ok
1382400 1382912 ok
1382912 1383424 ok
1383424 1383936 ok
1383936 1384448 ok
1384448 1384960 ok
1384960 1385472 ok
1385472 1385984 ok
1385984 1386496 ok
1386496 1387008 ok
1387008 1387520 ok
1387520 1388032 ok
1388032 1388544 ok
1388544 1389056 ok
1389056 1389568 ok
1389568 1390080 ok
1390080 1390592 ok
1390592 1391104 ok
1391104 1391616 ok
1391616 1392128 ok
1392128 1392640 ok
1392640 1393152 ok
1393152 1393664 ok
1393664 1394176 ok
1394176 1394688 ok
1394688 1395200 ok
1395200 1395712 ok
1395712 1396224 ok
1396224 1396736 ok
1396736 1397248 ok
1397248 1397760 ok
1397760 1398272 ok
1398272 1398784 ok
1398784 1399296 ok
1399296 1399808 ok
1399808 1400320 ok
1400320 1400832 ok
1400832 1401344 ok
1401344 1401856 ok
1401856 1402368 ok
1402368 1402880 ok
1402880 1403392 ok
1403392 1403904 ok
1403904 1404416 ok
1404416 1404928 ok
1404928 1405440 ok
1405440 1405952 ok
1405952 1406464 ok
1406464 1406976 ok
1406976 1407488 ok
1407488 1408000 ok
1408000 1408512 ok
1408512 1409024 ok
1409024 1409536 ok
1409536 1410048 ok
1410048 1410560 ok
1410560 1411072 ok
1411072 1411584 ok
1411584 1412096 ok
1412096 1412608 ok
1412608 1413120 ok
1413120 1413632 ok
1413632 1414144 ok
1414144 1414656 ok
1414656 1415168 ok
1415168 1415680 ok
1415680 1416192 ok
1416192 1416704 ok
1416704 1417216 ok
1417216 1417728 ok
1417728 1418240 ok
1418240 1418752 ok
1418752 1419264 ok
1419264 1419776 ok
1419776 1420288 ok
1420288 1420800 ok
1420800 1421312 ok
1421312 1421824 ok
1421824 1422336 ok
1422336 1422848 ok
1422848 1423360 ok
1423360 1423872 ok
1423872 1424384 ok
1424384 1424896 ok
1424896 1425408 ok
1425408 1425920 ok
1425920 1426432 ok
1426432 1426944 ok
1426944 1427456 ok
1427456 1427968 ok
1427968 1428480 ok
1428480 1428992 ok
1428992 1429504 ok
1429504 1430016 ok
1430016 1430528 ok
1430528 1431040 ok
1431040 1431552 ok
1431552 1432064 ok
1432064 1432576 ok
1432576 1433088 ok
1433088 1433600 ok
1433600 1434112 ok
1434112 1434624 ok
1434624 1435136 ok
1435136 1435648 ok
1435648 1436160 ok
1436160 1436672 ok
1436672 1437184 ok
1437184 1437696 ok
1437696 1438208 ok
1438208 1438720 ok
1438720 1439232 ok
1439232 1439744 ok
1439744 1440256 ok
1440256 1440768 ok
1440768 1441280 ok
1441280 1441792 ok
1441792 1442304 ok
1442304 1442816 ok
1442816 1443328 ok
1443328 1443840 ok
1443840 1444352 ok
1444352 1444864 ok
1444864 1445376 ok
1445376 1445888 ok
1445888 1446400 ok
1446400 1446912 ok
1446912 1447424 ok
1447424 1447936 ok
1447936 1448448 ok
1448448 1448960 ok
1448960 1449472 ok
1449472 1449984 ok
1449984 1450496 ok
1450496 1451008 ok
1451008 1451520 ok
1451520 1452032 ok
1452032 1452544 ok
1452544 1453056 ok
1453056 1453568 ok
1453568 1454080 ok
1454080 1454592 ok
1454592 1455104 ok
1455104 1455616 ok
1455616 1456128 ok
1456128 1456640 ok
1456640 1457152 ok
1457152 1457664 ok
1457664 1458176 ok
1458176 1458688 ok
1458688 1459200 ok
1459200 1459712 ok
1459712 1460224 ok
1460224 1460736 ok
1460736 1461248 ok
1461248 1461760 ok
1461760 1462272 ok
1462272 1462784 ok
1462784 1463296 ok
1463296 1463808 ok
1463808 1464320 ok
1464320 1464832 ok
1464832 1465344 ok
1465344 1465856 ok
1465856 1466368 ok
1466368 1466880 ok
1466880 1467392 ok
1467392 1467904 ok
1467904 1468416 ok
1468416 1468928 ok
1468928 1469440 ok
1469440 1469952 ok
1469952 1470464 ok
1470464 1470976 ok
1470976 1471488 ok
1471488 1472000 ok
1472000 1472512 ok
1472512 1473024 ok
1473024 1473536 ok
1473536 1474048 ok
1474048 1474560 ok
1474560 1475072 ok
1475072 1475584 ok
1475584 1476096 ok
1476096 1476608 ok
1476608 1477120 ok
1477120 1477632 ok
1477632 1478144 ok
1478144 1478656 ok
1478656 1479168 ok
1479168 1479680 ok
1479680 1480192 ok
1480192 1480704 ok
1480704 1481216 ok
1481216 1481728 ok
1481728 1482240 ok
1482240 1482752 ok
1482752 1483264 ok
1483264 1483776 ok
1483776 1484288 ok
1484288 1484800 ok
1484800 1485312 ok
1485312 1485824 ok
1485824 1486336 ok
1486336 1486848 ok
1486848 1487360 ok
1487360 1487872 ok
1487872 1488384 ok
1488384 1488896 ok
1488896 1489408 ok
1489408 1489920 ok
1489920 1490432 ok
1490432 1490944 ok
1490944 1491456 ok
1491456 1491968 ok
1491968 1492480 ok
1492480 1492992 ok
1492992 1493504 ok
1493504 1494016 ok
1494016 1494528 ok
1494528 1495040 ok
1495040 1495552 ok
1495552 1496064 ok
1496064 1496576 ok
1496576 1497088 ok
1497088 1497600 ok
1497600 1498112 ok
1498112 1498624 ok
1498624 1499136 ok
1499136 1499648 ok
1499648 1500160 ok
1500160 1500672 ok
1500672 1501184 ok
1501184 1501696 ok
1501696 1502208 ok
1502208 1502720 ok
1502720 1503232 ok
1503232 1503744 ok
1503744 1504256 ok
1504256 1504768 ok
1504768 1505280 ok
1505280 1505792 ok
1505792 1506304 ok
1506304 1506816 ok
1506816 1507328 ok
1507328 1507840 ok
1507840 1508352 ok
1508352 1508864 ok
1508864 1509376 ok
1509376 1509888 ok
1509888 1510400 ok
1510400 1510912 ok
1510912 1511424 ok
1511424 1511936 ok
1511936 1512448 ok
1512448 1512960 ok
1512960 1513472 ok
1513472 1513984 ok
1513984 1514496 ok
1514496 1515008 ok
1515008 1515520 ok
1515520 1516032 ok
1516032 1516544 ok
1516544 1517056 ok
1517056 1517568 ok
1517568 1518080 ok
1518080 1518592 ok
1518592 1519104 ok
1519104 1519616 ok
1519616 1520128 ok
1520128 1520640 ok
1520640 1521152 ok
1521152 1521664 ok
1521664 1522176 ok
1522176 1522688 ok
1522688 1523200 ok
1523200 1523712 ok
1523712 1524224 ok
1524224 1524736 ok
1524736 1525248 ok
1525248 1525760 ok
1525760 1526272 ok
1526272 1526784 ok
1526784 1527296 ok
1527296 1527808 ok
1527808 1528320 ok
1528320 1528832 ok
1528832 1529344 ok
1529344 1529856 ok
1529856 1530368 ok
1530368 1530880 ok
1530880 1531392 ok
1531392 1531904 ok
1531904 1532416 ok
1532416 1532928 ok
1532928 1533440 ok
1533440 1533952 ok
1533952 1534464 ok
1534464 1534976 ok
1534976 1535488 ok
1535488 1536000 ok
1536000 1536512 ok
1536512 1537024 ok
1537024 1537536 ok
1537536 1538048 ok
1538048 1538560 ok
1538560 1539072 ok
1539072 1539584 ok
1539584 1540096 ok
1540096 1540608 ok
1540608 1541120 ok
1541120 1541632 ok
1541632 1542144 ok
1542144 1542656 ok
1542656 1543168 ok
1543168 1543680 ok
1543680 1544192 ok
1544192 1544704 ok
1544704 1545216 ok
1545216 1545728 ok
1545728 1546240 ok
1546240 1546752 ok
1546752 1547264 ok
1547264 1547776 ok
1547776 1548288 ok
1548288 1548800 ok
1548800 1549312 ok
1549312 1549824 ok
1549824 1550336 ok
1550336 1550848 ok
1550848 1551360 ok
1551360 1551872 ok
1551872 1552384 ok
1552384 1552896 ok
1552896 1553408 ok
1553408 1553920 ok
1553920 1554432 ok
1554432 1554944 ok
1554944 1555456 ok
1555456 1555968 ok
1555968 1556480 ok
1556480 1556992 ok
1556992 1557504 ok
1557504 1558016 ok
1558016 1558528 ok
1558528 1559040 ok
1559040 1559552 ok
1559552 1560064 ok
1560064 1560576 ok
1560576 1561088 ok
1561088 1561600 ok
1561600 1562112 ok
1562112 1562624 ok
1562624 1563136 ok
1563136 1563648 ok
1563648 1564160 ok
1564160 1564672 ok
1564672 1565184 ok
1565184 1565696 ok
1565696 1566208 ok
1566208 1566720 ok
1566720 1567232 ok
1567232 1567744 ok
1567744 1568256 ok
1568256 1568768 ok
1568768 1569280 ok
1569280 1569792 ok
1569792 1570304 ok
1570304 1570816 ok
1570816 1571328 ok
1571328 1571840 ok
1571840 1572352 ok
1572352 1572864 ok
1572864 1573376 ok
1573376 1573888 ok
1573888 1574400 ok
1574400 1574912 ok
1574912 1575424 ok
1575424 1575936 ok
1575936 1576448 ok
1576448 1576960 ok
1576960 1577472 ok
1577472 1577984 ok
1577984 1578496 ok
1578496 1579008 ok
1579008 1579520 ok
1579520 1580032 ok
1580032 1580544 ok
1580544 1581056 ok
1581056 1581568 ok
1581568 1582080 ok
1582080 1582592 ok
1582592 1583104 ok
1583104 1583616 ok
1583616 1584128 ok
1584128 1584640 ok
1584640 1585152 ok
1585152 1585664 ok
1585664 1586176 ok
1586176 1586688 ok
1586688 1587200 ok
1587200 1587712 ok
1587712 1588224 ok
1588224 1588736 ok
1588736 1589248 ok
1589248 1589760 ok
1589760 1590272 ok
1590272 1590784 ok
1590784 1591296 ok
1591296 1591808 ok
1591808 1592320 ok
1592320 1592832 ok
1592832 1593344 ok
1593344 1593856 ok
1593856 1594368 ok
1594368 1594880 ok
1594880 1595392 ok
1595392 1595904 ok
1595904 1596416 ok
1596416 1596928 ok
1596928 1597440 ok
1597440 1597952 ok
1597952 1598464 ok
1598464 1598976 ok
1598976 1599488 ok
1599488 1600000 ok
1600000 1600512 ok
1600512 1601024 ok
1601024 1601536 ok
1601536 1602048 ok
1602048 1602560 ok
1602560 1603072 ok
1603072 1603584 ok
1603584 1604096 ok
1604096 1604608 ok
1604608 1605120 ok
1605120 1605632 ok
1605632 1606144 ok
1606144 1606656 ok
1606656 1607168 ok
1607168 1607680 ok
1607680 1608192 ok
1608192 1608704 ok
1608704 1609216 ok
1609216 1609728 ok
1609728 1610240 ok
1610240 1610752 ok
1610752 1611264 ok
1611264 1611776 ok
1611776 1612288 ok
1612288 1612800 ok
1612800 1613312 ok
1613312 1613824 ok
1613824 1614336 ok
1614336 1614848 ok
1614848 1615360 ok
1615360 1615872 ok
1615872 1616384 ok
1616384 1616896 ok
1616896 1617408 ok
1617408 1617920 ok
1617920 1618432 ok
1618432 1618944 ok
1618944 1619456 ok
1619456 1619968 ok
1619968 1620480 ok
1620480 1620992 ok
1620992 1621504 ok
1621504 1622016 ok
1622016 1622528 ok
1622528 1623040 ok
1623040 1623552 ok
1623552 1624064 ok
1624064 1624576 ok
1624576 1625088 ok
1625088 1625600 ok
1625600 1626112 ok
1626112 1626624 ok
1626624 1627136 ok
1627136 1627648 ok
1627648 1628160 ok
1628160 1628672 ok
1628672 1629184 ok
1629184 1629696 ok
1629696 1630208 ok
1630208 1630720 ok
1630720 1631232 ok
1631232 1631744 ok
1631744 1632256 ok
1632256 1632768 ok
1632768 1633280 ok
1633280 1633792 ok
1633792 1634304 ok
1634304 1634816 ok
1634816 1635328 ok
1635328 1635840 ok
1635840 1636352 ok
1636352 1636864 ok
1636864 1637376 ok
1637376 1637888 ok
1637888 1638400 ok
1638400 1638912 ok
1638912 1639424 ok
1639424 1639936 ok
1639936 1640448 ok
1640448 1640960 ok
1640960 1641472 ok
1641472 1641984 ok
1641984 1642496 ok
1642496 1643008 ok
1643008 1643520 ok
1643520 1644032 ok
1644032 1644544 ok
1644544 1645056 ok
1645056 1645568 ok
1645568 1646080 ok
1646080 1646592 ok
1646592 1647104 ok
1647104 1647616 ok
1647616 1648128 ok
1648128 1648640 ok
1648640 1649152 ok
1649152 1649664 ok
1649664 1650176 ok
1650176 1650688 ok
1650688 1651200 ok
1651200 1651712 ok
1651712 1652224 ok
1652224 1652736 ok
1652736 1653248 ok
1653248 1653760 ok
1653760 1654272 ok
1654272 1654784 ok
1654784 1655296 ok
1655296 1655808 ok
1655808 1656320 ok
1656320 1656832 ok
1656832 1657344 ok
1657344 1657856 ok
1657856 1658368 ok
1658368 1658880 ok
1658880 1659392 ok
1659392 1659904 ok
1659904 1660416 ok
1660416 1660928 ok
1660928 1661440 ok
1661440 1661952 ok
1661952 1662464 ok
1662464 1662976 ok
1662976 1663488 ok
1663488 1664000 ok
1664000 1664512 ok
1664512 1665024 ok
1665024 1665536 ok
1665536 1666048 ok
1666048 1666560 ok
1666560 1667072 ok
1667072 1667584 ok
1667584 1668096 ok
1668096 1668608 ok
1668608 1669120 ok
1669120 1669632 ok
1669632 1670144 ok
1670144 1670656 ok
1670656 1671168 ok
1671168 1671680 ok
1671680 1672192 ok
1672192 1672704 ok
1672704 1673216 ok
1673216 1673728 ok
1673728 1674240 ok
1674240 1674752 ok
1674752 1675264 ok
1675264 1675776 ok
1675776 1676288 ok
1676288 1676800 ok
1676800 1677312 ok
1677312 1677824 ok
1677824 1678336 ok
1678336 1678848 ok
1678848 1679360 ok
1679360 1679872 ok
1679872 1680384 ok
1680384 1680896 ok
1680896 1681408 ok
1681408 1681920 ok
1681920 1682432 ok
1682432 1682944 ok
1682944 1683456 ok
1683456 1683968 ok
1683968 1684480 ok
1684480 1684992 ok
1684992 1685504 ok
1685504 1686016 ok
1686016 1686528 ok
1686528 1687040 ok
1687040 1687552 ok
1687552 1688064 ok
1688064 1688576 ok
1688576 1689088 ok
1689088 1689600 ok
1689600 1690112 ok
1690112 1690624 ok
1690624 1691136 ok
1691136 1691648 ok
1691648 1692160 ok
1692160 1692672 ok
1692672 1693184 ok
1693184 1693696 ok
1693696 1694208 ok
1694208 1694720 ok
1694720 1695232 ok
1695232 1695744 ok
1695744 1696256 ok
1696256 1696768 ok
1696768 1697280 ok
1697280 1697792 ok
1697792 1698304 ok
1698304 1698816 ok
1698816 1699328 ok
1699328 1699840 ok
1699840 1700352 ok
1700352 1700864 ok
1700864 1701376 ok
1701376 1701888 ok
1701888 1702400 ok
1702400 1702912 ok
1702912 1703424 ok
1703424 1703936 ok
1703936 1704448 ok
1704448 1704960 ok
1704960 1705472 ok
1705472 1705984 ok
1705984 1706496 ok
1706496 1707008 ok
1707008 1707520 ok
1707520 1708032 ok
1708032 1708544 ok
1708544 1709056 ok
1709056 1709568 ok
1709568 1710080 ok
1710080 1710592 ok
1710592 1711104 ok
1711104 1711616 ok
1711616 1712128 ok
1712128 1712640 ok
1712640 1713152 ok
1713152 1713664 ok
1713664 1714176 ok
1714176 1714688 ok
1714688 1715200 ok
1715200 1715712 ok
1715712 1716224 ok
1716224 1716736 ok
1716736 1717248 ok
1717248 1717760 ok
1717760 1718272 ok
1718272 1718784 ok
1718784 1719296 ok
1719296 1719808 ok
1719808 1720320 ok
1720320 1720832 ok
1720832 1721344 ok
1721344 1721856 ok
1721856 1722368 ok
1722368 1722880 ok
1722880 1723392 ok
1723392 1723904 ok
1723904 1724416 ok
1724416 1724928 ok
1724928 1725440 ok
1725440 1725952 ok
1725952 1726464 ok
1726464 1726976 ok
1726976 1727488 ok
1727488 1728000 ok
1728000 1728512 ok
1728512 1729024 ok
1729024 1729536 ok
1729536 1730048 ok
1730048 1730560 ok
1730560 1731072 ok
1731072 1731584 ok
1731584 1732096 ok
1732096 1732608 ok
1732608 1733120 ok
1733120 1733632 ok
1733632 1734144 ok
1734144 1734656 ok
1734656 1735168 ok
1735168 1735680 ok
1735680 1736192 ok
1736192 1736704 ok
1736704 1737216 ok
1737216 1737728 ok
1737728 1738240 ok
1738240 1738752 ok
1738752 1739264 ok
1739264 1739776 ok
1739776 1740288 ok
1740288 1740800 ok
1740800 1741312 ok
1741312 1741824 ok
1741824 1742336 ok
1742336 1742848 ok
1742848 1743360 ok
1743360 1743872 ok
1743872 1744384 ok
1744384 1744896 ok
1744896 1745408 ok
1745408 1745920 ok
1745920 1746432 ok
1746432 1746944 ok
1746944 1747456 ok
1747456 1747968 ok
1747968 1748480 ok
1748480 1748992 ok
1748992 1749504 ok
1749504 1750016 ok
1750016 1750528 ok
1750528 1751040 ok
1751040 1751552 ok
1751552 1752064 ok
1752064 1752576 ok
1752576 1753088 ok
1753088 1753600 ok
1753600 1754112 ok
1754112 1754624 ok
1754624 1755136 ok
1755136 1755648 ok
1755648 1756160 ok
1756160 1756672 ok
1756672 1757184 ok
1757184 1757696 ok
1757696 1758208 ok
1758208 1758720 ok
1758720 1759232 ok
1759232 1759744 ok
1759744 1760256 ok
1760256 1760768 ok
1760768 1761280 ok
1761280 1761792 ok
1761792 1762304 ok
1762304 1762816 ok
1762816 1763328 ok
1763328 1763840 ok
1763840 1764352 ok
1764352 1764864 ok
1764864 1765376 ok
1765376 1765888 ok
1765888 1766400 ok
1766400 1766912 ok
1766912 1767424 ok
1767424 1767936 ok
1767936 1768448 ok
1768448 1768960 ok
1768960 1769472 ok
1769472 1769984 ok
1769984 1770496 ok
1770496 1771008 ok
1771008 1771520 ok
1771520 1772032 ok
1772032 1772544 ok
1772544 1773056 ok
1773056 1773568 ok
1773568 1774080 ok
1774080 1774592 ok
1774592 1775104 ok
1775104 1775616 ok
1775616 1776128 ok
1776128 1776640 ok
1776640 1777152 ok
1777152 1777664 ok
1777664 1778176 ok
1778176 1778688 ok
1778688 1779200 ok
1779200 1779712 ok
1779712 1780224 ok
1780224 1780736 ok
1780736 1781248 ok
1781248 1781760 ok
1781760 1782272 ok
1782272 1782784 ok
1782784 1783296 ok
1783296 1783808 ok
1783808 1784320 ok
1784320 1784832 ok
1784832 1785344 ok
1785344 1785856 ok
1785856 1786368 ok
1786368 1786880 ok
1786880 1787392 ok
1787392 1787904 ok
1787904 1788416 ok
1788416 1788928 ok
1788928 1789440 ok
1789440 1789952 ok
1789952 1790464 ok
1790464 1790976 ok
1790976 1791488 ok
1791488 1792000 ok
1792000 1792512 ok
1792512 1793024 ok
1793024 1793536 ok
1793536 1794048 ok
1794048 1794560 ok
1794560 1795072 ok
1795072 1795584 ok
1795584 1796096 ok
1796096 1796608 ok
1796608 1797120 ok
1797120 1797632 ok
1797632 1798144 ok
1798144 1798656 ok
1798656 1799168 ok
1799168 1799680 ok
1799680 1800192 ok
1800192 1800704 ok
1800704 1801216 ok
1801216 1801728 ok
1801728 1802240 ok
1802240 1802752 ok
1802752 1803264 ok
1803264 1803776 ok
1803776 1804288 ok
1804288 1804800 ok
1804800 1805312 ok
1805312 1805824 ok
1805824 1806336 ok
1806336 1806848 ok
1806848 1807360 ok
1807360 1807872 ok
1807872 1808384 ok
1808384 1808896 ok
1808896 1809408 ok
1809408 1809920 ok
1809920 1810432 ok
1810432 1810944 ok
1810944 1811456 ok
1811456 1811968 ok
1811968 1812480 ok
1812480 1812992 ok
1812992 1813504 ok
1813504 1814016 ok
1814016 1814528 ok
1814528 1815040 ok
1815040 1815552 ok
1815552 1816064 ok
1816064 1816576 ok
1816576 1817088 ok
1817088 1817600 ok
1817600 1818112 ok
1818112 1818624 ok
1818624 1819136 ok
1819136 1819648 ok
1819648 1820160 ok
1820160 1820672 ok
1820672 1821184 ok
1821184 1821696 ok
1821696 1822208 ok
1822208 1822720 ok
1822720 1823232 ok
1823232 1823744 ok
1823744 1824256 ok
1824256 1824768 ok
1824768 1825280 ok
1825280 1825792 ok
1825792 1826304 ok
1826304 1826816 ok
1826816 1827328 ok
1827328 1827840 ok
1827840 1828352 ok
1828352 1828864 ok
1828864 1829376 ok
1829376 1829888 ok
1829888 1830400 ok
1830400 1830912 ok
1830912 1831424 ok
1831424 1831936 ok
1831936 1832448 ok
1832448 1832960 ok
1832960 1833472 ok
1833472 1833984 ok
1833984 1834496 ok
1834496 1835008 ok
1835008 1835520 ok
1835520 1836032 ok
1836032 1836544 ok
1836544 1837056 ok
1837056 1837568 ok
1837568 1838080 ok
1838080 1838592 ok
1838592 1839104 ok
1839104 1839616 ok
1839616 1840128 ok
1840128 1840640 ok
1840640 1841152 ok
1841152 1841664 ok
1841664 1842176 ok
1842176 1842688 ok
1842688 1843200 ok
1843200 1843712 ok
1843712 1844224 ok
1844224 1844736 ok
1844736 1845248 ok
1845248 1845760 ok
1845760 1846272 ok
1846272 1846784 ok
1846784 1847296 ok
1847296 1847808 ok
1847808 1848320 ok
1848320 1848832 ok
1848832 1849344 ok
1849344 1849856 ok
1849856 1850368 ok
1850368 1850880 ok
1850880 1851392 ok
1851392 1851904 ok
1851904 1852416 ok
1852416 1852928 ok
1852928 1853440 ok
1853440 1853952 ok
1853952 1854464 ok
1854464 1854976 ok
1854976 1855488 ok
1855488 1856000 ok
1856000 1856512 ok
1856512 1857024 ok
1857024 1857536 ok
1857536 1858048 ok
1858048 1858560 ok
1858560 1859072 ok
1859072 1859584 ok
1859584 1860096 ok
1860096 1860608 ok
1860608 1861120 ok
1861120 1861632 ok
1861632 1862144 ok
1862144 1862656 ok
1862656 1863168 ok
1863168 1863680 ok
1863680 1864192 ok
1864192 1864704 ok
1864704 1865216 ok
1865216 1865728 ok
1865728 1866240 ok
1866240 1866752 ok
1866752 1867264 ok
1867264 1867776 ok
1867776 1868288 ok
1868288 1868800 ok
1868800 1869312 ok
1869312 1869824 ok
1869824 1870336 ok
1870336 1870848 ok
1870848 1871360 ok
1871360 1871872 ok
1871872 1872384 ok
1872384 1872896 ok
1872896 1873408 ok
1873408 1873920 ok
1873920 1874432 ok
1874432 1874944 ok
1874944 1875456 ok
1875456 1875968 ok
1875968 1876480 ok
1876480 1876992 ok
1876992 1877504 ok
1877504 1878016 ok
1878016 1878528 ok
1878528 1879040 ok
1879040 1879552 ok
1879552 1880064 ok
1880064 1880576 ok
1880576 1881088 ok
1881088 1881600 ok
1881600 1882112 ok
1882112 1882624 ok
1882624 1883136 ok
1883136 1883648 ok
1883648 1884160 ok
1884160 1884672 ok
1884672 1885184 ok
1885184 1885696 ok
1885696 1886208 ok
1886208 1886720 ok
1886720 1887232 ok
1887232 1887744 ok
1887744 1888256 ok
1888256 1888768 ok
1888768 1889280 ok
1889280 1889792 ok
1889792 1890304 ok
1890304 1890816 ok
1890816 1891328 ok
1891328 1891840 ok
1891840 1892352 ok
1892352 1892864 ok
1892864 1893376 ok
1893376 1893888 ok
1893888 1894400 ok
1894400 1894912 ok
1894912 1895424 ok
1895424 1895936 ok
1895936 1896448 ok
1896448 1896960 ok
1896960 1897472 ok
1897472 1897984 ok
1897984 1898496 ok
1898496 1899008 ok
1899008 1899520 ok
1899520 1900032 ok
1900032 1900544 ok
1900544 1901056 ok
1901056 1901568 ok
1901568 1902080 ok
1902080 1902592 ok
1902592 1903104 ok
1903104 1903616 ok
1903616 1904128 ok
1904128 1904640 ok
1904640 1905152 ok
1905152 1905664 ok
1905664 1906176 ok
1906176 1906688 ok
1906688 1907200 ok
1907200 1907712 ok
1907712 1908224 ok
1908224 1908736 ok
1908736 1909248 ok
1909248 1909760 ok
1909760 1910272 ok
1910272 1910784 ok
1910784 1911296 ok
1911296 1911808 ok
1911808 1912320 ok
1912320 1912832 ok
1912832 1913344 ok
1913344 1913856 ok
1913856 1914368 ok
1914368 1914880 ok
1914880 1915392 ok
1915392 1915904 ok
1915904 1916416 ok
1916416 1916928 ok
1916928 1917440 ok
1917440 1917952 ok
1917952 1918464 ok
1918464 1918976 ok
1918976 1919488 ok
1919488 1920000 ok
1920000 1920512 ok
1920512 1921024 ok
1921024 1921536 ok
1921536 1922048 ok
1922048 1922560 ok
1922560 1923072 ok
1923072 1923584 ok
1923584 1924096 ok
1924096 1924608 ok
1924608 1925120 ok
1925120 1925632 ok
1925632 1926144 ok
1926144 1926656 ok
1926656 1927168 ok
1927168 1927680 ok
1927680 1928192 ok
1928192 1928704 ok
1928704 1929216 ok
1929216 1929728 ok
1929728 1930240 ok
1930240 1930752 ok
1930752 1931264 ok
1931264 1931776 ok
1931776 1932288 ok
1932288 1932800 ok
1932800 1933312 ok
1933312 1933824 ok
1933824 1934336 ok
1934336 1934848 ok
1934848 1935360 ok
1935360 1935872 ok
1935872 1936384 ok
1936384 1936896 ok
1936896 1937408 ok
1937408 1937920 ok
1937920 1938432 ok
1938432 1938944 ok
1938944 1939456 ok
1939456 1939968 ok
1939968 1940480 ok
1940480 1940992 ok
1940992 1941504 ok
1941504 1942016 ok
1942016 1942528 ok
1942528 1943040 ok
1943040 1943552 ok
1943552 1944064 ok
1944064 1944576 ok
1944576 1945088 ok
1945088 1945600 ok
1945600 1946112 ok
1946112 1946624 ok
1946624 1947136 ok
1947136 1947648 ok
1947648 1948160 ok
1948160 1948672 ok
1948672 1949184 ok
1949184 1949696 ok
1949696 1950208 ok
1950208 1950720 ok
1950720 1951232 ok
1951232 1951744 ok
1951744 1952256 ok
1952256 1952768 ok
1952768 1953280 ok
1953280 1953792 ok
1953792 1954304 ok
1954304 1954816 ok
1954816 1955328 ok
1955328 1955840 ok
1955840 1956352 ok
1956352 1956864 ok
1956864 1957376 ok
1957376 1957888 ok
1957888 1958400 ok
1958400 1958912 ok
1958912 1959424 ok
1959424 1959936 ok
1959936 1960448 ok
1960448 1960960 ok
1960960 1961472 ok
1961472 1961984 ok
1961984 1962496 ok
1962496 1963008 ok
1963008 1963520 ok
1963520 1964032 ok
1964032 1964544 ok
1964544 1965056 ok
1965056 1965568 ok
1965568 1966080 ok
Sep 22 23:14:38.718 DEBG IO Write 1027 has deps [JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)]
Sep 22 23:14:38.718 DEBG up_ds_listen was notified
Sep 22 23:14:38.719 DEBG up_ds_listen process 1027
Sep 22 23:14:38.719 DEBG [A] ack job 1027:28, : downstairs
Sep 22 23:14:38.719 DEBG up_ds_listen checked 1 jobs, back to waiting
test test::test_pantry_import_from_url_ovmf ... ok
{"msg":"Scrub at offset 2560/3840 sp:2560","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:38.781102132Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:38.897 DEBG Write     :1001 deps:[] res:true
Sep 22 23:14:38.928 DEBG Write     :1001 deps:[] res:true
Sep 22 23:14:38.929 WARN returning error on flush!
Sep 22 23:14:38.929 DEBG Flush     :1002 extent_limit None deps:[JobId(1001), JobId(1000)] res:false f:1 g:1
Sep 22 23:14:38.960 DEBG Write     :1001 deps:[] res:true
Sep 22 23:14:38.961 WARN returning error on flush!
Sep 22 23:14:38.961 DEBG Flush     :1002 extent_limit None deps:[JobId(1001), JobId(1000)] res:false f:1 g:1
Sep 22 23:14:38.980 DEBG Flush     :1002 extent_limit None deps:[JobId(1001), JobId(1000)] res:true f:1 g:1
Sep 22 23:14:38.980 DEBG Flush     :1002 extent_limit None deps:[JobId(1001), JobId(1000)] res:true f:1 g:1
Sep 22 23:14:38.980 DEBG Flush     :1002 extent_limit None deps:[JobId(1001), JobId(1000)] res:true f:1 g:1
Sep 22 23:14:38.980 INFO [lossy] sleeping 1 second
Sep 22 23:14:38.980 DEBG IO Flush 1028 has deps [JobId(1027), JobId(1026)]
{"msg":"Scrub at offset 2816/3840 sp:2816","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:39.019012828Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:39.061 DEBG Flush     :1015 extent_limit None deps:[JobId(1014), JobId(1013)] res:true f:5 g:2
Sep 22 23:14:39.061 DEBG Flush     :1015 extent_limit None deps:[JobId(1014), JobId(1013)] res:true f:5 g:2
Sep 22 23:14:39.061 DEBG Flush     :1015 extent_limit None deps:[JobId(1014), JobId(1013)] res:true f:5 g:2
{"msg":"Scrub at offset 3072/3840 sp:3072","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:39.245840538Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:39.315 DEBG IO Write 1029 has deps [JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)]
Sep 22 23:14:39.315 DEBG up_ds_listen was notified
Sep 22 23:14:39.315 DEBG up_ds_listen process 1029
Sep 22 23:14:39.315 DEBG [A] ack job 1029:30, : downstairs
Sep 22 23:14:39.315 DEBG up_ds_listen checked 1 jobs, back to waiting
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:39.441876209Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/job/9582738c-4762-4d3d-9423-661cc66a6ac0/is_finished","method":"GET","req_id":"ac8f04b1-d23a-4521-ad73-8c1868967f89","remote_addr":"127.0.0.1:43906","local_addr":"127.0.0.1:49411","component":"dropshot","latency_us":294,"response_code":"200"}
{"msg":"Scrub at offset 3328/3840 sp:3328","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:39.472376854Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:39.562 DEBG Flush     :1018 extent_limit None deps:[JobId(1017), JobId(1016)] res:true f:6 g:2
Sep 22 23:14:39.562 DEBG Flush     :1018 extent_limit None deps:[JobId(1017), JobId(1016)] res:true f:6 g:2
Sep 22 23:14:39.562 DEBG Flush     :1018 extent_limit None deps:[JobId(1017), JobId(1016)] res:true f:6 g:2
Sep 22 23:14:39.652 DEBG IO Write 1030 has deps [JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)]
Sep 22 23:14:39.652 DEBG up_ds_listen was notified
Sep 22 23:14:39.652 DEBG up_ds_listen process 1030
Sep 22 23:14:39.653 DEBG [A] ack job 1030:31, : downstairs
Sep 22 23:14:39.653 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:39.653 DEBG IO Flush 1031 has deps [JobId(1030), JobId(1029), JobId(1028)]
{"msg":"Scrub at offset 3584/3840 sp:3584","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:39.698403309Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Scrub at offset 3840/3840 sp:3840","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:39.92521643Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"Scrub fc5b0c26-5d3b-433c-b382-b2a002415eea done in 3 seconds. Retries:0 scrub_size:131072 size:3840 pause_milli:0","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:39.926382126Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:39.929 DEBG Flush     :1021 extent_limit None deps:[JobId(1020), JobId(1019)] res:true f:7 g:2
Sep 22 23:14:39.929 DEBG Flush     :1021 extent_limit None deps:[JobId(1020), JobId(1019)] res:true f:7 g:2
Sep 22 23:14:39.929 DEBG Flush     :1021 extent_limit None deps:[JobId(1020), JobId(1019)] res:true f:7 g:2
Sep 22 23:14:39.989 DEBG IO Write 1032 has deps [JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)]
Sep 22 23:14:39.989 DEBG up_ds_listen was notified
Sep 22 23:14:39.989 DEBG up_ds_listen process 1032
Sep 22 23:14:39.989 DEBG [A] ack job 1032:33, : downstairs
Sep 22 23:14:39.989 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:39.990 INFO [lossy] sleeping 1 second
Sep 22 23:14:40.326 DEBG IO Write 1033 has deps [JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)]
Sep 22 23:14:40.326 DEBG up_ds_listen was notified
Sep 22 23:14:40.326 DEBG up_ds_listen process 1033
Sep 22 23:14:40.327 DEBG [A] ack job 1033:34, : downstairs
Sep 22 23:14:40.327 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:40.327 DEBG IO Flush 1034 has deps [JobId(1033), JobId(1032), JobId(1031)]
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.473676708Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/job/9582738c-4762-4d3d-9423-661cc66a6ac0/is_finished","method":"GET","req_id":"7c4006b9-5718-4f1e-b45c-a8eea8ded2bb","remote_addr":"127.0.0.1:43906","local_addr":"127.0.0.1:49411","component":"dropshot","latency_us":244,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.474383923Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/job/9582738c-4762-4d3d-9423-661cc66a6ac0/ok","method":"GET","req_id":"05db0b20-31e1-4c60-9e9c-a667babfa6a4","remote_addr":"127.0.0.1:43906","local_addr":"127.0.0.1:49411","component":"dropshot","latency_us":175,"response_code":"200"}
{"msg":"detach removing entry for volume fc5b0c26-5d3b-433c-b382-b2a002415eea","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.474965393Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"detaching volume fc5b0c26-5d3b-433c-b382-b2a002415eea","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.474997806Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
Sep 22 23:14:40.475 DEBG Flush     :1022 extent_limit None deps:[] res:true f:8 g:2
Sep 22 23:14:40.475 DEBG Flush     :1022 extent_limit None deps:[] res:true f:8 g:2
Sep 22 23:14:40.475 DEBG Flush     :1022 extent_limit None deps:[] res:true f:8 g:2
{"msg":"Request to deactivate this guest","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.476110438Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"521f2004-29de-4d41-9494-447a49569d29 set deactivating.","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.47614297Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.478288885Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"uri":"/crucible/pantry/0/volume/fc5b0c26-5d3b-433c-b382-b2a002415eea","method":"DELETE","req_id":"96e59cb2-2739-4ad0-9fb4-f89347e35f8b","remote_addr":"127.0.0.1:43906","local_addr":"127.0.0.1:49411","component":"dropshot","latency_us":3422,"response_code":"204"}
{"msg":"Upstairs starts","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.478678218Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"Crucible Version: BuildInfo {\n    version: \"0.0.1\",\n    git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n    git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n    git_branch: \"main\",\n    rustc_semver: \"1.70.0\",\n    rustc_channel: \"stable\",\n    rustc_host_triple: \"x86_64-unknown-illumos\",\n    rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n    cargo_triple: \"x86_64-unknown-illumos\",\n    debug: true,\n    opt_level: 0,\n}","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.478727374Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.478751039Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"Crucible stats registered with UUID: 521f2004-29de-4d41-9494-447a49569d29","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.478773985Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"Crucible 521f2004-29de-4d41-9494-447a49569d29 has session id: b89f643a-f10d-4978-b118-fac458329f63","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.47879705Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[0] connecting to 127.0.0.1:63639","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.478836532Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"0"}
{"msg":"[1] connecting to 127.0.0.1:43949","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.478975939Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"1"}
{"msg":"[2] connecting to 127.0.0.1:63175","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.479040981Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"2"}
{"msg":"up_listen starts","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.479084614Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"task":"up_listen"}
{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.479109415Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"Flush timeout: 0.5","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.479130321Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[0] 521f2004-29de-4d41-9494-447a49569d29 looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.479299903Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"0"}
{"msg":"[0] Proc runs for 127.0.0.1:63639 in state New","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.479330316Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[1] 521f2004-29de-4d41-9494-447a49569d29 looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.479354481Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"1"}
{"msg":"[1] Proc runs for 127.0.0.1:43949 in state New","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.479376841Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[2] 521f2004-29de-4d41-9494-447a49569d29 looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.47939858Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"2"}
{"msg":"[2] Proc runs for 127.0.0.1:63175 in state New","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.479426159Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
Sep 22 23:14:40.479 INFO accepted connection from 127.0.0.1:51284, task: main
Sep 22 23:14:40.479 INFO accepted connection from 127.0.0.1:54524, task: main
Sep 22 23:14:40.479 INFO accepted connection from 127.0.0.1:49615, task: main
Sep 22 23:14:40.479 INFO Connection request from 521f2004-29de-4d41-9494-447a49569d29 with version 4, task: proc
Sep 22 23:14:40.479 INFO upstairs UpstairsConnection { upstairs_id: 521f2004-29de-4d41-9494-447a49569d29, session_id: 92660512-15df-4806-a057-1d8e47122931, gen: 3 } connected, version 4, task: proc
Sep 22 23:14:40.479 INFO Connection request from 521f2004-29de-4d41-9494-447a49569d29 with version 4, task: proc
Sep 22 23:14:40.479 INFO upstairs UpstairsConnection { upstairs_id: 521f2004-29de-4d41-9494-447a49569d29, session_id: 92660512-15df-4806-a057-1d8e47122931, gen: 3 } connected, version 4, task: proc
Sep 22 23:14:40.479 INFO Connection request from 521f2004-29de-4d41-9494-447a49569d29 with version 4, task: proc
Sep 22 23:14:40.479 INFO upstairs UpstairsConnection { upstairs_id: 521f2004-29de-4d41-9494-447a49569d29, session_id: 92660512-15df-4806-a057-1d8e47122931, gen: 3 } connected, version 4, task: proc
{"msg":"[0] 521f2004-29de-4d41-9494-447a49569d29 (92660512-15df-4806-a057-1d8e47122931) New New New ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.480221977Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.480251098Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[1] 521f2004-29de-4d41-9494-447a49569d29 (92660512-15df-4806-a057-1d8e47122931) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.480275227Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.480296014Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[2] 521f2004-29de-4d41-9494-447a49569d29 (92660512-15df-4806-a057-1d8e47122931) WaitActive WaitActive New ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.480319724Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.48034061Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
The guest has requested activation
{"msg":"521f2004-29de-4d41-9494-447a49569d29 active request set","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.480451879Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[0] received activate with gen 3","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.480487972Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[0] client got ds_active_rx, promote! session 92660512-15df-4806-a057-1d8e47122931","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.480511479Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[1] received activate with gen 3","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.480532456Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[1] client got ds_active_rx, promote! session 92660512-15df-4806-a057-1d8e47122931","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.480554337Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[2] received activate with gen 3","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.480580366Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[2] client got ds_active_rx, promote! session 92660512-15df-4806-a057-1d8e47122931","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.48060313Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
Sep 22 23:14:40.480 WARN Attempting RW takeover from UpstairsConnection { upstairs_id: 521f2004-29de-4d41-9494-447a49569d29, session_id: 1e7e774e-f305-4cf5-a9d4-7b0b96e564fc, gen: 2 } to UpstairsConnection { upstairs_id: 521f2004-29de-4d41-9494-447a49569d29, session_id: 92660512-15df-4806-a057-1d8e47122931, gen: 3 }
Sep 22 23:14:40.480 WARN Signaling to UpstairsConnection { upstairs_id: 521f2004-29de-4d41-9494-447a49569d29, session_id: 1e7e774e-f305-4cf5-a9d4-7b0b96e564fc, gen: 2 } thread that UpstairsConnection { upstairs_id: 521f2004-29de-4d41-9494-447a49569d29, session_id: 92660512-15df-4806-a057-1d8e47122931, gen: 3 } is being promoted (read-write)
Sep 22 23:14:40.480 WARN Attempting RW takeover from UpstairsConnection { upstairs_id: 521f2004-29de-4d41-9494-447a49569d29, session_id: 1e7e774e-f305-4cf5-a9d4-7b0b96e564fc, gen: 2 } to UpstairsConnection { upstairs_id: 521f2004-29de-4d41-9494-447a49569d29, session_id: 92660512-15df-4806-a057-1d8e47122931, gen: 3 }
Sep 22 23:14:40.480 WARN Signaling to UpstairsConnection { upstairs_id: 521f2004-29de-4d41-9494-447a49569d29, session_id: 1e7e774e-f305-4cf5-a9d4-7b0b96e564fc, gen: 2 } thread that UpstairsConnection { upstairs_id: 521f2004-29de-4d41-9494-447a49569d29, session_id: 92660512-15df-4806-a057-1d8e47122931, gen: 3 } is being promoted (read-write)
Sep 22 23:14:40.481 WARN Attempting RW takeover from UpstairsConnection { upstairs_id: 521f2004-29de-4d41-9494-447a49569d29, session_id: 1e7e774e-f305-4cf5-a9d4-7b0b96e564fc, gen: 2 } to UpstairsConnection { upstairs_id: 521f2004-29de-4d41-9494-447a49569d29, session_id: 92660512-15df-4806-a057-1d8e47122931, gen: 3 }
Sep 22 23:14:40.481 WARN Signaling to UpstairsConnection { upstairs_id: 521f2004-29de-4d41-9494-447a49569d29, session_id: 1e7e774e-f305-4cf5-a9d4-7b0b96e564fc, gen: 2 } thread that UpstairsConnection { upstairs_id: 521f2004-29de-4d41-9494-447a49569d29, session_id: 92660512-15df-4806-a057-1d8e47122931, gen: 3 } is being promoted (read-write)
Sep 22 23:14:40.481 WARN Another upstairs UpstairsConnection { upstairs_id: 521f2004-29de-4d41-9494-447a49569d29, session_id: 92660512-15df-4806-a057-1d8e47122931, gen: 3 } promoted to active, shutting down connection for UpstairsConnection { upstairs_id: 521f2004-29de-4d41-9494-447a49569d29, session_id: 1e7e774e-f305-4cf5-a9d4-7b0b96e564fc, gen: 2 }, task: main
Sep 22 23:14:40.481 INFO UpstairsConnection { upstairs_id: 521f2004-29de-4d41-9494-447a49569d29, session_id: 92660512-15df-4806-a057-1d8e47122931, gen: 3 } is now active (read-write)
Sep 22 23:14:40.481 WARN Another upstairs UpstairsConnection { upstairs_id: 521f2004-29de-4d41-9494-447a49569d29, session_id: 92660512-15df-4806-a057-1d8e47122931, gen: 3 } promoted to active, shutting down connection for UpstairsConnection { upstairs_id: 521f2004-29de-4d41-9494-447a49569d29, session_id: 1e7e774e-f305-4cf5-a9d4-7b0b96e564fc, gen: 2 }, task: main
Sep 22 23:14:40.481 INFO UpstairsConnection { upstairs_id: 521f2004-29de-4d41-9494-447a49569d29, session_id: 92660512-15df-4806-a057-1d8e47122931, gen: 3 } is now active (read-write)
Sep 22 23:14:40.481 WARN Another upstairs UpstairsConnection { upstairs_id: 521f2004-29de-4d41-9494-447a49569d29, session_id: 92660512-15df-4806-a057-1d8e47122931, gen: 3 } promoted to active, shutting down connection for UpstairsConnection { upstairs_id: 521f2004-29de-4d41-9494-447a49569d29, session_id: 1e7e774e-f305-4cf5-a9d4-7b0b96e564fc, gen: 2 }, task: main
Sep 22 23:14:40.481 INFO UpstairsConnection { upstairs_id: 521f2004-29de-4d41-9494-447a49569d29, session_id: 92660512-15df-4806-a057-1d8e47122931, gen: 3 } is now active (read-write)
Sep 22 23:14:40.481 INFO connection (127.0.0.1:33714): all done
Sep 22 23:14:40.481 INFO connection (127.0.0.1:36045): all done
Sep 22 23:14:40.481 INFO connection (127.0.0.1:62745): all done
{"msg":"[0] 521f2004-29de-4d41-9494-447a49569d29 (1e7e774e-f305-4cf5-a9d4-7b0b96e564fc) cmd_loop saw YouAreNoLongerActive 521f2004-29de-4d41-9494-447a49569d29 92660512-15df-4806-a057-1d8e47122931 3","v":0,"name":"crucible-pantry","level":50,"time":"2023-09-22T23:14:40.482176202Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] 521f2004-29de-4d41-9494-447a49569d29 (1e7e774e-f305-4cf5-a9d4-7b0b96e564fc) Active Active Active ds_transition to Disabled","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.482209245Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] Transition from Active to Disabled","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.482232573Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"521f2004-29de-4d41-9494-447a49569d29 set inactive, session 1e7e774e-f305-4cf5-a9d4-7b0b96e564fc","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.482256331Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"127.0.0.1:63639: proc: Generation number is too low: saw YouAreNoLongerActive with larger gen 3 than ours 2","v":0,"name":"crucible-pantry","level":50,"time":"2023-09-22T23:14:40.48228049Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"0","component":"datafile"}
{"msg":"[0] 521f2004-29de-4d41-9494-447a49569d29 Gone missing, transition from Disabled to Disconnected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.482305675Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] 521f2004-29de-4d41-9494-447a49569d29 connection to 127.0.0.1:63639 closed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.482372565Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"0","component":"datafile"}
{"msg":"[1] 521f2004-29de-4d41-9494-447a49569d29 (1e7e774e-f305-4cf5-a9d4-7b0b96e564fc) cmd_loop saw YouAreNoLongerActive 521f2004-29de-4d41-9494-447a49569d29 92660512-15df-4806-a057-1d8e47122931 3","v":0,"name":"crucible-pantry","level":50,"time":"2023-09-22T23:14:40.482405009Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] 521f2004-29de-4d41-9494-447a49569d29 (1e7e774e-f305-4cf5-a9d4-7b0b96e564fc) Disconnected Active Active ds_transition to Disabled","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.482430577Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] Transition from Active to Disabled","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.482452893Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"521f2004-29de-4d41-9494-447a49569d29 set inactive, session 1e7e774e-f305-4cf5-a9d4-7b0b96e564fc","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.482476483Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"127.0.0.1:43949: proc: Generation number is too low: saw YouAreNoLongerActive with larger gen 3 than ours 2","v":0,"name":"crucible-pantry","level":50,"time":"2023-09-22T23:14:40.482500285Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"1","component":"datafile"}
{"msg":"[1] 521f2004-29de-4d41-9494-447a49569d29 Gone missing, transition from Disabled to Disconnected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.482525002Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] 521f2004-29de-4d41-9494-447a49569d29 connection to 127.0.0.1:43949 closed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.482547906Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"1","component":"datafile"}
{"msg":"[2] 521f2004-29de-4d41-9494-447a49569d29 (1e7e774e-f305-4cf5-a9d4-7b0b96e564fc) cmd_loop saw YouAreNoLongerActive 521f2004-29de-4d41-9494-447a49569d29 92660512-15df-4806-a057-1d8e47122931 3","v":0,"name":"crucible-pantry","level":50,"time":"2023-09-22T23:14:40.482580059Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] 521f2004-29de-4d41-9494-447a49569d29 (1e7e774e-f305-4cf5-a9d4-7b0b96e564fc) Disconnected Disconnected Active ds_transition to Disabled","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.482606046Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] Transition from Active to Disabled","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.482628508Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"521f2004-29de-4d41-9494-447a49569d29 set inactive, session 1e7e774e-f305-4cf5-a9d4-7b0b96e564fc","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.482652139Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"127.0.0.1:63175: proc: Generation number is too low: saw YouAreNoLongerActive with larger gen 3 than ours 2","v":0,"name":"crucible-pantry","level":50,"time":"2023-09-22T23:14:40.482675875Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"2","component":"datafile"}
{"msg":"[2] 521f2004-29de-4d41-9494-447a49569d29 Gone missing, transition from Disabled to Disconnected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.482700612Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] 521f2004-29de-4d41-9494-447a49569d29 connection to 127.0.0.1:63175 closed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.482723904Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"looper":"2","component":"datafile"}
{"msg":"[0] pm_task rx.recv() is None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:14:40.482746962Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] 127.0.0.1:63639 task reports connection:false","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.482769699Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"521f2004-29de-4d41-9494-447a49569d29 Disconnected Disconnected Disconnected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.482792613Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] 127.0.0.1:63639 task reports offline","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.482814566Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] 127.0.0.1:43949 task reports connection:false","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.482836746Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"521f2004-29de-4d41-9494-447a49569d29 Disconnected Disconnected Disconnected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.482859715Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] 127.0.0.1:43949 task reports offline","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.482881706Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] 127.0.0.1:63175 task reports connection:false","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.482903715Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"521f2004-29de-4d41-9494-447a49569d29 Disconnected Disconnected Disconnected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.482926551Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] 127.0.0.1:63175 task reports offline","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.482953107Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[1] pm_task rx.recv() is None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:14:40.482975504Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[2] pm_task rx.recv() is None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:14:40.482997524Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301,"component":"datafile"}
{"msg":"[0] downstairs client at 127.0.0.1:63639 has UUID bafba53d-f7d4-4132-95f6-92b6a45487a9","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.483020834Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 512, shift: 9 }, extent_count: 188, uuid: bafba53d-f7d4-4132-95f6-92b6a45487a9, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.483046727Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"521f2004-29de-4d41-9494-447a49569d29 WaitActive WaitActive WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.483068327Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[1] downstairs client at 127.0.0.1:43949 has UUID 1e7a075d-e51e-4361-bdf9-32f916f89647","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.483090299Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 512, shift: 9 }, extent_count: 188, uuid: 1e7a075d-e51e-4361-bdf9-32f916f89647, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.483115888Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"521f2004-29de-4d41-9494-447a49569d29 WaitActive WaitActive WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.483137417Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[2] downstairs client at 127.0.0.1:63175 has UUID b89ddae9-a405-4d4a-865d-e95f1fd0a715","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.48315933Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 512, shift: 9 }, extent_count: 188, uuid: b89ddae9-a405-4d4a-865d-e95f1fd0a715, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.483184707Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"521f2004-29de-4d41-9494-447a49569d29 WaitActive WaitActive WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.483206306Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
Sep 22 23:14:40.490 INFO Current flush_numbers [0..12]: [1, 2, 3, 4, 5, 6, 7, 7, 0, 0, 0, 0]
Sep 22 23:14:40.491 INFO Current flush_numbers [0..12]: [1, 2, 3, 4, 5, 6, 7, 7, 0, 0, 0, 0]
Sep 22 23:14:40.493 INFO Current flush_numbers [0..12]: [1, 2, 3, 4, 5, 6, 7, 7, 0, 0, 0, 0]
Sep 22 23:14:40.505 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:40.506 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:14:40.507 INFO Downstairs has completed Negotiation, task: proc
{"msg":"[0] 521f2004-29de-4d41-9494-447a49569d29 (92660512-15df-4806-a057-1d8e47122931) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.507690458Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.507728926Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:14:40.507755599Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.50777659Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[1] 521f2004-29de-4d41-9494-447a49569d29 (92660512-15df-4806-a057-1d8e47122931) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.507938257Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.507967207Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:14:40.5079887Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.508009315Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[2] 521f2004-29de-4d41-9494-447a49569d29 (92660512-15df-4806-a057-1d8e47122931) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.508174414Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.508203231Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:14:40.508224974Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.508245618Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[0] 127.0.0.1:63639 task reports connection:true","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.508266913Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"521f2004-29de-4d41-9494-447a49569d29 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.50828861Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[0]R flush_numbers[0..12]: [1, 2, 3, 4, 5, 6, 7, 7, 0, 0, 0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.50830982Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[0]R generation[0..12]: [2, 2, 2, 2, 2, 2, 2, 2, 0, 0, 0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.508331005Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.508353367Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[1]R flush_numbers[0..12]: [1, 2, 3, 4, 5, 6, 7, 7, 0, 0, 0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.50837491Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[1]R generation[0..12]: [2, 2, 2, 2, 2, 2, 2, 2, 0, 0, 0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.508396132Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.508418387Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[2]R flush_numbers[0..12]: [1, 2, 3, 4, 5, 6, 7, 7, 0, 0, 0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.508445324Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[2]R generation[0..12]: [2, 2, 2, 2, 2, 2, 2, 2, 0, 0, 0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.508467648Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.508490067Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"Max found gen is 3","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.508510006Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"Generation requested: 3 >= found:3","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.508530498Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"Next flush: 8","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.508550618Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"All extents match","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.508570881Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
The guest has finished waiting for activation
{"msg":"No downstairs repair required","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.508591165Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"No initial repair work was required","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.508611989Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.508632367Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"521f2004-29de-4d41-9494-447a49569d29 is now active with session: 92660512-15df-4806-a057-1d8e47122931","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.508654575Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"521f2004-29de-4d41-9494-447a49569d29 Set Active after no repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.508675645Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.508696448Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"Set check for repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.508716639Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[1] 127.0.0.1:43949 task reports connection:true","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.508737389Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"521f2004-29de-4d41-9494-447a49569d29 Active Active Active","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.508758254Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"Set check for repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.508778352Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[2] 127.0.0.1:63175 task reports connection:true","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.508799054Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"521f2004-29de-4d41-9494-447a49569d29 Active Active Active","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.50882002Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"Set check for repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.508844645Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[0] received reconcile message","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.508865862Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.508886071Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.508906007Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[1] received reconcile message","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.50892619Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.508946349Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.508966456Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[2] received reconcile message","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.50898672Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.509007017Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:14:40.50902722Z","hostname":"ip-10-150-1-74.us-west-2.compute.internal","pid":4301}
Sep 22 23:14:40.548 DEBG Read      :1000 deps:[] res:true
Sep 22 23:14:40.606 DEBG Read      :1000 deps:[] res:true
Sep 22 23:14:40.664 DEBG Read      :1000 deps:[] res:true
Sep 22 23:14:40.666 DEBG IO Write 1035 has deps [JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)]
Sep 22 23:14:40.666 DEBG up_ds_listen was notified
Sep 22 23:14:40.666 DEBG up_ds_listen process 1035
Sep 22 23:14:40.666 DEBG [A] ack job 1035:36, : downstairs
Sep 22 23:14:40.666 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:41.006 DEBG IO Write 1036 has deps [JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)]
Sep 22 23:14:41.006 DEBG up_ds_listen was notified
Sep 22 23:14:41.006 DEBG up_ds_listen process 1036
Sep 22 23:14:41.007 DEBG [A] ack job 1036:37, : downstairs
Sep 22 23:14:41.007 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:41.007 DEBG IO Flush 1037 has deps [JobId(1036), JobId(1035), JobId(1034)]
Sep 22 23:14:41.349 DEBG IO Write 1038 has deps [JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)]
Sep 22 23:14:41.349 DEBG up_ds_listen was notified
Sep 22 23:14:41.350 DEBG up_ds_listen process 1038
Sep 22 23:14:41.350 DEBG [A] ack job 1038:39, : downstairs
Sep 22 23:14:41.350 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:41.692 DEBG IO Write 1039 has deps [JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)]
Sep 22 23:14:41.692 DEBG up_ds_listen was notified
Sep 22 23:14:41.692 DEBG up_ds_listen process 1039
Sep 22 23:14:41.692 DEBG [A] ack job 1039:40, : downstairs
Sep 22 23:14:41.692 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:41.693 DEBG IO Flush 1040 has deps [JobId(1039), JobId(1038), JobId(1037)]
Sep 22 23:14:42.098 DEBG IO Write 1041 has deps [JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)]
Sep 22 23:14:42.099 DEBG up_ds_listen was notified
Sep 22 23:14:42.099 DEBG up_ds_listen process 1041
Sep 22 23:14:42.099 DEBG [A] ack job 1041:42, : downstairs
Sep 22 23:14:42.099 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:42.257 INFO [lossy] skipping 1003
Sep 22 23:14:42.288 DEBG Write     :1003 deps:[JobId(1002)] res:true
Sep 22 23:14:42.319 DEBG Write     :1003 deps:[JobId(1002)] res:true
Sep 22 23:14:42.350 DEBG Write     :1003 deps:[JobId(1002)] res:true
Sep 22 23:14:42.351 DEBG IO Flush 1042 has deps [JobId(1041), JobId(1040)]
Sep 22 23:14:42.685 DEBG IO Write 1043 has deps [JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)]
Sep 22 23:14:42.686 DEBG up_ds_listen was notified
Sep 22 23:14:42.686 DEBG up_ds_listen process 1043
Sep 22 23:14:42.686 DEBG [A] ack job 1043:44, : downstairs
Sep 22 23:14:42.686 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:43.023 DEBG IO Write 1044 has deps [JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)]
Sep 22 23:14:43.024 DEBG up_ds_listen was notified
Sep 22 23:14:43.024 DEBG up_ds_listen process 1044
Sep 22 23:14:43.024 DEBG [A] ack job 1044:45, : downstairs
Sep 22 23:14:43.024 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:43.024 DEBG IO Flush 1045 has deps [JobId(1044), JobId(1043), JobId(1042)]
test test::test_pantry_scrub ... ok
Sep 22 23:14:43.360 DEBG IO Write 1046 has deps [JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)]
Sep 22 23:14:43.360 DEBG up_ds_listen was notified
Sep 22 23:14:43.361 DEBG up_ds_listen process 1046
Sep 22 23:14:43.361 DEBG [A] ack job 1046:47, : downstairs
Sep 22 23:14:43.361 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:43.695 DEBG IO Write 1047 has deps [JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)]
Sep 22 23:14:43.696 DEBG up_ds_listen was notified
Sep 22 23:14:43.696 DEBG up_ds_listen process 1047
Sep 22 23:14:43.696 DEBG [A] ack job 1047:48, : downstairs
Sep 22 23:14:43.696 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:43.696 DEBG IO Flush 1048 has deps [JobId(1047), JobId(1046), JobId(1045)]
Sep 22 23:14:44.031 DEBG IO Write 1049 has deps [JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)]
Sep 22 23:14:44.031 DEBG up_ds_listen was notified
Sep 22 23:14:44.031 DEBG up_ds_listen process 1049
Sep 22 23:14:44.031 DEBG [A] ack job 1049:50, : downstairs
Sep 22 23:14:44.031 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:44.368 DEBG IO Write 1050 has deps [JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)]
Sep 22 23:14:44.368 DEBG up_ds_listen was notified
Sep 22 23:14:44.368 DEBG up_ds_listen process 1050
Sep 22 23:14:44.368 DEBG [A] ack job 1050:51, : downstairs
Sep 22 23:14:44.368 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:44.369 DEBG IO Flush 1051 has deps [JobId(1050), JobId(1049), JobId(1048)]
Sep 22 23:14:44.705 DEBG IO Write 1052 has deps [JobId(1051), JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)]
Sep 22 23:14:44.705 DEBG up_ds_listen was notified
Sep 22 23:14:44.705 DEBG up_ds_listen process 1052
Sep 22 23:14:44.705 DEBG [A] ack job 1052:53, : downstairs
Sep 22 23:14:44.705 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:45.042 DEBG IO Write 1053 has deps [JobId(1051), JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)]
Sep 22 23:14:45.042 DEBG up_ds_listen was notified
Sep 22 23:14:45.042 DEBG up_ds_listen process 1053
Sep 22 23:14:45.043 DEBG [A] ack job 1053:54, : downstairs
Sep 22 23:14:45.043 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:45.043 DEBG IO Flush 1054 has deps [JobId(1053), JobId(1052), JobId(1051)]
Sep 22 23:14:45.043 INFO [lossy] sleeping 1 second
Sep 22 23:14:45.043 INFO [lossy] sleeping 1 second
Sep 22 23:14:45.043 INFO [lossy] sleeping 1 second
Sep 22 23:14:45.380 DEBG IO Write 1055 has deps [JobId(1054), JobId(1051), JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)]
Sep 22 23:14:45.380 DEBG up_ds_listen was notified
Sep 22 23:14:45.380 DEBG up_ds_listen process 1055
Sep 22 23:14:45.380 DEBG [A] ack job 1055:56, : downstairs
Sep 22 23:14:45.380 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:45.779 DEBG IO Write 1056 has deps [JobId(1054), JobId(1051), JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)]
Sep 22 23:14:45.779 DEBG up_ds_listen was notified
Sep 22 23:14:45.779 DEBG up_ds_listen process 1056
Sep 22 23:14:45.780 DEBG [A] ack job 1056:57, : downstairs
Sep 22 23:14:45.780 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:45.937 DEBG IO Flush 1057 has deps [JobId(1056), JobId(1055), JobId(1054)]
Sep 22 23:14:46.275 DEBG IO Write 1058 has deps [JobId(1057), JobId(1054), JobId(1051), JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)]
Sep 22 23:14:46.275 DEBG up_ds_listen was notified
Sep 22 23:14:46.275 DEBG up_ds_listen process 1058
Sep 22 23:14:46.275 DEBG [A] ack job 1058:59, : downstairs
Sep 22 23:14:46.275 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:46.308 DEBG Write     :1004 deps:[JobId(1002)] res:true
Sep 22 23:14:46.339 DEBG Write     :1004 deps:[JobId(1002)] res:true
Sep 22 23:14:46.340 WARN returning error on flush!
Sep 22 23:14:46.340 DEBG Flush     :1005 extent_limit None deps:[JobId(1004), JobId(1003), JobId(1002)] res:false f:2 g:1
Sep 22 23:14:46.340 WARN returning error on flush!
Sep 22 23:14:46.340 DEBG Flush     :1005 extent_limit None deps:[JobId(1004), JobId(1003), JobId(1002)] res:false f:2 g:1
Sep 22 23:14:46.340 INFO [lossy] skipping 1005
Sep 22 23:14:46.370 DEBG Write     :1004 deps:[JobId(1002)] res:true
Sep 22 23:14:46.391 DEBG Flush     :1005 extent_limit None deps:[JobId(1004), JobId(1003), JobId(1002)] res:true f:2 g:1
Sep 22 23:14:46.391 DEBG Flush     :1005 extent_limit None deps:[JobId(1004), JobId(1003), JobId(1002)] res:true f:2 g:1
Sep 22 23:14:46.391 DEBG Flush     :1005 extent_limit None deps:[JobId(1004), JobId(1003), JobId(1002)] res:true f:2 g:1
Sep 22 23:14:46.724 DEBG IO Write 1059 has deps [JobId(1057), JobId(1054), JobId(1051), JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)]
Sep 22 23:14:46.725 DEBG up_ds_listen was notified
Sep 22 23:14:46.725 DEBG up_ds_listen process 1059
Sep 22 23:14:46.725 DEBG [A] ack job 1059:60, : downstairs
Sep 22 23:14:46.725 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:46.725 DEBG IO Flush 1060 has deps [JobId(1059), JobId(1058), JobId(1057)]
Sep 22 23:14:47.063 DEBG IO Write 1061 has deps [JobId(1060), JobId(1057), JobId(1054), JobId(1051), JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)]
Sep 22 23:14:47.063 DEBG up_ds_listen was notified
Sep 22 23:14:47.063 DEBG up_ds_listen process 1061
Sep 22 23:14:47.063 DEBG [A] ack job 1061:62, : downstairs
Sep 22 23:14:47.063 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:47.401 DEBG IO Write 1062 has deps [JobId(1060), JobId(1057), JobId(1054), JobId(1051), JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)]
Sep 22 23:14:47.401 DEBG up_ds_listen was notified
Sep 22 23:14:47.401 DEBG up_ds_listen process 1062
Sep 22 23:14:47.401 DEBG [A] ack job 1062:63, : downstairs
Sep 22 23:14:47.401 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:47.402 DEBG IO Flush 1063 has deps [JobId(1062), JobId(1061), JobId(1060)]
Sep 22 23:14:47.743 DEBG IO Write 1064 has deps [JobId(1063), JobId(1060), JobId(1057), JobId(1054), JobId(1051), JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)]
Sep 22 23:14:47.743 DEBG up_ds_listen was notified
Sep 22 23:14:47.743 DEBG up_ds_listen process 1064
Sep 22 23:14:47.743 DEBG [A] ack job 1064:65, : downstairs
Sep 22 23:14:47.743 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:47.817 DEBG IO Write 1000 has deps []
Sep 22 23:14:47.817 INFO Checking if live repair is needed
Sep 22 23:14:47.817 INFO No Live Repair required at this time
Sep 22 23:14:47.817 DEBG up_ds_listen was notified
Sep 22 23:14:47.817 DEBG up_ds_listen process 1000
Sep 22 23:14:47.821 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:14:47.821 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:48.087 DEBG IO Write 1065 has deps [JobId(1063), JobId(1060), JobId(1057), JobId(1054), JobId(1051), JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)]
Sep 22 23:14:48.087 DEBG up_ds_listen was notified
Sep 22 23:14:48.087 DEBG up_ds_listen process 1065
Sep 22 23:14:48.087 DEBG [A] ack job 1065:66, : downstairs
Sep 22 23:14:48.087 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:48.088 DEBG IO Flush 1066 has deps [JobId(1065), JobId(1064), JobId(1063)]
Sep 22 23:14:48.429 DEBG IO Write 1067 has deps [JobId(1066), JobId(1063), JobId(1060), JobId(1057), JobId(1054), JobId(1051), JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)]
Sep 22 23:14:48.429 DEBG up_ds_listen was notified
Sep 22 23:14:48.429 DEBG up_ds_listen process 1067
Sep 22 23:14:48.429 DEBG [A] ack job 1067:68, : downstairs
Sep 22 23:14:48.429 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:48.769 DEBG IO Write 1068 has deps [JobId(1066), JobId(1063), JobId(1060), JobId(1057), JobId(1054), JobId(1051), JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)]
Sep 22 23:14:48.769 DEBG up_ds_listen was notified
Sep 22 23:14:48.769 DEBG up_ds_listen process 1068
Sep 22 23:14:48.769 DEBG [A] ack job 1068:69, : downstairs
Sep 22 23:14:48.769 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:48.770 DEBG IO Flush 1069 has deps [JobId(1068), JobId(1067), JobId(1066)]
Sep 22 23:14:49.171 DEBG IO Write 1070 has deps [JobId(1069), JobId(1066), JobId(1063), JobId(1060), JobId(1057), JobId(1054), JobId(1051), JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)]
Sep 22 23:14:49.171 DEBG up_ds_listen was notified
Sep 22 23:14:49.171 DEBG up_ds_listen process 1070
Sep 22 23:14:49.171 DEBG [A] ack job 1070:71, : downstairs
Sep 22 23:14:49.171 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:14:49.356 DEBG Write     :1006 deps:[JobId(1005), JobId(1002)] res:true
Sep 22 23:14:49.358 WARN returning error on write!
Sep 22 23:14:49.358 DEBG Write     :1006 deps:[JobId(1005), JobId(1002)] res:false
Sep 22 23:14:49.358 INFO [lossy] skipping 1006
Sep 22 23:14:49.359 WARN returning error on write!
Sep 22 23:14:49.359 DEBG Write     :1006 deps:[JobId(1005), JobId(1002)] res:false
Sep 22 23:14:49.389 DEBG Write     :1006 deps:[JobId(1005), JobId(1002)] res:true
Sep 22 23:14:49.390 INFO [lossy] sleeping 1 second
Sep 22 23:14:49.390 DEBG IO Flush 1071 has deps [JobId(1070), JobId(1069)]
Sep 22 23:14:49.391 DEBG IO Read  1072 has deps [JobId(1071)]
Sep 22 23:14:49.601 INFO [lossy] sleeping 1 second
Sep 22 23:14:49.632 DEBG Write     :1007 deps:[JobId(1005), JobId(1002)] res:true
Sep 22 23:14:49.633 INFO [lossy] skipping 1008
Sep 22 23:14:49.633 INFO [lossy] skipping 1008
Sep 22 23:14:49.640 DEBG Flush     :1008 extent_limit None deps:[JobId(1007), JobId(1006), JobId(1005)] res:true f:3 g:1
Sep 22 23:14:49.857 INFO [lossy] skipping 1009
Sep 22 23:14:49.888 DEBG Write     :1009 deps:[JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:49.891 DEBG IO Flush 1073 has deps [JobId(1072), JobId(1071)]
Sep 22 23:14:50.107 INFO [lossy] sleeping 1 second
Sep 22 23:14:50.174 ERRO [0] job id 1000 saw error GenericError("test error")
Sep 22 23:14:50.174 ERRO [0] job id 1002 saw error GenericError("test error")
Sep 22 23:14:50.174 ERRO [2] job id 1002 saw error GenericError("test error")
Sep 22 23:14:50.174 ERRO [2] job id 1005 saw error GenericError("test error")
Sep 22 23:14:50.174 ERRO [2] job id 1005 saw error GenericError("test error")
Sep 22 23:14:50.174 ERRO [2] job id 1006 saw error GenericError("test error")
Sep 22 23:14:50.174 ERRO [2] job id 1006 saw error GenericError("test error")
Sep 22 23:14:50.174 DEBG up_ds_listen was notified
Sep 22 23:14:50.174 DEBG up_ds_listen process 1002
Sep 22 23:14:50.174 DEBG [A] ack job 1002:3, : downstairs
Sep 22 23:14:50.177 DEBG [rc] retire 1002 clears [JobId(1000), JobId(1001), JobId(1002)], : downstairs
Sep 22 23:14:50.177 DEBG up_ds_listen process 1005
Sep 22 23:14:50.177 DEBG [A] ack job 1005:6, : downstairs
Sep 22 23:14:50.181 DEBG [rc] retire 1005 clears [JobId(1003), JobId(1004), JobId(1005)], : downstairs
Sep 22 23:14:50.181 DEBG up_ds_listen checked 2 jobs, back to waiting
Sep 22 23:14:50.181 DEBG up_ds_listen was notified
Sep 22 23:14:50.181 DEBG up_ds_listen checked 0 jobs, back to waiting
Sep 22 23:14:50.575 DEBG Write     :1006 deps:[JobId(1005), JobId(1002)] res:true
Sep 22 23:14:50.576 INFO [lossy] skipping 1007
Sep 22 23:14:50.576 INFO [lossy] skipping 1008
Sep 22 23:14:50.576 INFO [lossy] skipping 1009
Sep 22 23:14:50.576 INFO [lossy] skipping 1010
Sep 22 23:14:50.577 WARN returning error on write!
Sep 22 23:14:50.577 DEBG Write     :1007 deps:[JobId(1005), JobId(1002)] res:false
Sep 22 23:14:50.577 INFO [lossy] skipping 1008
Sep 22 23:14:50.577 INFO [lossy] skipping 1009
Sep 22 23:14:50.578 WARN returning error on write!
Sep 22 23:14:50.578 DEBG Write     :1007 deps:[JobId(1005), JobId(1002)] res:false
Sep 22 23:14:50.578 INFO [lossy] skipping 1007
Sep 22 23:14:50.608 DEBG Write     :1007 deps:[JobId(1005), JobId(1002)] res:true
Sep 22 23:14:50.609 INFO [lossy] skipping 1008
Sep 22 23:14:50.609 INFO [lossy] skipping 1009
Sep 22 23:14:50.609 INFO [lossy] skipping 1011
Sep 22 23:14:50.609 INFO [lossy] skipping 1014
Sep 22 23:14:50.609 INFO [lossy] skipping 1008
Sep 22 23:14:50.609 INFO [lossy] skipping 1008
Sep 22 23:14:50.609 INFO [lossy] skipping 1008
Sep 22 23:14:50.615 DEBG Flush     :1008 extent_limit None deps:[JobId(1007), JobId(1006), JobId(1005)] res:true f:3 g:1
Sep 22 23:14:50.616 WARN returning error on write!
Sep 22 23:14:50.616 DEBG Write     :1009 deps:[JobId(1008), JobId(1005), JobId(1002)] res:false
Sep 22 23:14:50.617 INFO [lossy] skipping 1010
Sep 22 23:14:50.617 WARN 1011 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:14:50.617 WARN 1012 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:14:50.617 WARN 1013 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:14:50.647 DEBG Write     :1009 deps:[JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:50.678 DEBG Write     :1010 deps:[JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:50.679 INFO [lossy] skipping 1011
Sep 22 23:14:50.686 DEBG Flush     :1011 extent_limit None deps:[JobId(1010), JobId(1009), JobId(1008)] res:true f:4 g:1
Sep 22 23:14:50.716 DEBG Write     :1012 deps:[JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:50.717 INFO [lossy] skipping 1013
Sep 22 23:14:50.717 WARN 1014 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:14:50.717 INFO [lossy] skipping 1013
Sep 22 23:14:50.748 DEBG Write     :1013 deps:[JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:50.755 DEBG Flush     :1014 extent_limit None deps:[JobId(1013), JobId(1012), JobId(1011)] res:true f:5 g:1
Sep 22 23:14:50.756 INFO [lossy] sleeping 1 second
Sep 22 23:14:50.786 DEBG Write     :1007 deps:[JobId(1005), JobId(1002)] res:true
Sep 22 23:14:50.787 INFO [lossy] skipping 1008
Sep 22 23:14:50.787 INFO [lossy] skipping 1009
Sep 22 23:14:50.787 WARN returning error on flush!
Sep 22 23:14:50.787 DEBG Flush     :1008 extent_limit None deps:[JobId(1007), JobId(1006), JobId(1005)] res:false f:3 g:1
Sep 22 23:14:50.787 INFO [lossy] skipping 1008
Sep 22 23:14:50.788 WARN returning error on flush!
Sep 22 23:14:50.788 DEBG Flush     :1008 extent_limit None deps:[JobId(1007), JobId(1006), JobId(1005)] res:false f:3 g:1
Sep 22 23:14:50.788 INFO [lossy] skipping 1008
Sep 22 23:14:50.788 INFO [lossy] skipping 1008
Sep 22 23:14:50.788 INFO [lossy] skipping 1008
Sep 22 23:14:50.794 DEBG Flush     :1008 extent_limit None deps:[JobId(1007), JobId(1006), JobId(1005)] res:true f:3 g:1
Sep 22 23:14:50.794 INFO [lossy] sleeping 1 second
Sep 22 23:14:51.163 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:51.163 INFO Created new region file "/tmp/downstairs-vrx8aK6L/region.json"
Sep 22 23:14:51.241 DEBG Write     :1010 deps:[JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:51.249 DEBG Flush     :1011 extent_limit None deps:[JobId(1010), JobId(1009), JobId(1008)] res:true f:4 g:1
Sep 22 23:14:51.249 INFO [lossy] skipping 1012
Sep 22 23:14:51.279 DEBG Write     :1013 deps:[JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:51.280 INFO [lossy] skipping 1014
Sep 22 23:14:51.280 INFO [lossy] skipping 1015
Sep 22 23:14:51.281 WARN returning error on write!
Sep 22 23:14:51.281 DEBG Write     :1012 deps:[JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:false
Sep 22 23:14:51.282 INFO [lossy] skipping 1015
Sep 22 23:14:51.282 WARN returning error on write!
Sep 22 23:14:51.282 DEBG Write     :1012 deps:[JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:false
Sep 22 23:14:51.282 INFO [lossy] skipping 1015
Sep 22 23:14:51.313 DEBG Write     :1012 deps:[JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:51.314 INFO [lossy] skipping 1015
Sep 22 23:14:51.314 WARN returning error on flush!
Sep 22 23:14:51.314 DEBG Flush     :1014 extent_limit None deps:[JobId(1013), JobId(1012), JobId(1011)] res:false f:5 g:1
Sep 22 23:14:51.321 DEBG Flush     :1014 extent_limit None deps:[JobId(1013), JobId(1012), JobId(1011)] res:true f:5 g:1
Sep 22 23:14:51.321 INFO [lossy] sleeping 1 second
Sep 22 23:14:51.473 DEBG IO Flush 1001 has deps [JobId(1000)]
Sep 22 23:14:51.473 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:14:51.473 INFO Opened existing region file "/tmp/downstairs-vrx8aK6L/region.json"
Sep 22 23:14:51.473 INFO Database read version 1
Sep 22 23:14:51.473 INFO Database write version 1
Sep 22 23:14:51.525 INFO UUID: 28f50308-941f-4d7f-bd0a-39a00206f2e7
Sep 22 23:14:51.525 INFO Blocks per extent:512 Total Extents: 188
Sep 22 23:14:51.525 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:14:51.525 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:14:51.525 INFO Using address: 127.0.0.1:50216, task: main
Sep 22 23:14:51.525 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:14:51.526 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:38873, task: repair
Sep 22 23:14:51.526 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:38873, task: repair
Sep 22 23:14:51.526 INFO listening, local_addr: 127.0.0.1:38873, task: repair
Sep 22 23:14:51.526 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:38873, task: repair
Sep 22 23:14:51.526 INFO Using repair address: 127.0.0.1:38873, task: main
Sep 22 23:14:51.526 INFO No SSL acceptor configured, task: main
Sep 22 23:14:51.534 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:14:51.534 WARN 6e5c0f70-fd56-4280-9d20-71288e488216 request to replace downstairs 127.0.0.1:42762 with 127.0.0.1:50216
Sep 22 23:14:51.534 INFO 6e5c0f70-fd56-4280-9d20-71288e488216 found old target: 127.0.0.1:42762 at 0
Sep 22 23:14:51.534 INFO 6e5c0f70-fd56-4280-9d20-71288e488216 replacing old: 127.0.0.1:42762 at 0
Sep 22 23:14:51.534 INFO [0] client skip 2 in process jobs because fault, : downstairs
Sep 22 23:14:51.534 INFO [0] changed 2 jobs to fault skipped, : downstairs
Sep 22 23:14:51.534 INFO [0] 6e5c0f70-fd56-4280-9d20-71288e488216 (f334c95e-b851-4a8a-a731-3fb69e42e934) Active Active Active ds_transition to Replacing
Sep 22 23:14:51.534 INFO [0] Transition from Active to Replacing
Sep 22 23:14:51.788 DEBG Write     :1015 deps:[JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:51.790 WARN returning error on write!
Sep 22 23:14:51.790 DEBG Write     :1016 deps:[JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:false
Sep 22 23:14:51.790 INFO [lossy] skipping 1019
Sep 22 23:14:51.820 DEBG Write     :1016 deps:[JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:51.820 INFO [lossy] skipping 1019
Sep 22 23:14:51.821 INFO [lossy] skipping 1019
Sep 22 23:14:51.821 WARN returning error on flush!
Sep 22 23:14:51.821 DEBG Flush     :1017 extent_limit None deps:[JobId(1016), JobId(1015), JobId(1014)] res:false f:6 g:1
Sep 22 23:14:51.821 INFO [lossy] skipping 1018
Sep 22 23:14:51.827 DEBG Flush     :1017 extent_limit None deps:[JobId(1016), JobId(1015), JobId(1014)] res:true f:6 g:1
Sep 22 23:14:51.827 INFO [lossy] skipping 1018
Sep 22 23:14:51.857 DEBG Write     :1018 deps:[JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:51.859 WARN returning error on write!
Sep 22 23:14:51.859 DEBG Write     :1019 deps:[JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:false
Sep 22 23:14:51.859 INFO [lossy] skipping 1020
Sep 22 23:14:51.889 DEBG Write     :1019 deps:[JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:51.890 INFO [lossy] skipping 1020
Sep 22 23:14:51.890 WARN returning error on flush!
Sep 22 23:14:51.890 DEBG Flush     :1020 extent_limit None deps:[JobId(1019), JobId(1018), JobId(1017)] res:false f:7 g:1
Sep 22 23:14:51.890 INFO [lossy] skipping 1020
Sep 22 23:14:51.896 DEBG Flush     :1020 extent_limit None deps:[JobId(1019), JobId(1018), JobId(1017)] res:true f:7 g:1
Sep 22 23:14:51.927 DEBG Write     :1009 deps:[JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:51.928 INFO [lossy] skipping 1010
Sep 22 23:14:51.928 INFO [lossy] skipping 1011
Sep 22 23:14:51.928 WARN 1012 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:14:51.928 INFO [lossy] skipping 1013
Sep 22 23:14:51.928 INFO [lossy] skipping 1017
Sep 22 23:14:51.928 INFO [lossy] skipping 1019
Sep 22 23:14:51.928 INFO [lossy] skipping 1010
Sep 22 23:14:51.928 WARN 1011 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:14:51.928 INFO [lossy] skipping 1013
Sep 22 23:14:51.928 INFO [lossy] skipping 1017
Sep 22 23:14:51.929 WARN returning error on write!
Sep 22 23:14:51.929 DEBG Write     :1010 deps:[JobId(1008), JobId(1005), JobId(1002)] res:false
Sep 22 23:14:51.929 WARN 1013 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:14:51.930 INFO [lossy] skipping 1017
Sep 22 23:14:51.959 DEBG Write     :1010 deps:[JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:51.967 DEBG Flush     :1011 extent_limit None deps:[JobId(1010), JobId(1009), JobId(1008)] res:true f:4 g:1
Sep 22 23:14:51.968 WARN returning error on write!
Sep 22 23:14:51.968 DEBG Write     :1012 deps:[JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:false
Sep 22 23:14:51.999 DEBG Write     :1013 deps:[JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:52.001 WARN 1014 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:14:52.001 INFO [lossy] skipping 1015
Sep 22 23:14:52.001 WARN 1016 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:14:52.001 WARN 1018 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:14:52.001 WARN 1019 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:14:52.001 INFO [lossy] skipping 1020
Sep 22 23:14:52.001 INFO [lossy] skipping 1012
Sep 22 23:14:52.001 WARN 1015 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:14:52.031 DEBG Write     :1012 deps:[JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:52.033 WARN returning error on flush!
Sep 22 23:14:52.033 DEBG Flush     :1014 extent_limit None deps:[JobId(1013), JobId(1012), JobId(1011)] res:false f:5 g:1
Sep 22 23:14:52.033 INFO [lossy] skipping 1014
Sep 22 23:14:52.033 INFO [lossy] skipping 1014
Sep 22 23:14:52.039 DEBG Flush     :1014 extent_limit None deps:[JobId(1013), JobId(1012), JobId(1011)] res:true f:5 g:1
Sep 22 23:14:52.070 DEBG Write     :1015 deps:[JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:52.101 DEBG Write     :1016 deps:[JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:52.108 DEBG Flush     :1017 extent_limit None deps:[JobId(1016), JobId(1015), JobId(1014)] res:true f:6 g:1
Sep 22 23:14:52.108 INFO [lossy] skipping 1018
Sep 22 23:14:52.138 DEBG Write     :1019 deps:[JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:52.139 INFO [lossy] skipping 1020
Sep 22 23:14:52.170 DEBG Write     :1018 deps:[JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:52.171 WARN returning error on flush!
Sep 22 23:14:52.171 DEBG Flush     :1020 extent_limit None deps:[JobId(1019), JobId(1018), JobId(1017)] res:false f:7 g:1
Sep 22 23:14:52.177 DEBG Flush     :1020 extent_limit None deps:[JobId(1019), JobId(1018), JobId(1017)] res:true f:7 g:1
Sep 22 23:14:52.177 INFO [lossy] sleeping 1 second
Sep 22 23:14:52.424 DEBG Write     :1021 deps:[JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:52.455 DEBG Write     :1015 deps:[JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:52.456 INFO [lossy] skipping 1016
Sep 22 23:14:52.456 WARN 1017 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:14:52.457 WARN returning error on write!
Sep 22 23:14:52.457 DEBG Write     :1016 deps:[JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:false
Sep 22 23:14:52.458 INFO [lossy] skipping 1016
Sep 22 23:14:52.458 WARN returning error on write!
Sep 22 23:14:52.458 DEBG Write     :1016 deps:[JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:false
Sep 22 23:14:52.488 DEBG Write     :1016 deps:[JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:52.489 INFO [lossy] sleeping 1 second
Sep 22 23:14:52.739 DEBG Write     :1022 deps:[JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:52.740 WARN returning error on flush!
Sep 22 23:14:52.740 DEBG Flush     :1023 extent_limit None deps:[JobId(1022), JobId(1021), JobId(1020)] res:false f:8 g:1
Sep 22 23:14:52.740 WARN returning error on flush!
Sep 22 23:14:52.740 DEBG Flush     :1023 extent_limit None deps:[JobId(1022), JobId(1021), JobId(1020)] res:false f:8 g:1
Sep 22 23:14:52.740 INFO [lossy] skipping 1023
Sep 22 23:14:52.740 INFO [lossy] skipping 1023
Sep 22 23:14:52.747 DEBG Flush     :1023 extent_limit None deps:[JobId(1022), JobId(1021), JobId(1020)] res:true f:8 g:1
Sep 22 23:14:52.963 INFO [lossy] skipping 1024
Sep 22 23:14:52.994 DEBG Write     :1024 deps:[JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:53.225 INFO [lossy] sleeping 1 second
Sep 22 23:14:53.225 INFO [lossy] skipping 1021
Sep 22 23:14:53.227 WARN returning error on write!
Sep 22 23:14:53.227 DEBG Write     :1022 deps:[JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:false
Sep 22 23:14:53.227 INFO [lossy] skipping 1025
Sep 22 23:14:53.258 DEBG Write     :1021 deps:[JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:53.289 DEBG Write     :1022 deps:[JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:53.289 INFO [lossy] sleeping 1 second
Sep 22 23:14:53.531 DEBG Flush     :1017 extent_limit None deps:[JobId(1016), JobId(1015), JobId(1014)] res:true f:6 g:1
Sep 22 23:14:53.561 DEBG Write     :1018 deps:[JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:53.562 INFO [lossy] skipping 1019
Sep 22 23:14:53.562 INFO [lossy] skipping 1020
Sep 22 23:14:53.562 WARN 1021 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:14:53.562 INFO [lossy] skipping 1022
Sep 22 23:14:53.562 INFO [lossy] skipping 1025
Sep 22 23:14:53.562 INFO [lossy] skipping 1028
Sep 22 23:14:53.592 DEBG Write     :1019 deps:[JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:53.593 INFO [lossy] skipping 1020
Sep 22 23:14:53.599 DEBG Flush     :1020 extent_limit None deps:[JobId(1019), JobId(1018), JobId(1017)] res:true f:7 g:1
Sep 22 23:14:53.629 DEBG Write     :1021 deps:[JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:53.660 DEBG Write     :1022 deps:[JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:53.668 DEBG Flush     :1023 extent_limit None deps:[JobId(1022), JobId(1021), JobId(1020)] res:true f:8 g:1
Sep 22 23:14:53.698 DEBG Write     :1024 deps:[JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:53.729 DEBG Write     :1025 deps:[JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:53.737 DEBG Flush     :1026 extent_limit None deps:[JobId(1025), JobId(1024), JobId(1023)] res:true f:9 g:1
Sep 22 23:14:53.738 WARN returning error on write!
Sep 22 23:14:53.738 DEBG Write     :1027 deps:[JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:false
Sep 22 23:14:53.738 INFO [lossy] skipping 1028
Sep 22 23:14:53.738 INFO [lossy] skipping 1027
Sep 22 23:14:53.738 INFO [lossy] skipping 1028
Sep 22 23:14:53.738 INFO [lossy] skipping 1027
Sep 22 23:14:53.738 INFO [lossy] skipping 1028
Sep 22 23:14:53.738 INFO [lossy] skipping 1027
Sep 22 23:14:53.738 INFO [lossy] skipping 1028
Sep 22 23:14:53.738 INFO [lossy] skipping 1027
Sep 22 23:14:53.738 WARN 1028 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:14:53.768 DEBG Write     :1027 deps:[JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:53.770 WARN returning error on flush!
Sep 22 23:14:53.770 DEBG Flush     :1028 extent_limit None deps:[JobId(1027), JobId(1026)] res:false f:10 g:1
Sep 22 23:14:53.770 WARN returning error on flush!
Sep 22 23:14:53.770 DEBG Flush     :1028 extent_limit None deps:[JobId(1027), JobId(1026)] res:false f:10 g:1
Sep 22 23:14:53.770 INFO [lossy] skipping 1028
Sep 22 23:14:53.773 DEBG Flush     :1028 extent_limit None deps:[JobId(1027), JobId(1026)] res:true f:10 g:1
Sep 22 23:14:53.773 INFO [lossy] sleeping 1 second
Sep 22 23:14:54.235 INFO [lossy] skipping 1025
Sep 22 23:14:54.235 INFO [lossy] skipping 1027
Sep 22 23:14:54.235 INFO [lossy] skipping 1028
Sep 22 23:14:54.235 INFO [lossy] skipping 1030
Sep 22 23:14:54.235 INFO [lossy] skipping 1025
Sep 22 23:14:54.235 INFO [lossy] skipping 1025
Sep 22 23:14:54.267 DEBG Write     :1025 deps:[JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:54.274 DEBG Flush     :1026 extent_limit None deps:[JobId(1025), JobId(1024), JobId(1023)] res:true f:9 g:1
Sep 22 23:14:54.274 INFO [lossy] skipping 1027
Sep 22 23:14:54.274 WARN 1028 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:14:54.274 WARN 1029 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:14:54.274 INFO [lossy] skipping 1030
Sep 22 23:14:54.304 DEBG Write     :1027 deps:[JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:54.305 WARN 1030 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:14:54.308 DEBG Flush     :1028 extent_limit None deps:[JobId(1027), JobId(1026)] res:true f:10 g:1
Sep 22 23:14:54.339 DEBG Write     :1029 deps:[JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:54.340 WARN returning error on write!
Sep 22 23:14:54.341 DEBG Write     :1030 deps:[JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:false
Sep 22 23:14:54.341 INFO [lossy] skipping 1031
Sep 22 23:14:54.341 INFO [lossy] skipping 1030
Sep 22 23:14:54.341 WARN 1031 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:14:54.341 INFO [lossy] skipping 1030
Sep 22 23:14:54.371 DEBG Write     :1030 deps:[JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:54.378 DEBG Flush     :1031 extent_limit None deps:[JobId(1030), JobId(1029), JobId(1028)] res:true f:11 g:1
Sep 22 23:14:54.378 INFO [lossy] sleeping 1 second
Sep 22 23:14:54.379 INFO [lossy] skipping 1023
Sep 22 23:14:54.379 INFO [lossy] skipping 1030
Sep 22 23:14:54.379 WARN returning error on flush!
Sep 22 23:14:54.379 DEBG Flush     :1023 extent_limit None deps:[JobId(1022), JobId(1021), JobId(1020)] res:false f:8 g:1
Sep 22 23:14:54.379 WARN returning error on flush!
Sep 22 23:14:54.379 DEBG Flush     :1023 extent_limit None deps:[JobId(1022), JobId(1021), JobId(1020)] res:false f:8 g:1
Sep 22 23:14:54.385 DEBG Flush     :1023 extent_limit None deps:[JobId(1022), JobId(1021), JobId(1020)] res:true f:8 g:1
Sep 22 23:14:54.385 INFO [lossy] skipping 1024
Sep 22 23:14:54.416 DEBG Write     :1025 deps:[JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:54.417 WARN 1026 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:14:54.417 WARN 1027 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:14:54.417 WARN 1029 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:14:54.417 WARN 1030 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:14:54.417 INFO [lossy] skipping 1031
Sep 22 23:14:54.448 DEBG Write     :1024 deps:[JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:54.449 INFO [lossy] skipping 1031
Sep 22 23:14:54.449 INFO [lossy] skipping 1031
Sep 22 23:14:54.449 INFO [lossy] skipping 1026
Sep 22 23:14:54.449 INFO [lossy] skipping 1030
Sep 22 23:14:54.449 WARN returning error on flush!
Sep 22 23:14:54.449 DEBG Flush     :1026 extent_limit None deps:[JobId(1025), JobId(1024), JobId(1023)] res:false f:9 g:1
Sep 22 23:14:54.449 INFO [lossy] skipping 1030
Sep 22 23:14:54.449 WARN returning error on flush!
Sep 22 23:14:54.449 DEBG Flush     :1026 extent_limit None deps:[JobId(1025), JobId(1024), JobId(1023)] res:false f:9 g:1
Sep 22 23:14:54.455 DEBG Flush     :1026 extent_limit None deps:[JobId(1025), JobId(1024), JobId(1023)] res:true f:9 g:1
Sep 22 23:14:54.486 DEBG Write     :1027 deps:[JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:54.490 DEBG Flush     :1028 extent_limit None deps:[JobId(1027), JobId(1026)] res:true f:10 g:1
Sep 22 23:14:54.521 DEBG Write     :1029 deps:[JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:54.523 WARN returning error on write!
Sep 22 23:14:54.523 DEBG Write     :1030 deps:[JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:false
Sep 22 23:14:54.523 WARN 1031 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:14:54.524 WARN returning error on write!
Sep 22 23:14:54.524 DEBG Write     :1030 deps:[JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:false
Sep 22 23:14:54.554 DEBG Write     :1030 deps:[JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:54.562 DEBG Flush     :1031 extent_limit None deps:[JobId(1030), JobId(1029), JobId(1028)] res:true f:11 g:1
Sep 22 23:14:54.562 INFO [lossy] sleeping 1 second
Sep 22 23:14:54.784 WARN returning error on write!
Sep 22 23:14:54.784 DEBG Write     :1029 deps:[JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:false
Sep 22 23:14:54.816 DEBG Write     :1030 deps:[JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:54.817 WARN returning error on write!
Sep 22 23:14:54.817 DEBG Write     :1029 deps:[JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:false
Sep 22 23:14:54.818 INFO [lossy] skipping 1029
Sep 22 23:14:54.818 INFO [lossy] skipping 1029
Sep 22 23:14:54.848 DEBG Write     :1029 deps:[JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:54.855 DEBG Flush     :1031 extent_limit None deps:[JobId(1030), JobId(1029), JobId(1028)] res:true f:11 g:1
Sep 22 23:14:54.855 INFO [lossy] skipping 1032
Sep 22 23:14:54.855 INFO [lossy] skipping 1032
Sep 22 23:14:54.885 DEBG Write     :1032 deps:[JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:54.886 INFO [lossy] sleeping 1 second
Sep 22 23:14:55.555 WARN returning error on write!
Sep 22 23:14:55.555 DEBG Write     :1032 deps:[JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:false
Sep 22 23:14:55.556 INFO [lossy] skipping 1033
Sep 22 23:14:55.556 INFO [lossy] skipping 1034
Sep 22 23:14:55.556 INFO [lossy] skipping 1035
Sep 22 23:14:55.586 DEBG Write     :1032 deps:[JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:55.617 DEBG Write     :1033 deps:[JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:55.624 DEBG Flush     :1034 extent_limit None deps:[JobId(1033), JobId(1032), JobId(1031)] res:true f:12 g:1
Sep 22 23:14:55.624 INFO [lossy] skipping 1035
Sep 22 23:14:55.624 INFO [lossy] skipping 1035
Sep 22 23:14:55.624 INFO [lossy] skipping 1035
Sep 22 23:14:55.624 INFO [lossy] skipping 1035
Sep 22 23:14:55.654 DEBG Write     :1035 deps:[JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:55.655 INFO [lossy] sleeping 1 second
Sep 22 23:14:55.656 WARN returning error on write!
Sep 22 23:14:55.656 DEBG Write     :1032 deps:[JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:false
Sep 22 23:14:55.658 WARN returning error on write!
Sep 22 23:14:55.658 DEBG Write     :1033 deps:[JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:false
Sep 22 23:14:55.658 INFO [lossy] skipping 1034
Sep 22 23:14:55.658 INFO [lossy] skipping 1036
Sep 22 23:14:55.688 DEBG Write     :1032 deps:[JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:55.719 DEBG Write     :1033 deps:[JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:55.727 DEBG Flush     :1034 extent_limit None deps:[JobId(1033), JobId(1032), JobId(1031)] res:true f:12 g:1
Sep 22 23:14:55.757 DEBG Write     :1036 deps:[JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:55.758 INFO [lossy] sleeping 1 second
Sep 22 23:14:56.004 DEBG Write     :1033 deps:[JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:56.005 WARN returning error on flush!
Sep 22 23:14:56.005 DEBG Flush     :1034 extent_limit None deps:[JobId(1033), JobId(1032), JobId(1031)] res:false f:12 g:1
Sep 22 23:14:56.005 INFO [lossy] skipping 1036
Sep 22 23:14:56.005 WARN returning error on flush!
Sep 22 23:14:56.005 DEBG Flush     :1034 extent_limit None deps:[JobId(1033), JobId(1032), JobId(1031)] res:false f:12 g:1
Sep 22 23:14:56.012 DEBG Flush     :1034 extent_limit None deps:[JobId(1033), JobId(1032), JobId(1031)] res:true f:12 g:1
Sep 22 23:14:56.012 INFO [lossy] skipping 1035
Sep 22 23:14:56.042 DEBG Write     :1036 deps:[JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:56.043 WARN 1037 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:14:56.043 INFO [lossy] skipping 1038
Sep 22 23:14:56.073 DEBG Write     :1035 deps:[JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:56.074 WARN 1038 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:14:56.074 INFO [lossy] skipping 1037
Sep 22 23:14:56.080 DEBG Flush     :1037 extent_limit None deps:[JobId(1036), JobId(1035), JobId(1034)] res:true f:13 g:1
Sep 22 23:14:56.080 INFO [lossy] sleeping 1 second
Sep 22 23:14:56.767 DEBG Write     :1036 deps:[JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:56.768 INFO [lossy] skipping 1037
Sep 22 23:14:56.768 INFO [lossy] skipping 1037
Sep 22 23:14:56.768 INFO [lossy] skipping 1037
Sep 22 23:14:56.775 DEBG Flush     :1037 extent_limit None deps:[JobId(1036), JobId(1035), JobId(1034)] res:true f:13 g:1
Sep 22 23:14:56.805 DEBG Write     :1038 deps:[JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:56.806 WARN returning error on write!
Sep 22 23:14:56.806 DEBG Write     :1039 deps:[JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:false
Sep 22 23:14:56.807 WARN 1040 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:14:56.807 WARN 1041 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:14:56.807 INFO [lossy] skipping 1043
Sep 22 23:14:56.807 INFO [lossy] skipping 1039
Sep 22 23:14:56.807 INFO [lossy] skipping 1043
Sep 22 23:14:56.837 DEBG Write     :1039 deps:[JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:56.837 WARN 1043 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:14:56.844 DEBG Flush     :1040 extent_limit None deps:[JobId(1039), JobId(1038), JobId(1037)] res:true f:14 g:1
Sep 22 23:14:56.844 INFO [lossy] skipping 1041
Sep 22 23:14:56.844 INFO [lossy] skipping 1042
Sep 22 23:14:56.844 INFO [lossy] skipping 1043
Sep 22 23:14:56.874 DEBG Write     :1041 deps:[JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:56.875 INFO [lossy] skipping 1042
Sep 22 23:14:56.875 WARN 1043 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:14:56.878 DEBG Flush     :1042 extent_limit None deps:[JobId(1041), JobId(1040)] res:true f:15 g:1
Sep 22 23:14:56.908 DEBG Write     :1043 deps:[JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:56.910 INFO [lossy] sleeping 1 second
Sep 22 23:14:56.940 DEBG Write     :1035 deps:[JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:56.941 WARN returning error on flush!
Sep 22 23:14:56.941 DEBG Flush     :1037 extent_limit None deps:[JobId(1036), JobId(1035), JobId(1034)] res:false f:13 g:1
Sep 22 23:14:56.948 DEBG Flush     :1037 extent_limit None deps:[JobId(1036), JobId(1035), JobId(1034)] res:true f:13 g:1
Sep 22 23:14:56.948 INFO [lossy] sleeping 1 second
Sep 22 23:14:57.193 DEBG Write     :1038 deps:[JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:57.225 DEBG Write     :1039 deps:[JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:57.232 DEBG Flush     :1040 extent_limit None deps:[JobId(1039), JobId(1038), JobId(1037)] res:true f:14 g:1
Sep 22 23:14:57.263 DEBG Write     :1041 deps:[JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:57.267 DEBG Flush     :1042 extent_limit None deps:[JobId(1041), JobId(1040)] res:true f:15 g:1
Sep 22 23:14:57.268 WARN returning error on write!
Sep 22 23:14:57.268 DEBG Write     :1043 deps:[JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:false
Sep 22 23:14:57.268 INFO [lossy] skipping 1044
Sep 22 23:14:57.268 INFO [lossy] skipping 1045
Sep 22 23:14:57.268 INFO [lossy] skipping 1043
Sep 22 23:14:57.299 DEBG Write     :1044 deps:[JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:57.300 WARN returning error on write!
Sep 22 23:14:57.300 DEBG Write     :1043 deps:[JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:false
Sep 22 23:14:57.331 DEBG Write     :1043 deps:[JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:57.332 INFO [lossy] sleeping 1 second
Sep 22 23:14:57.984 WARN returning error on write!
Sep 22 23:14:57.984 DEBG Write     :1044 deps:[JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:false
Sep 22 23:14:57.984 INFO [lossy] skipping 1049
Sep 22 23:14:57.985 WARN returning error on write!
Sep 22 23:14:57.985 DEBG Write     :1044 deps:[JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:false
Sep 22 23:14:58.016 DEBG Write     :1044 deps:[JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:58.017 INFO [lossy] sleeping 1 second
Sep 22 23:14:58.018 WARN returning error on write!
Sep 22 23:14:58.018 DEBG Write     :1038 deps:[JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:false
Sep 22 23:14:58.019 WARN returning error on write!
Sep 22 23:14:58.019 DEBG Write     :1039 deps:[JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:false
Sep 22 23:14:58.019 WARN 1040 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:14:58.019 WARN 1041 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:14:58.019 INFO [lossy] skipping 1043
Sep 22 23:14:58.020 WARN returning error on write!
Sep 22 23:14:58.020 DEBG Write     :1038 deps:[JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:false
Sep 22 23:14:58.050 DEBG Write     :1039 deps:[JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:58.051 WARN 1043 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:14:58.081 DEBG Write     :1038 deps:[JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:58.082 INFO [lossy] skipping 1040
Sep 22 23:14:58.082 INFO [lossy] skipping 1046
Sep 22 23:14:58.082 INFO [lossy] skipping 1048
Sep 22 23:14:58.089 DEBG Flush     :1040 extent_limit None deps:[JobId(1039), JobId(1038), JobId(1037)] res:true f:14 g:1
Sep 22 23:14:58.089 WARN 1046 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:14:58.119 DEBG Write     :1041 deps:[JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:58.123 DEBG Flush     :1042 extent_limit None deps:[JobId(1041), JobId(1040)] res:true f:15 g:1
Sep 22 23:14:58.124 INFO [lossy] skipping 1043
Sep 22 23:14:58.124 WARN returning error on write!
Sep 22 23:14:58.124 DEBG Write     :1044 deps:[JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:false
Sep 22 23:14:58.125 INFO [lossy] skipping 1045
Sep 22 23:14:58.125 INFO [lossy] skipping 1046
Sep 22 23:14:58.125 WARN 1047 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:14:58.125 WARN 1049 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:14:58.125 INFO [lossy] skipping 1043
Sep 22 23:14:58.125 INFO [lossy] skipping 1044
Sep 22 23:14:58.125 WARN 1045 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:14:58.125 INFO [lossy] skipping 1046
Sep 22 23:14:58.155 DEBG Write     :1043 deps:[JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:58.186 DEBG Write     :1044 deps:[JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:58.187 WARN 1046 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:14:58.194 DEBG Flush     :1045 extent_limit None deps:[JobId(1044), JobId(1043), JobId(1042)] res:true f:16 g:1
Sep 22 23:14:58.194 INFO [lossy] skipping 1046
Sep 22 23:14:58.224 DEBG Write     :1047 deps:[JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:58.225 WARN 1048 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:14:58.225 WARN 1049 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:14:58.225 INFO [lossy] skipping 1046
Sep 22 23:14:58.256 DEBG Write     :1046 deps:[JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:58.257 WARN returning error on flush!
Sep 22 23:14:58.257 DEBG Flush     :1048 extent_limit None deps:[JobId(1047), JobId(1046), JobId(1045)] res:false f:17 g:1
Sep 22 23:14:58.257 INFO [lossy] skipping 1048
Sep 22 23:14:58.263 DEBG Flush     :1048 extent_limit None deps:[JobId(1047), JobId(1046), JobId(1045)] res:true f:17 g:1
Sep 22 23:14:58.293 DEBG Write     :1049 deps:[JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:58.295 INFO [lossy] sleeping 1 second
Sep 22 23:14:58.515 DEBG Flush     :1045 extent_limit None deps:[JobId(1044), JobId(1043), JobId(1042)] res:true f:16 g:1
Sep 22 23:14:58.515 INFO [lossy] skipping 1046
Sep 22 23:14:58.545 DEBG Write     :1047 deps:[JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:58.546 INFO [lossy] skipping 1046
Sep 22 23:14:58.576 DEBG Write     :1046 deps:[JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:58.577 INFO [lossy] skipping 1048
Sep 22 23:14:58.584 DEBG Flush     :1048 extent_limit None deps:[JobId(1047), JobId(1046), JobId(1045)] res:true f:17 g:1
Sep 22 23:14:58.584 INFO [lossy] skipping 1049
Sep 22 23:14:58.614 DEBG Write     :1049 deps:[JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:58.615 INFO [lossy] sleeping 1 second
Sep 22 23:14:59.049 DEBG Flush     :1045 extent_limit None deps:[JobId(1044), JobId(1043), JobId(1042)] res:true f:16 g:1
Sep 22 23:14:59.080 DEBG Write     :1046 deps:[JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:59.081 INFO [lossy] skipping 1047
Sep 22 23:14:59.081 WARN 1048 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:14:59.081 WARN 1049 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:14:59.081 INFO [lossy] skipping 1050
Sep 22 23:14:59.081 INFO [lossy] skipping 1053
Sep 22 23:14:59.081 INFO [lossy] skipping 1047
Sep 22 23:14:59.081 INFO [lossy] skipping 1053
Sep 22 23:14:59.082 WARN returning error on write!
Sep 22 23:14:59.082 DEBG Write     :1047 deps:[JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:false
Sep 22 23:14:59.083 INFO [lossy] skipping 1053
Sep 22 23:14:59.083 WARN returning error on write!
Sep 22 23:14:59.083 DEBG Write     :1047 deps:[JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:false
Sep 22 23:14:59.113 DEBG Write     :1047 deps:[JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:59.121 DEBG Flush     :1048 extent_limit None deps:[JobId(1047), JobId(1046), JobId(1045)] res:true f:17 g:1
Sep 22 23:14:59.151 DEBG Write     :1049 deps:[JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:59.182 DEBG Write     :1050 deps:[JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:59.189 DEBG Flush     :1051 extent_limit None deps:[JobId(1050), JobId(1049), JobId(1048)] res:true f:18 g:1
Sep 22 23:14:59.220 DEBG Write     :1052 deps:[JobId(1051), JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:59.252 DEBG Write     :1053 deps:[JobId(1051), JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:59.260 DEBG Flush     :1054 extent_limit None deps:[JobId(1053), JobId(1052), JobId(1051)] res:true f:19 g:1
Sep 22 23:14:59.260 INFO [lossy] sleeping 1 second
Sep 22 23:14:59.470 WARN returning error on write!
Sep 22 23:14:59.470 DEBG Write     :1050 deps:[JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:false
Sep 22 23:14:59.470 INFO [lossy] skipping 1051
Sep 22 23:14:59.471 WARN returning error on write!
Sep 22 23:14:59.471 DEBG Write     :1050 deps:[JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:false
Sep 22 23:14:59.501 DEBG Write     :1050 deps:[JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:59.502 INFO [lossy] sleeping 1 second
Sep 22 23:14:59.747 DEBG Write     :1050 deps:[JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:59.755 DEBG Flush     :1051 extent_limit None deps:[JobId(1050), JobId(1049), JobId(1048)] res:true f:18 g:1
Sep 22 23:14:59.755 WARN returning error on write!
Sep 22 23:14:59.755 DEBG Write     :1052 deps:[JobId(1051), JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:false
Sep 22 23:14:59.786 DEBG Write     :1053 deps:[JobId(1051), JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:59.788 WARN returning error on write!
Sep 22 23:14:59.788 DEBG Write     :1052 deps:[JobId(1051), JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:false
Sep 22 23:14:59.788 INFO [lossy] skipping 1052
Sep 22 23:14:59.789 WARN returning error on write!
Sep 22 23:14:59.789 DEBG Write     :1052 deps:[JobId(1051), JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:false
Sep 22 23:14:59.819 DEBG Write     :1052 deps:[JobId(1051), JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:59.820 WARN returning error on flush!
Sep 22 23:14:59.820 DEBG Flush     :1054 extent_limit None deps:[JobId(1053), JobId(1052), JobId(1051)] res:false f:19 g:1
Sep 22 23:14:59.820 WARN returning error on flush!
Sep 22 23:14:59.820 DEBG Flush     :1054 extent_limit None deps:[JobId(1053), JobId(1052), JobId(1051)] res:false f:19 g:1
Sep 22 23:14:59.820 INFO [lossy] skipping 1054
Sep 22 23:14:59.820 INFO [lossy] skipping 1054
Sep 22 23:14:59.820 INFO [lossy] skipping 1054
Sep 22 23:14:59.820 WARN returning error on flush!
Sep 22 23:14:59.820 DEBG Flush     :1054 extent_limit None deps:[JobId(1053), JobId(1052), JobId(1051)] res:false f:19 g:1
Sep 22 23:14:59.826 DEBG Flush     :1054 extent_limit None deps:[JobId(1053), JobId(1052), JobId(1051)] res:true f:19 g:1
Sep 22 23:14:59.856 DEBG Write     :1055 deps:[JobId(1054), JobId(1051), JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:59.858 INFO [lossy] skipping 1056
Sep 22 23:14:59.858 WARN 1057 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:14:59.858 INFO [lossy] skipping 1056
Sep 22 23:14:59.888 DEBG Write     :1056 deps:[JobId(1054), JobId(1051), JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:14:59.889 INFO [lossy] sleeping 1 second
Sep 22 23:15:00.339 DEBG Write     :1055 deps:[JobId(1054), JobId(1051), JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:15:00.341 WARN returning error on write!
Sep 22 23:15:00.341 DEBG Write     :1056 deps:[JobId(1054), JobId(1051), JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:false
Sep 22 23:15:00.341 INFO [lossy] skipping 1057
Sep 22 23:15:00.371 DEBG Write     :1056 deps:[JobId(1054), JobId(1051), JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:15:00.372 INFO [lossy] skipping 1057
Sep 22 23:15:00.378 DEBG Flush     :1057 extent_limit None deps:[JobId(1056), JobId(1055), JobId(1054)] res:true f:20 g:1
Sep 22 23:15:00.379 WARN returning error on write!
Sep 22 23:15:00.379 DEBG Write     :1058 deps:[JobId(1057), JobId(1054), JobId(1051), JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:false
Sep 22 23:15:00.410 DEBG Write     :1059 deps:[JobId(1057), JobId(1054), JobId(1051), JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:15:00.411 INFO [lossy] skipping 1060
Sep 22 23:15:00.441 DEBG Write     :1058 deps:[JobId(1057), JobId(1054), JobId(1051), JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:15:00.442 WARN returning error on flush!
Sep 22 23:15:00.442 DEBG Flush     :1060 extent_limit None deps:[JobId(1059), JobId(1058), JobId(1057)] res:false f:21 g:1
Sep 22 23:15:00.448 DEBG Flush     :1060 extent_limit None deps:[JobId(1059), JobId(1058), JobId(1057)] res:true f:21 g:1
Sep 22 23:15:00.448 INFO [lossy] sleeping 1 second
Sep 22 23:15:00.656 INFO [lossy] skipping 1051
Sep 22 23:15:00.656 INFO [lossy] skipping 1058
Sep 22 23:15:00.656 INFO [lossy] skipping 1059
Sep 22 23:15:00.663 DEBG Flush     :1051 extent_limit None deps:[JobId(1050), JobId(1049), JobId(1048)] res:true f:18 g:1
Sep 22 23:15:00.663 INFO [lossy] skipping 1059
Sep 22 23:15:00.663 INFO [lossy] skipping 1059
Sep 22 23:15:00.664 WARN returning error on write!
Sep 22 23:15:00.664 DEBG Write     :1052 deps:[JobId(1051), JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:false
Sep 22 23:15:00.695 DEBG Write     :1053 deps:[JobId(1051), JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:15:00.696 INFO [lossy] skipping 1054
Sep 22 23:15:00.696 WARN 1055 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:15:00.696 INFO [lossy] skipping 1056
Sep 22 23:15:00.696 INFO [lossy] skipping 1060
Sep 22 23:15:00.696 INFO [lossy] skipping 1052
Sep 22 23:15:00.696 WARN 1054 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:15:00.696 INFO [lossy] skipping 1056
Sep 22 23:15:00.696 INFO [lossy] skipping 1060
Sep 22 23:15:00.725 DEBG Write     :1052 deps:[JobId(1051), JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:15:00.726 INFO [lossy] skipping 1056
Sep 22 23:15:00.726 WARN 1056 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:15:00.733 DEBG Flush     :1054 extent_limit None deps:[JobId(1053), JobId(1052), JobId(1051)] res:true f:19 g:1
Sep 22 23:15:00.763 DEBG Write     :1055 deps:[JobId(1054), JobId(1051), JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:15:00.765 WARN returning error on write!
Sep 22 23:15:00.765 DEBG Write     :1056 deps:[JobId(1054), JobId(1051), JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:false
Sep 22 23:15:00.765 WARN 1057 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:15:00.765 WARN 1058 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:15:00.765 WARN 1059 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:15:00.766 WARN returning error on write!
Sep 22 23:15:00.766 DEBG Write     :1056 deps:[JobId(1054), JobId(1051), JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:false
Sep 22 23:15:00.767 WARN returning error on write!
Sep 22 23:15:00.767 DEBG Write     :1056 deps:[JobId(1054), JobId(1051), JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:false
Sep 22 23:15:00.767 INFO [lossy] skipping 1056
Sep 22 23:15:00.797 DEBG Write     :1056 deps:[JobId(1054), JobId(1051), JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:15:00.805 DEBG Flush     :1057 extent_limit None deps:[JobId(1056), JobId(1055), JobId(1054)] res:true f:20 g:1
Sep 22 23:15:00.806 WARN returning error on write!
Sep 22 23:15:00.806 DEBG Write     :1058 deps:[JobId(1057), JobId(1054), JobId(1051), JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:false
Sep 22 23:15:00.806 INFO [lossy] skipping 1059
Sep 22 23:15:00.806 INFO [lossy] skipping 1060
Sep 22 23:15:00.807 WARN returning error on write!
Sep 22 23:15:00.807 DEBG Write     :1058 deps:[JobId(1057), JobId(1054), JobId(1051), JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:false
Sep 22 23:15:00.808 WARN returning error on write!
Sep 22 23:15:00.808 DEBG Write     :1059 deps:[JobId(1057), JobId(1054), JobId(1051), JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:false
Sep 22 23:15:00.808 WARN 1060 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:15:00.838 DEBG Write     :1058 deps:[JobId(1057), JobId(1054), JobId(1051), JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:15:00.840 WARN returning error on write!
Sep 22 23:15:00.840 DEBG Write     :1059 deps:[JobId(1057), JobId(1054), JobId(1051), JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:false
Sep 22 23:15:00.870 DEBG Write     :1059 deps:[JobId(1057), JobId(1054), JobId(1051), JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:15:00.878 DEBG Flush     :1060 extent_limit None deps:[JobId(1059), JobId(1058), JobId(1057)] res:true f:21 g:1
Sep 22 23:15:00.878 INFO [lossy] sleeping 1 second
Sep 22 23:15:01.093 DEBG Flush     :1057 extent_limit None deps:[JobId(1056), JobId(1055), JobId(1054)] res:true f:20 g:1
Sep 22 23:15:01.123 DEBG Write     :1058 deps:[JobId(1057), JobId(1054), JobId(1051), JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:15:01.154 DEBG Write     :1059 deps:[JobId(1057), JobId(1054), JobId(1051), JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:15:01.162 DEBG Flush     :1060 extent_limit None deps:[JobId(1059), JobId(1058), JobId(1057)] res:true f:21 g:1
Sep 22 23:15:01.192 DEBG Write     :1061 deps:[JobId(1060), JobId(1057), JobId(1054), JobId(1051), JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:15:01.194 INFO [lossy] sleeping 1 second
Sep 22 23:15:01.614 INFO [lossy] skipping 1061
Sep 22 23:15:01.646 DEBG Write     :1062 deps:[JobId(1060), JobId(1057), JobId(1054), JobId(1051), JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:15:01.647 INFO [lossy] skipping 1063
Sep 22 23:15:01.647 INFO [lossy] skipping 1061
Sep 22 23:15:01.647 INFO [lossy] skipping 1061
Sep 22 23:15:01.679 DEBG Write     :1061 deps:[JobId(1060), JobId(1057), JobId(1054), JobId(1051), JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:15:01.680 INFO [lossy] skipping 1063
Sep 22 23:15:01.680 INFO [lossy] skipping 1063
Sep 22 23:15:01.686 DEBG Flush     :1063 extent_limit None deps:[JobId(1062), JobId(1061), JobId(1060)] res:true f:22 g:1
Sep 22 23:15:01.687 INFO [lossy] sleeping 1 second
Sep 22 23:15:01.928 DEBG Write     :1061 deps:[JobId(1060), JobId(1057), JobId(1054), JobId(1051), JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:15:01.959 DEBG Write     :1062 deps:[JobId(1060), JobId(1057), JobId(1054), JobId(1051), JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:15:01.961 INFO [lossy] skipping 1063
Sep 22 23:15:01.961 INFO [lossy] skipping 1065
Sep 22 23:15:01.961 INFO [lossy] skipping 1067
Sep 22 23:15:01.967 DEBG Flush     :1063 extent_limit None deps:[JobId(1062), JobId(1061), JobId(1060)] res:true f:22 g:1
Sep 22 23:15:01.998 DEBG Write     :1065 deps:[JobId(1063), JobId(1060), JobId(1057), JobId(1054), JobId(1051), JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:15:02.030 DEBG Write     :1064 deps:[JobId(1063), JobId(1060), JobId(1057), JobId(1054), JobId(1051), JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:15:02.038 DEBG Flush     :1066 extent_limit None deps:[JobId(1065), JobId(1064), JobId(1063)] res:true f:23 g:1
Sep 22 23:15:02.069 DEBG Write     :1067 deps:[JobId(1066), JobId(1063), JobId(1060), JobId(1057), JobId(1054), JobId(1051), JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:15:02.070 INFO [lossy] sleeping 1 second
Sep 22 23:15:02.271 DEBG Write     :1000 deps:[] res:true
Sep 22 23:15:02.312 DEBG Write     :1062 deps:[JobId(1060), JobId(1057), JobId(1054), JobId(1051), JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:15:02.320 DEBG Flush     :1063 extent_limit None deps:[JobId(1062), JobId(1061), JobId(1060)] res:true f:22 g:1
Sep 22 23:15:02.351 DEBG Write     :1064 deps:[JobId(1063), JobId(1060), JobId(1057), JobId(1054), JobId(1051), JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:15:02.383 DEBG Write     :1065 deps:[JobId(1063), JobId(1060), JobId(1057), JobId(1054), JobId(1051), JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:15:02.391 DEBG Flush     :1066 extent_limit None deps:[JobId(1065), JobId(1064), JobId(1063)] res:true f:23 g:1
Sep 22 23:15:02.423 DEBG Write     :1067 deps:[JobId(1066), JobId(1063), JobId(1060), JobId(1057), JobId(1054), JobId(1051), JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:15:02.425 WARN returning error on write!
Sep 22 23:15:02.425 DEBG Write     :1068 deps:[JobId(1066), JobId(1063), JobId(1060), JobId(1057), JobId(1054), JobId(1051), JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:false
Sep 22 23:15:02.456 DEBG Write     :1068 deps:[JobId(1066), JobId(1063), JobId(1060), JobId(1057), JobId(1054), JobId(1051), JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:15:02.458 WARN returning error on flush!
Sep 22 23:15:02.458 DEBG Flush     :1069 extent_limit None deps:[JobId(1068), JobId(1067), JobId(1066)] res:false f:24 g:1
Sep 22 23:15:02.458 INFO [lossy] skipping 1069
Sep 22 23:15:02.458 WARN returning error on flush!
Sep 22 23:15:02.458 DEBG Flush     :1069 extent_limit None deps:[JobId(1068), JobId(1067), JobId(1066)] res:false f:24 g:1
Sep 22 23:15:02.464 DEBG Flush     :1069 extent_limit None deps:[JobId(1068), JobId(1067), JobId(1066)] res:true f:24 g:1
Sep 22 23:15:02.465 INFO [lossy] sleeping 1 second
Sep 22 23:15:02.618 ERRO [0] job id 1007 saw error GenericError("test error")
Sep 22 23:15:02.618 ERRO [0] job id 1007 saw error GenericError("test error")
Sep 22 23:15:02.618 ERRO [0] job id 1009 saw error GenericError("test error")
Sep 22 23:15:02.619 ERRO [0] job id 1016 saw error GenericError("test error")
Sep 22 23:15:02.619 ERRO [0] job id 1017 saw error GenericError("test error")
Sep 22 23:15:02.619 ERRO [0] job id 1019 saw error GenericError("test error")
Sep 22 23:15:02.619 ERRO [0] job id 1020 saw error GenericError("test error")
Sep 22 23:15:02.619 ERRO [0] job id 1023 saw error GenericError("test error")
Sep 22 23:15:02.619 ERRO [0] job id 1023 saw error GenericError("test error")
Sep 22 23:15:02.620 DEBG up_ds_listen was notified
Sep 22 23:15:02.620 DEBG up_ds_listen process 1008
Sep 22 23:15:02.620 DEBG [A] ack job 1008:9, : downstairs
Sep 22 23:15:02.620 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:02.620 ERRO [1] job id 1008 saw error GenericError("test error")
Sep 22 23:15:02.620 DEBG up_ds_listen was notified
Sep 22 23:15:02.620 DEBG up_ds_listen process 1011
Sep 22 23:15:02.620 DEBG [A] ack job 1011:12, : downstairs
Sep 22 23:15:02.620 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:02.620 ERRO [1] job id 1008 saw error GenericError("test error")
Sep 22 23:15:02.620 ERRO [2] job id 1012 saw error GenericError("test error")
Sep 22 23:15:02.623 DEBG [rc] retire 1008 clears [JobId(1006), JobId(1007), JobId(1008)], : downstairs
Sep 22 23:15:02.623 ERRO [2] job id 1012 saw error GenericError("test error")
Sep 22 23:15:02.623 ERRO [1] job id 1010 saw error GenericError("test error")
Sep 22 23:15:02.623 ERRO [2] job id 1014 saw error GenericError("test error")
Sep 22 23:15:02.626 DEBG [rc] retire 1011 clears [JobId(1009), JobId(1010), JobId(1011)], : downstairs
Sep 22 23:15:02.626 DEBG up_ds_listen was notified
Sep 22 23:15:02.626 DEBG up_ds_listen process 1014
Sep 22 23:15:02.626 DEBG [A] ack job 1014:15, : downstairs
Sep 22 23:15:02.626 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:02.626 ERRO [0] job id 1030 saw error GenericError("test error")
Sep 22 23:15:02.626 ERRO [1] job id 1012 saw error GenericError("test error")
Sep 22 23:15:02.633 ERRO [2] job id 1016 saw error GenericError("test error")
Sep 22 23:15:02.633 ERRO [2] job id 1016 saw error GenericError("test error")
Sep 22 23:15:02.634 ERRO [0] job id 1032 saw error GenericError("test error")
Sep 22 23:15:02.634 ERRO [1] job id 1014 saw error GenericError("test error")
Sep 22 23:15:02.637 DEBG [rc] retire 1014 clears [JobId(1012), JobId(1013), JobId(1014)], : downstairs
Sep 22 23:15:02.637 DEBG up_ds_listen was notified
Sep 22 23:15:02.637 DEBG up_ds_listen process 1017
Sep 22 23:15:02.637 DEBG [A] ack job 1017:18, : downstairs
Sep 22 23:15:02.637 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:02.640 DEBG [rc] retire 1017 clears [JobId(1015), JobId(1016), JobId(1017)], : downstairs
Sep 22 23:15:02.640 DEBG up_ds_listen was notified
Sep 22 23:15:02.640 DEBG up_ds_listen process 1020
Sep 22 23:15:02.640 DEBG [A] ack job 1020:21, : downstairs
Sep 22 23:15:02.640 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:02.640 ERRO [1] job id 1020 saw error GenericError("test error")
Sep 22 23:15:02.640 DEBG up_ds_listen was notified
Sep 22 23:15:02.640 DEBG up_ds_listen process 1023
Sep 22 23:15:02.640 DEBG [A] ack job 1023:24, : downstairs
Sep 22 23:15:02.640 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:02.640 ERRO [0] job id 1039 saw error GenericError("test error")
Sep 22 23:15:02.643 DEBG [rc] retire 1020 clears [JobId(1018), JobId(1019), JobId(1020)], : downstairs
Sep 22 23:15:02.644 ERRO [1] job id 1022 saw error GenericError("test error")
Sep 22 23:15:02.644 DEBG up_ds_listen was notified
Sep 22 23:15:02.644 DEBG up_ds_listen process 1026
Sep 22 23:15:02.644 DEBG [A] ack job 1026:27, : downstairs
Sep 22 23:15:02.644 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:02.644 ERRO [2] job id 1027 saw error GenericError("test error")
Sep 22 23:15:02.644 ERRO [1] job id 1023 saw error GenericError("test error")
Sep 22 23:15:02.644 ERRO [2] job id 1028 saw error GenericError("test error")
Sep 22 23:15:02.644 ERRO [1] job id 1023 saw error GenericError("test error")
Sep 22 23:15:02.644 ERRO [0] job id 1044 saw error GenericError("test error")
Sep 22 23:15:02.644 ERRO [2] job id 1028 saw error GenericError("test error")
Sep 22 23:15:02.647 DEBG [rc] retire 1023 clears [JobId(1021), JobId(1022), JobId(1023)], : downstairs
Sep 22 23:15:02.647 ERRO [0] job id 1044 saw error GenericError("test error")
Sep 22 23:15:02.647 DEBG up_ds_listen was notified
Sep 22 23:15:02.647 DEBG up_ds_listen process 1028
Sep 22 23:15:02.647 DEBG [A] ack job 1028:29, : downstairs
Sep 22 23:15:02.647 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:02.647 ERRO [2] job id 1029 saw error GenericError("test error")
Sep 22 23:15:02.647 ERRO [1] job id 1026 saw error GenericError("test error")
Sep 22 23:15:02.647 ERRO [2] job id 1029 saw error GenericError("test error")
Sep 22 23:15:02.647 ERRO [1] job id 1026 saw error GenericError("test error")
Sep 22 23:15:02.647 ERRO [0] job id 1047 saw error GenericError("test error")
Sep 22 23:15:02.650 DEBG [rc] retire 1026 clears [JobId(1024), JobId(1025), JobId(1026)], : downstairs
Sep 22 23:15:02.650 ERRO [0] job id 1047 saw error GenericError("test error")
Sep 22 23:15:02.651 DEBG up_ds_listen was notified
Sep 22 23:15:02.651 DEBG up_ds_listen process 1031
Sep 22 23:15:02.651 DEBG [A] ack job 1031:32, : downstairs
Sep 22 23:15:02.651 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:02.652 DEBG [rc] retire 1028 clears [JobId(1027), JobId(1028)], : downstairs
Sep 22 23:15:02.652 ERRO [2] job id 1034 saw error GenericError("test error")
Sep 22 23:15:02.652 ERRO [1] job id 1030 saw error GenericError("test error")
Sep 22 23:15:02.652 ERRO [2] job id 1034 saw error GenericError("test error")
Sep 22 23:15:02.652 ERRO [1] job id 1030 saw error GenericError("test error")
Sep 22 23:15:02.652 DEBG up_ds_listen was notified
Sep 22 23:15:02.652 DEBG up_ds_listen process 1034
Sep 22 23:15:02.652 DEBG [A] ack job 1034:35, : downstairs
Sep 22 23:15:02.652 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:02.656 DEBG [rc] retire 1031 clears [JobId(1029), JobId(1030), JobId(1031)], : downstairs
Sep 22 23:15:02.656 ERRO [1] job id 1032 saw error GenericError("test error")
Sep 22 23:15:02.656 ERRO [1] job id 1033 saw error GenericError("test error")
Sep 22 23:15:02.656 DEBG up_ds_listen was notified
Sep 22 23:15:02.656 DEBG up_ds_listen process 1037
Sep 22 23:15:02.656 DEBG [A] ack job 1037:38, : downstairs
Sep 22 23:15:02.656 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:02.656 ERRO [0] job id 1056 saw error GenericError("test error")
Sep 22 23:15:02.659 DEBG [rc] retire 1034 clears [JobId(1032), JobId(1033), JobId(1034)], : downstairs
Sep 22 23:15:02.659 DEBG up_ds_listen was notified
Sep 22 23:15:02.659 DEBG up_ds_listen process 1040
Sep 22 23:15:02.659 DEBG [A] ack job 1040:41, : downstairs
Sep 22 23:15:02.659 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:02.659 ERRO [0] job id 1058 saw error GenericError("test error")
Sep 22 23:15:02.659 DEBG up_ds_listen was notified
Sep 22 23:15:02.659 DEBG up_ds_listen process 1042
Sep 22 23:15:02.659 DEBG [A] ack job 1042:43, : downstairs
Sep 22 23:15:02.659 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:02.659 ERRO [2] job id 1043 saw error GenericError("test error")
Sep 22 23:15:02.659 ERRO [1] job id 1037 saw error GenericError("test error")
Sep 22 23:15:02.662 DEBG [rc] retire 1037 clears [JobId(1035), JobId(1036), JobId(1037)], : downstairs
Sep 22 23:15:02.663 ERRO [0] job id 1060 saw error GenericError("test error")
Sep 22 23:15:02.663 ERRO [2] job id 1043 saw error GenericError("test error")
Sep 22 23:15:02.663 ERRO [1] job id 1038 saw error GenericError("test error")
Sep 22 23:15:02.663 ERRO [1] job id 1039 saw error GenericError("test error")
Sep 22 23:15:02.663 ERRO [1] job id 1038 saw error GenericError("test error")
Sep 22 23:15:02.663 DEBG up_ds_listen was notified
Sep 22 23:15:02.663 DEBG up_ds_listen process 1045
Sep 22 23:15:02.663 DEBG [A] ack job 1045:46, : downstairs
Sep 22 23:15:02.663 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:02.666 DEBG [rc] retire 1040 clears [JobId(1038), JobId(1039), JobId(1040)], : downstairs
Sep 22 23:15:02.666 DEBG up_ds_listen was notified
Sep 22 23:15:02.666 DEBG up_ds_listen process 1048
Sep 22 23:15:02.666 DEBG [A] ack job 1048:49, : downstairs
Sep 22 23:15:02.666 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:02.667 DEBG [rc] retire 1042 clears [JobId(1041), JobId(1042)], : downstairs
Sep 22 23:15:02.667 ERRO [1] job id 1044 saw error GenericError("test error")
Sep 22 23:15:02.667 DEBG up_ds_listen was notified
Sep 22 23:15:02.667 DEBG up_ds_listen process 1051
Sep 22 23:15:02.667 DEBG [A] ack job 1051:52, : downstairs
Sep 22 23:15:02.667 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:02.667 ERRO [2] job id 1052 saw error GenericError("test error")
Sep 22 23:15:02.668 ERRO [2] job id 1052 saw error GenericError("test error")
Sep 22 23:15:02.670 DEBG [rc] retire 1045 clears [JobId(1043), JobId(1044), JobId(1045)], : downstairs
Sep 22 23:15:02.670 ERRO [2] job id 1052 saw error GenericError("test error")
Sep 22 23:15:02.670 ERRO [2] job id 1054 saw error GenericError("test error")
Sep 22 23:15:02.670 ERRO [1] job id 1048 saw error GenericError("test error")
Sep 22 23:15:02.671 ERRO [2] job id 1054 saw error GenericError("test error")
Sep 22 23:15:02.673 DEBG [rc] retire 1048 clears [JobId(1046), JobId(1047), JobId(1048)], : downstairs
Sep 22 23:15:02.673 ERRO [2] job id 1054 saw error GenericError("test error")
Sep 22 23:15:02.673 ERRO [1] job id 1050 saw error GenericError("test error")
Sep 22 23:15:02.674 DEBG up_ds_listen was notified
Sep 22 23:15:02.674 DEBG up_ds_listen process 1054
Sep 22 23:15:02.674 DEBG [A] ack job 1054:55, : downstairs
Sep 22 23:15:02.674 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:02.674 ERRO [1] job id 1050 saw error GenericError("test error")
Sep 22 23:15:02.677 DEBG [rc] retire 1051 clears [JobId(1049), JobId(1050), JobId(1051)], : downstairs
Sep 22 23:15:02.677 DEBG up_ds_listen was notified
Sep 22 23:15:02.677 DEBG up_ds_listen process 1057
Sep 22 23:15:02.677 DEBG [A] ack job 1057:58, : downstairs
Sep 22 23:15:02.677 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:02.677 ERRO [1] job id 1052 saw error GenericError("test error")
Sep 22 23:15:02.677 DEBG up_ds_listen was notified
Sep 22 23:15:02.677 DEBG up_ds_listen process 1060
Sep 22 23:15:02.677 DEBG [A] ack job 1060:61, : downstairs
Sep 22 23:15:02.677 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:02.680 DEBG [rc] retire 1054 clears [JobId(1052), JobId(1053), JobId(1054)], : downstairs
Sep 22 23:15:02.680 ERRO [1] job id 1056 saw error GenericError("test error")
Sep 22 23:15:02.680 DEBG up_ds_listen was notified
Sep 22 23:15:02.680 DEBG up_ds_listen process 1063
Sep 22 23:15:02.680 DEBG [A] ack job 1063:64, : downstairs
Sep 22 23:15:02.680 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:02.680 ERRO [1] job id 1056 saw error GenericError("test error")
Sep 22 23:15:02.680 ERRO [1] job id 1056 saw error GenericError("test error")
Sep 22 23:15:02.683 DEBG [rc] retire 1057 clears [JobId(1055), JobId(1056), JobId(1057)], : downstairs
Sep 22 23:15:02.683 ERRO [2] job id 1068 saw error GenericError("test error")
Sep 22 23:15:02.683 ERRO [1] job id 1058 saw error GenericError("test error")
Sep 22 23:15:02.683 ERRO [1] job id 1058 saw error GenericError("test error")
Sep 22 23:15:02.683 ERRO [2] job id 1069 saw error GenericError("test error")
Sep 22 23:15:02.683 ERRO [1] job id 1059 saw error GenericError("test error")
Sep 22 23:15:02.683 ERRO [2] job id 1069 saw error GenericError("test error")
Sep 22 23:15:02.683 ERRO [1] job id 1059 saw error GenericError("test error")
Sep 22 23:15:02.686 DEBG [rc] retire 1060 clears [JobId(1058), JobId(1059), JobId(1060)], : downstairs
Sep 22 23:15:02.689 DEBG [rc] retire 1063 clears [JobId(1061), JobId(1062), JobId(1063)], : downstairs
Sep 22 23:15:02.689 DEBG up_ds_listen was notified
Sep 22 23:15:02.689 DEBG up_ds_listen process 1066
Sep 22 23:15:02.689 DEBG [A] ack job 1066:67, : downstairs
Sep 22 23:15:02.689 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:02.717 DEBG Write     :1000 deps:[] res:true
Sep 22 23:15:02.721 DEBG Write     :1064 deps:[JobId(1063), JobId(1060), JobId(1057), JobId(1054), JobId(1051), JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:15:02.723 WARN returning error on write!
Sep 22 23:15:02.723 DEBG Write     :1065 deps:[JobId(1063), JobId(1060), JobId(1057), JobId(1054), JobId(1051), JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:false
Sep 22 23:15:02.724 INFO [lossy] skipping 1070
Sep 22 23:15:02.724 INFO [lossy] skipping 1072
Sep 22 23:15:02.754 DEBG Write     :1065 deps:[JobId(1063), JobId(1060), JobId(1057), JobId(1054), JobId(1051), JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:15:02.756 WARN returning error on flush!
Sep 22 23:15:02.756 DEBG Flush     :1066 extent_limit None deps:[JobId(1065), JobId(1064), JobId(1063)] res:false f:23 g:1
Sep 22 23:15:02.756 INFO [lossy] skipping 1067
Sep 22 23:15:02.756 INFO [lossy] skipping 1070
Sep 22 23:15:02.762 DEBG Flush     :1066 extent_limit None deps:[JobId(1065), JobId(1064), JobId(1063)] res:true f:23 g:1
Sep 22 23:15:02.793 DEBG Write     :1067 deps:[JobId(1066), JobId(1063), JobId(1060), JobId(1057), JobId(1054), JobId(1051), JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:15:02.794 WARN 1070 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:15:02.825 DEBG Write     :1068 deps:[JobId(1066), JobId(1063), JobId(1060), JobId(1057), JobId(1054), JobId(1051), JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:15:02.833 DEBG Flush     :1069 extent_limit None deps:[JobId(1068), JobId(1067), JobId(1066)] res:true f:24 g:1
Sep 22 23:15:02.834 WARN returning error on write!
Sep 22 23:15:02.834 DEBG Write     :1070 deps:[JobId(1069), JobId(1066), JobId(1063), JobId(1060), JobId(1057), JobId(1054), JobId(1051), JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:false
Sep 22 23:15:02.834 WARN 1071 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:15:02.834 INFO [lossy] skipping 1072
Sep 22 23:15:02.834 INFO [lossy] skipping 1073
Sep 22 23:15:02.835 WARN returning error on write!
Sep 22 23:15:02.835 DEBG Write     :1070 deps:[JobId(1069), JobId(1066), JobId(1063), JobId(1060), JobId(1057), JobId(1054), JobId(1051), JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:false
Sep 22 23:15:02.865 DEBG Write     :1070 deps:[JobId(1069), JobId(1066), JobId(1063), JobId(1060), JobId(1057), JobId(1054), JobId(1051), JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:15:02.867 INFO [lossy] sleeping 1 second
Sep 22 23:15:02.867 ERRO [0] job id 1065 saw error GenericError("test error")
Sep 22 23:15:02.867 ERRO [0] job id 1066 saw error GenericError("test error")
Sep 22 23:15:02.870 DEBG [rc] retire 1066 clears [JobId(1064), JobId(1065), JobId(1066)], : downstairs
Sep 22 23:15:02.870 ERRO [0] job id 1070 saw error GenericError("test error")
Sep 22 23:15:02.870 ERRO [0] job id 1070 saw error GenericError("test error")
Sep 22 23:15:02.870 DEBG up_ds_listen was notified
Sep 22 23:15:02.870 DEBG up_ds_listen process 1069
Sep 22 23:15:02.870 DEBG [A] ack job 1069:70, : downstairs
Sep 22 23:15:02.870 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:03.073 WARN returning error on write!
Sep 22 23:15:03.073 DEBG Write     :1068 deps:[JobId(1066), JobId(1063), JobId(1060), JobId(1057), JobId(1054), JobId(1051), JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:false
Sep 22 23:15:03.073 INFO [lossy] skipping 1069
Sep 22 23:15:03.073 INFO [lossy] skipping 1073
Sep 22 23:15:03.104 DEBG Write     :1068 deps:[JobId(1066), JobId(1063), JobId(1060), JobId(1057), JobId(1054), JobId(1051), JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:15:03.105 WARN returning error on flush!
Sep 22 23:15:03.105 DEBG Flush     :1069 extent_limit None deps:[JobId(1068), JobId(1067), JobId(1066)] res:false f:24 g:1
Sep 22 23:15:03.105 WARN returning error on flush!
Sep 22 23:15:03.105 DEBG Flush     :1069 extent_limit None deps:[JobId(1068), JobId(1067), JobId(1066)] res:false f:24 g:1
Sep 22 23:15:03.105 INFO [lossy] skipping 1069
Sep 22 23:15:03.112 DEBG Flush     :1069 extent_limit None deps:[JobId(1068), JobId(1067), JobId(1066)] res:true f:24 g:1
Sep 22 23:15:03.112 INFO [lossy] sleeping 1 second
Sep 22 23:15:03.112 ERRO [1] job id 1068 saw error GenericError("test error")
Sep 22 23:15:03.112 ERRO [1] job id 1069 saw error GenericError("test error")
Sep 22 23:15:03.112 ERRO [1] job id 1069 saw error GenericError("test error")
Sep 22 23:15:03.115 DEBG [rc] retire 1069 clears [JobId(1067), JobId(1068), JobId(1069)], : downstairs
Sep 22 23:15:03.159 DEBG Write     :1000 deps:[] res:true
Sep 22 23:15:03.453 WARN 6e5c0f70-fd56-4280-9d20-71288e488216 request to replace downstairs 127.0.0.1:42762 with 127.0.0.1:50216
Sep 22 23:15:03.453 INFO 6e5c0f70-fd56-4280-9d20-71288e488216 found new target: 127.0.0.1:50216 at 0
Waited for some repair work, proceeding with test
Sep 22 23:15:03.495 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:1 g:1
Sep 22 23:15:03.496 DEBG Write     :1070 deps:[JobId(1069), JobId(1066), JobId(1063), JobId(1060), JobId(1057), JobId(1054), JobId(1051), JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:15:03.500 DEBG Flush     :1071 extent_limit None deps:[JobId(1070), JobId(1069)] res:true f:25 g:1
Sep 22 23:15:03.500 INFO [lossy] skipping 1072
Sep 22 23:15:03.507 DEBG Read      :1072 deps:[JobId(1071)] res:true
Sep 22 23:15:03.530 INFO [lossy] sleeping 1 second
Sep 22 23:15:03.537 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:1 g:1
Sep 22 23:15:03.537 WARN [0] will exit pm_task, this downstairs Replacing
Sep 22 23:15:03.538 ERRO 127.0.0.1:42762: proc: [0] client work task ended, Ok(Err([0] This downstairs now in Replacing)), so we end too, looper: 0
Sep 22 23:15:03.538 INFO [0] 6e5c0f70-fd56-4280-9d20-71288e488216 Gone missing, transition from Replacing to Replaced
Sep 22 23:15:03.538 INFO [0] 6e5c0f70-fd56-4280-9d20-71288e488216 connection to 127.0.0.1:42762 closed, looper: 0
Sep 22 23:15:03.538 INFO [0] 127.0.0.1:42762 task reports connection:false
Sep 22 23:15:03.538 INFO 6e5c0f70-fd56-4280-9d20-71288e488216 Replaced Active Active
Sep 22 23:15:03.538 INFO [0] 127.0.0.1:42762 task reports offline
Sep 22 23:15:03.538 INFO Upstairs starts
Sep 22 23:15:03.538 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:15:03.538 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:15:03.538 INFO Crucible stats registered with UUID: 6e5c0f70-fd56-4280-9d20-71288e488216
Sep 22 23:15:03.538 INFO Crucible 6e5c0f70-fd56-4280-9d20-71288e488216 has session id: bc593f63-fb7c-4043-a1b6-d037ee9b50f2
Sep 22 23:15:03.538 WARN upstairs UpstairsConnection { upstairs_id: 6e5c0f70-fd56-4280-9d20-71288e488216, session_id: f334c95e-b851-4a8a-a731-3fb69e42e934, gen: 1 } disconnected, 0 jobs left, task: main
Sep 22 23:15:03.538 WARN upstairs UpstairsConnection { upstairs_id: 6e5c0f70-fd56-4280-9d20-71288e488216, session_id: f334c95e-b851-4a8a-a731-3fb69e42e934, gen: 1 } was previously active, clearing, task: main
Sep 22 23:15:03.538 INFO connection (127.0.0.1:49772): all done
Sep 22 23:15:03.538 INFO [0] connecting to 127.0.0.1:50216, looper: 0
Sep 22 23:15:03.539 INFO [1] connecting to 127.0.0.1:64149, looper: 1
Sep 22 23:15:03.539 INFO [2] connecting to 127.0.0.1:58182, looper: 2
Sep 22 23:15:03.539 INFO up_listen starts, task: up_listen
Sep 22 23:15:03.539 INFO Wait for all three downstairs to come online
Sep 22 23:15:03.539 INFO Flush timeout: 0.5
Sep 22 23:15:03.539 DEBG up_ds_listen was notified
Sep 22 23:15:03.539 DEBG up_ds_listen process 1001
Sep 22 23:15:03.539 DEBG [A] ack job 1001:2, : downstairs
Sep 22 23:15:03.609 DEBG [rc] retire 1001 clears [JobId(1000), JobId(1001)], : downstairs
Sep 22 23:15:03.610 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:03.610 INFO [0] 6e5c0f70-fd56-4280-9d20-71288e488216 looper connected, looper: 0
Sep 22 23:15:03.610 INFO [0] Proc runs for 127.0.0.1:50216 in state New
Sep 22 23:15:03.610 INFO [1] 6e5c0f70-fd56-4280-9d20-71288e488216 looper connected, looper: 1
Sep 22 23:15:03.610 INFO [1] Proc runs for 127.0.0.1:64149 in state New
Sep 22 23:15:03.610 INFO [2] 6e5c0f70-fd56-4280-9d20-71288e488216 looper connected, looper: 2
Sep 22 23:15:03.610 INFO [2] Proc runs for 127.0.0.1:58182 in state New
Sep 22 23:15:03.610 INFO accepted connection from 127.0.0.1:33442, task: main
Sep 22 23:15:03.610 INFO accepted connection from 127.0.0.1:39895, task: main
Sep 22 23:15:03.610 INFO accepted connection from 127.0.0.1:37976, task: main
Sep 22 23:15:03.611 INFO Connection request from 6e5c0f70-fd56-4280-9d20-71288e488216 with version 4, task: proc
Sep 22 23:15:03.611 INFO upstairs UpstairsConnection { upstairs_id: 6e5c0f70-fd56-4280-9d20-71288e488216, session_id: 2cf90053-4dbd-4cc4-8468-9d35f085e47a, gen: 2 } connected, version 4, task: proc
Sep 22 23:15:03.611 INFO Connection request from 6e5c0f70-fd56-4280-9d20-71288e488216 with version 4, task: proc
Sep 22 23:15:03.611 INFO upstairs UpstairsConnection { upstairs_id: 6e5c0f70-fd56-4280-9d20-71288e488216, session_id: 2cf90053-4dbd-4cc4-8468-9d35f085e47a, gen: 2 } connected, version 4, task: proc
Sep 22 23:15:03.611 INFO Connection request from 6e5c0f70-fd56-4280-9d20-71288e488216 with version 4, task: proc
Sep 22 23:15:03.611 INFO upstairs UpstairsConnection { upstairs_id: 6e5c0f70-fd56-4280-9d20-71288e488216, session_id: 2cf90053-4dbd-4cc4-8468-9d35f085e47a, gen: 2 } connected, version 4, task: proc
The guest has requested activation
Sep 22 23:15:03.611 INFO [0] 6e5c0f70-fd56-4280-9d20-71288e488216 (2cf90053-4dbd-4cc4-8468-9d35f085e47a) New New New ds_transition to WaitActive
Sep 22 23:15:03.611 INFO [0] Transition from New to WaitActive
Sep 22 23:15:03.611 INFO [1] 6e5c0f70-fd56-4280-9d20-71288e488216 (2cf90053-4dbd-4cc4-8468-9d35f085e47a) WaitActive New New ds_transition to WaitActive
Sep 22 23:15:03.611 INFO [1] Transition from New to WaitActive
Sep 22 23:15:03.611 INFO [2] 6e5c0f70-fd56-4280-9d20-71288e488216 (2cf90053-4dbd-4cc4-8468-9d35f085e47a) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:15:03.611 INFO [2] Transition from New to WaitActive
Sep 22 23:15:03.611 INFO 6e5c0f70-fd56-4280-9d20-71288e488216 active request set
Sep 22 23:15:03.611 INFO [0] received activate with gen 2
Sep 22 23:15:03.611 INFO [0] client got ds_active_rx, promote! session 2cf90053-4dbd-4cc4-8468-9d35f085e47a
Sep 22 23:15:03.611 INFO [1] received activate with gen 2
Sep 22 23:15:03.611 INFO [1] client got ds_active_rx, promote! session 2cf90053-4dbd-4cc4-8468-9d35f085e47a
Sep 22 23:15:03.611 INFO [2] received activate with gen 2
Sep 22 23:15:03.611 INFO [2] client got ds_active_rx, promote! session 2cf90053-4dbd-4cc4-8468-9d35f085e47a
Sep 22 23:15:03.612 WARN Attempting RW takeover from UpstairsConnection { upstairs_id: 6e5c0f70-fd56-4280-9d20-71288e488216, session_id: f334c95e-b851-4a8a-a731-3fb69e42e934, gen: 1 } to UpstairsConnection { upstairs_id: 6e5c0f70-fd56-4280-9d20-71288e488216, session_id: 2cf90053-4dbd-4cc4-8468-9d35f085e47a, gen: 2 }
Sep 22 23:15:03.612 WARN Signaling to UpstairsConnection { upstairs_id: 6e5c0f70-fd56-4280-9d20-71288e488216, session_id: f334c95e-b851-4a8a-a731-3fb69e42e934, gen: 1 } thread that UpstairsConnection { upstairs_id: 6e5c0f70-fd56-4280-9d20-71288e488216, session_id: 2cf90053-4dbd-4cc4-8468-9d35f085e47a, gen: 2 } is being promoted (read-write)
Sep 22 23:15:03.612 WARN Attempting RW takeover from UpstairsConnection { upstairs_id: 6e5c0f70-fd56-4280-9d20-71288e488216, session_id: f334c95e-b851-4a8a-a731-3fb69e42e934, gen: 1 } to UpstairsConnection { upstairs_id: 6e5c0f70-fd56-4280-9d20-71288e488216, session_id: 2cf90053-4dbd-4cc4-8468-9d35f085e47a, gen: 2 }
Sep 22 23:15:03.612 WARN Signaling to UpstairsConnection { upstairs_id: 6e5c0f70-fd56-4280-9d20-71288e488216, session_id: f334c95e-b851-4a8a-a731-3fb69e42e934, gen: 1 } thread that UpstairsConnection { upstairs_id: 6e5c0f70-fd56-4280-9d20-71288e488216, session_id: 2cf90053-4dbd-4cc4-8468-9d35f085e47a, gen: 2 } is being promoted (read-write)
Sep 22 23:15:03.612 INFO UpstairsConnection { upstairs_id: 6e5c0f70-fd56-4280-9d20-71288e488216, session_id: 2cf90053-4dbd-4cc4-8468-9d35f085e47a, gen: 2 } is now active (read-write)
Sep 22 23:15:03.612 WARN Another upstairs UpstairsConnection { upstairs_id: 6e5c0f70-fd56-4280-9d20-71288e488216, session_id: 2cf90053-4dbd-4cc4-8468-9d35f085e47a, gen: 2 } promoted to active, shutting down connection for UpstairsConnection { upstairs_id: 6e5c0f70-fd56-4280-9d20-71288e488216, session_id: f334c95e-b851-4a8a-a731-3fb69e42e934, gen: 1 }, task: main
Sep 22 23:15:03.612 INFO UpstairsConnection { upstairs_id: 6e5c0f70-fd56-4280-9d20-71288e488216, session_id: 2cf90053-4dbd-4cc4-8468-9d35f085e47a, gen: 2 } is now active (read-write)
Sep 22 23:15:03.612 WARN Another upstairs UpstairsConnection { upstairs_id: 6e5c0f70-fd56-4280-9d20-71288e488216, session_id: 2cf90053-4dbd-4cc4-8468-9d35f085e47a, gen: 2 } promoted to active, shutting down connection for UpstairsConnection { upstairs_id: 6e5c0f70-fd56-4280-9d20-71288e488216, session_id: f334c95e-b851-4a8a-a731-3fb69e42e934, gen: 1 }, task: main
Sep 22 23:15:03.612 INFO UpstairsConnection { upstairs_id: 6e5c0f70-fd56-4280-9d20-71288e488216, session_id: 2cf90053-4dbd-4cc4-8468-9d35f085e47a, gen: 2 } is now active (read-write)
Sep 22 23:15:03.612 INFO connection (127.0.0.1:48323): all done
Sep 22 23:15:03.612 INFO connection (127.0.0.1:59274): all done
Sep 22 23:15:03.613 ERRO [1] 6e5c0f70-fd56-4280-9d20-71288e488216 (f334c95e-b851-4a8a-a731-3fb69e42e934) cmd_loop saw YouAreNoLongerActive 6e5c0f70-fd56-4280-9d20-71288e488216 2cf90053-4dbd-4cc4-8468-9d35f085e47a 2
Sep 22 23:15:03.613 INFO [1] 6e5c0f70-fd56-4280-9d20-71288e488216 (f334c95e-b851-4a8a-a731-3fb69e42e934) Replaced Active Active ds_transition to Disabled
Sep 22 23:15:03.613 INFO [1] Transition from Active to Disabled
Sep 22 23:15:03.613 INFO 6e5c0f70-fd56-4280-9d20-71288e488216 set inactive, session f334c95e-b851-4a8a-a731-3fb69e42e934
Sep 22 23:15:03.613 ERRO 127.0.0.1:64149: proc: Generation number is too low: saw YouAreNoLongerActive with larger gen 2 than ours 1, looper: 1
Sep 22 23:15:03.613 INFO [1] 6e5c0f70-fd56-4280-9d20-71288e488216 Gone missing, transition from Disabled to Disconnected
Sep 22 23:15:03.613 INFO [1] 6e5c0f70-fd56-4280-9d20-71288e488216 connection to 127.0.0.1:64149 closed, looper: 1
Sep 22 23:15:03.613 ERRO [2] 6e5c0f70-fd56-4280-9d20-71288e488216 (f334c95e-b851-4a8a-a731-3fb69e42e934) cmd_loop saw YouAreNoLongerActive 6e5c0f70-fd56-4280-9d20-71288e488216 2cf90053-4dbd-4cc4-8468-9d35f085e47a 2
Sep 22 23:15:03.613 INFO [2] 6e5c0f70-fd56-4280-9d20-71288e488216 (f334c95e-b851-4a8a-a731-3fb69e42e934) Replaced Disconnected Active ds_transition to Disabled
Sep 22 23:15:03.613 INFO [2] Transition from Active to Disabled
Sep 22 23:15:03.613 INFO 6e5c0f70-fd56-4280-9d20-71288e488216 set inactive, session f334c95e-b851-4a8a-a731-3fb69e42e934
Sep 22 23:15:03.613 ERRO 127.0.0.1:58182: proc: Generation number is too low: saw YouAreNoLongerActive with larger gen 2 than ours 1, looper: 2
Sep 22 23:15:03.613 INFO [2] 6e5c0f70-fd56-4280-9d20-71288e488216 Gone missing, transition from Disabled to Disconnected
Sep 22 23:15:03.613 INFO [2] 6e5c0f70-fd56-4280-9d20-71288e488216 connection to 127.0.0.1:58182 closed, looper: 2
Sep 22 23:15:03.613 WARN [1] pm_task rx.recv() is None
Sep 22 23:15:03.613 INFO [1] 127.0.0.1:64149 task reports connection:false
Sep 22 23:15:03.613 INFO 6e5c0f70-fd56-4280-9d20-71288e488216 Replaced Disconnected Disconnected
Sep 22 23:15:03.613 INFO [1] 127.0.0.1:64149 task reports offline
Sep 22 23:15:03.613 INFO [2] 127.0.0.1:58182 task reports connection:false
Sep 22 23:15:03.613 INFO 6e5c0f70-fd56-4280-9d20-71288e488216 Replaced Disconnected Disconnected
Sep 22 23:15:03.613 INFO [2] 127.0.0.1:58182 task reports offline
Sep 22 23:15:03.613 WARN [2] pm_task rx.recv() is None
Sep 22 23:15:03.614 INFO [0] downstairs client at 127.0.0.1:50216 has UUID 28f50308-941f-4d7f-bd0a-39a00206f2e7
Sep 22 23:15:03.614 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 512, shift: 9 }, extent_count: 188, uuid: 28f50308-941f-4d7f-bd0a-39a00206f2e7, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:15:03.614 INFO 6e5c0f70-fd56-4280-9d20-71288e488216 WaitActive WaitActive WaitActive
Sep 22 23:15:03.614 INFO [1] downstairs client at 127.0.0.1:64149 has UUID df366b3b-ce8f-4a81-b993-9efbc4225b81
Sep 22 23:15:03.614 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 512, shift: 9 }, extent_count: 188, uuid: df366b3b-ce8f-4a81-b993-9efbc4225b81, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:15:03.614 INFO 6e5c0f70-fd56-4280-9d20-71288e488216 WaitActive WaitActive WaitActive
Sep 22 23:15:03.614 INFO [2] downstairs client at 127.0.0.1:58182 has UUID 1361b8e1-8898-44ce-892b-e72b28f2a9df
Sep 22 23:15:03.614 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 512, shift: 9 }, extent_count: 188, uuid: 1361b8e1-8898-44ce-892b-e72b28f2a9df, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:15:03.614 INFO 6e5c0f70-fd56-4280-9d20-71288e488216 WaitActive WaitActive WaitActive
Sep 22 23:15:03.623 INFO Current flush_numbers [0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:15:03.625 INFO Current flush_numbers [0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
Sep 22 23:15:03.626 INFO Current flush_numbers [0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
Sep 22 23:15:03.639 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:15:03.640 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:15:03.641 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:15:03.641 INFO [0] 6e5c0f70-fd56-4280-9d20-71288e488216 (2cf90053-4dbd-4cc4-8468-9d35f085e47a) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:15:03.641 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:15:03.641 WARN [0] new RM replaced this: None
Sep 22 23:15:03.641 INFO [0] Starts reconcile loop
Sep 22 23:15:03.641 INFO [1] 6e5c0f70-fd56-4280-9d20-71288e488216 (2cf90053-4dbd-4cc4-8468-9d35f085e47a) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:15:03.641 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:15:03.641 WARN [1] new RM replaced this: None
Sep 22 23:15:03.641 INFO [1] Starts reconcile loop
Sep 22 23:15:03.642 INFO [2] 6e5c0f70-fd56-4280-9d20-71288e488216 (2cf90053-4dbd-4cc4-8468-9d35f085e47a) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:15:03.642 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:15:03.642 WARN [2] new RM replaced this: None
Sep 22 23:15:03.642 INFO [2] Starts reconcile loop
Sep 22 23:15:03.642 INFO [0] 127.0.0.1:50216 task reports connection:true
Sep 22 23:15:03.642 INFO 6e5c0f70-fd56-4280-9d20-71288e488216 WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:15:03.642 INFO [0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:15:03.642 INFO [0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:15:03.642 INFO [0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]
Sep 22 23:15:03.642 INFO [1]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
Sep 22 23:15:03.642 INFO [1]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
Sep 22 23:15:03.642 INFO [1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]
Sep 22 23:15:03.642 INFO [2]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
Sep 22 23:15:03.642 INFO [2]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
Sep 22 23:15:03.642 INFO [2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]
Sep 22 23:15:03.642 INFO Max found gen is 2
Sep 22 23:15:03.642 INFO Generation requested: 2 >= found:2
Sep 22 23:15:03.642 INFO Next flush: 2
Sep 22 23:15:03.642 INFO Extent 0 has flush number mismatch, : mend
Sep 22 23:15:03.642 INFO First source client ID for extent 0, mrl: flush_mismatch, : mend
Sep 22 23:15:03.642 INFO extent:0  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.642 INFO extent:0  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.642 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.642 INFO extent:0  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.642 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.642 INFO find dest for source 1 for extent at index 0, mrl: flush_mismatch, : mend
Sep 22 23:15:03.642 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.642 INFO Extent 1 has flush number mismatch, : mend
Sep 22 23:15:03.642 INFO First source client ID for extent 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.642 INFO extent:1  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.642 INFO extent:1  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.642 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.642 INFO extent:1  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.642 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.642 INFO find dest for source 1 for extent at index 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.642 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.642 INFO Extent 2 has flush number mismatch, : mend
Sep 22 23:15:03.642 INFO First source client ID for extent 2, mrl: flush_mismatch, : mend
Sep 22 23:15:03.642 INFO extent:2  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.642 INFO extent:2  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.642 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.642 INFO extent:2  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.642 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.642 INFO find dest for source 1 for extent at index 2, mrl: flush_mismatch, : mend
Sep 22 23:15:03.642 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.642 INFO Extent 3 has flush number mismatch, : mend
Sep 22 23:15:03.642 INFO First source client ID for extent 3, mrl: flush_mismatch, : mend
Sep 22 23:15:03.642 INFO extent:3  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.642 INFO extent:3  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.642 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.642 INFO extent:3  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.642 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.642 INFO find dest for source 1 for extent at index 3, mrl: flush_mismatch, : mend
Sep 22 23:15:03.642 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.642 INFO Extent 4 has flush number mismatch, : mend
Sep 22 23:15:03.643 INFO First source client ID for extent 4, mrl: flush_mismatch, : mend
Sep 22 23:15:03.643 INFO extent:4  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.643 INFO extent:4  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.643 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.643 INFO extent:4  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.643 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.643 INFO find dest for source 1 for extent at index 4, mrl: flush_mismatch, : mend
Sep 22 23:15:03.643 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.643 INFO Extent 5 has flush number mismatch, : mend
Sep 22 23:15:03.643 INFO First source client ID for extent 5, mrl: flush_mismatch, : mend
Sep 22 23:15:03.643 INFO extent:5  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.643 INFO extent:5  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.643 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.643 INFO extent:5  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.643 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.643 INFO find dest for source 1 for extent at index 5, mrl: flush_mismatch, : mend
Sep 22 23:15:03.643 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.643 INFO Extent 6 has flush number mismatch, : mend
Sep 22 23:15:03.643 INFO First source client ID for extent 6, mrl: flush_mismatch, : mend
Sep 22 23:15:03.643 INFO extent:6  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.643 INFO extent:6  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.643 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.643 INFO extent:6  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.643 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.643 INFO find dest for source 1 for extent at index 6, mrl: flush_mismatch, : mend
Sep 22 23:15:03.643 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.643 INFO Extent 7 has flush number mismatch, : mend
Sep 22 23:15:03.643 INFO First source client ID for extent 7, mrl: flush_mismatch, : mend
Sep 22 23:15:03.643 INFO extent:7  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.643 INFO extent:7  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.643 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.643 INFO extent:7  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.643 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.643 INFO find dest for source 1 for extent at index 7, mrl: flush_mismatch, : mend
Sep 22 23:15:03.643 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.643 INFO Extent 8 has flush number mismatch, : mend
Sep 22 23:15:03.643 INFO First source client ID for extent 8, mrl: flush_mismatch, : mend
Sep 22 23:15:03.643 INFO extent:8  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.643 INFO extent:8  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.643 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.643 INFO extent:8  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.643 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.643 INFO find dest for source 1 for extent at index 8, mrl: flush_mismatch, : mend
Sep 22 23:15:03.643 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.643 INFO Extent 9 has flush number mismatch, : mend
Sep 22 23:15:03.643 INFO First source client ID for extent 9, mrl: flush_mismatch, : mend
Sep 22 23:15:03.643 INFO extent:9  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.643 INFO extent:9  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.643 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.643 INFO extent:9  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.643 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.643 INFO find dest for source 1 for extent at index 9, mrl: flush_mismatch, : mend
Sep 22 23:15:03.643 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.643 INFO Extent 10 has flush number mismatch, : mend
Sep 22 23:15:03.643 INFO First source client ID for extent 10, mrl: flush_mismatch, : mend
Sep 22 23:15:03.643 INFO extent:10  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.643 INFO extent:10  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.643 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.643 INFO extent:10  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.643 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.643 INFO find dest for source 1 for extent at index 10, mrl: flush_mismatch, : mend
Sep 22 23:15:03.643 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.643 INFO Extent 11 has flush number mismatch, : mend
Sep 22 23:15:03.643 INFO First source client ID for extent 11, mrl: flush_mismatch, : mend
Sep 22 23:15:03.643 INFO extent:11  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.643 INFO extent:11  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.643 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.643 INFO extent:11  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.643 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.643 INFO find dest for source 1 for extent at index 11, mrl: flush_mismatch, : mend
Sep 22 23:15:03.643 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.643 INFO Extent 12 has flush number mismatch, : mend
Sep 22 23:15:03.643 INFO First source client ID for extent 12, mrl: flush_mismatch, : mend
Sep 22 23:15:03.643 INFO extent:12  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.643 INFO extent:12  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.643 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.643 INFO extent:12  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.643 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.643 INFO find dest for source 1 for extent at index 12, mrl: flush_mismatch, : mend
Sep 22 23:15:03.643 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.643 INFO Extent 13 has flush number mismatch, : mend
Sep 22 23:15:03.643 INFO First source client ID for extent 13, mrl: flush_mismatch, : mend
Sep 22 23:15:03.643 INFO extent:13  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.643 INFO extent:13  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.643 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.643 INFO extent:13  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.643 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.643 INFO find dest for source 1 for extent at index 13, mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO Extent 14 has flush number mismatch, : mend
Sep 22 23:15:03.644 INFO First source client ID for extent 14, mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO extent:14  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO extent:14  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO extent:14  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO find dest for source 1 for extent at index 14, mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO Extent 15 has flush number mismatch, : mend
Sep 22 23:15:03.644 INFO First source client ID for extent 15, mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO extent:15  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO extent:15  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO extent:15  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO find dest for source 1 for extent at index 15, mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO Extent 16 has flush number mismatch, : mend
Sep 22 23:15:03.644 INFO First source client ID for extent 16, mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO extent:16  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO extent:16  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO extent:16  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO find dest for source 1 for extent at index 16, mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO Extent 17 has flush number mismatch, : mend
Sep 22 23:15:03.644 INFO First source client ID for extent 17, mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO extent:17  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO extent:17  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO extent:17  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO find dest for source 1 for extent at index 17, mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO Extent 18 has flush number mismatch, : mend
Sep 22 23:15:03.644 INFO First source client ID for extent 18, mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO extent:18  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO extent:18  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO extent:18  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO find dest for source 1 for extent at index 18, mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO Extent 19 has flush number mismatch, : mend
Sep 22 23:15:03.644 INFO First source client ID for extent 19, mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO extent:19  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO extent:19  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO extent:19  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO find dest for source 1 for extent at index 19, mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO Extent 20 has flush number mismatch, : mend
Sep 22 23:15:03.644 INFO First source client ID for extent 20, mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO extent:20  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO extent:20  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO extent:20  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO find dest for source 1 for extent at index 20, mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO Extent 21 has flush number mismatch, : mend
Sep 22 23:15:03.644 INFO First source client ID for extent 21, mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO extent:21  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO extent:21  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO extent:21  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO find dest for source 1 for extent at index 21, mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO Extent 22 has flush number mismatch, : mend
Sep 22 23:15:03.644 INFO First source client ID for extent 22, mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO extent:22  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO extent:22  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO extent:22  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO find dest for source 1 for extent at index 22, mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO Extent 23 has flush number mismatch, : mend
Sep 22 23:15:03.644 INFO First source client ID for extent 23, mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO extent:23  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO extent:23  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO extent:23  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.644 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO find dest for source 1 for extent at index 23, mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO Extent 24 has flush number mismatch, : mend
Sep 22 23:15:03.645 INFO First source client ID for extent 24, mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO extent:24  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO extent:24  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO extent:24  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO find dest for source 1 for extent at index 24, mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO Extent 25 has flush number mismatch, : mend
Sep 22 23:15:03.645 INFO First source client ID for extent 25, mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO extent:25  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO extent:25  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO extent:25  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO find dest for source 1 for extent at index 25, mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO Extent 26 has flush number mismatch, : mend
Sep 22 23:15:03.645 INFO First source client ID for extent 26, mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO extent:26  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO extent:26  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO extent:26  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO find dest for source 1 for extent at index 26, mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO Extent 27 has flush number mismatch, : mend
Sep 22 23:15:03.645 INFO First source client ID for extent 27, mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO extent:27  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO extent:27  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO extent:27  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO find dest for source 1 for extent at index 27, mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO Extent 28 has flush number mismatch, : mend
Sep 22 23:15:03.645 INFO First source client ID for extent 28, mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO extent:28  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO extent:28  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO extent:28  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO find dest for source 1 for extent at index 28, mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO Extent 29 has flush number mismatch, : mend
Sep 22 23:15:03.645 INFO First source client ID for extent 29, mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO extent:29  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO extent:29  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO extent:29  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO find dest for source 1 for extent at index 29, mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO Extent 30 has flush number mismatch, : mend
Sep 22 23:15:03.645 INFO First source client ID for extent 30, mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO extent:30  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO extent:30  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO extent:30  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO find dest for source 1 for extent at index 30, mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO Extent 31 has flush number mismatch, : mend
Sep 22 23:15:03.645 INFO First source client ID for extent 31, mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO extent:31  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO extent:31  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO extent:31  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO find dest for source 1 for extent at index 31, mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO Extent 32 has flush number mismatch, : mend
Sep 22 23:15:03.645 INFO First source client ID for extent 32, mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO extent:32  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO extent:32  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO extent:32  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO find dest for source 1 for extent at index 32, mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO Extent 33 has flush number mismatch, : mend
Sep 22 23:15:03.645 INFO First source client ID for extent 33, mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO extent:33  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO extent:33  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.645 INFO extent:33  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO find dest for source 1 for extent at index 33, mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO Extent 34 has flush number mismatch, : mend
Sep 22 23:15:03.646 INFO First source client ID for extent 34, mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO extent:34  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO extent:34  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO extent:34  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO find dest for source 1 for extent at index 34, mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO Extent 35 has flush number mismatch, : mend
Sep 22 23:15:03.646 INFO First source client ID for extent 35, mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO extent:35  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO extent:35  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO extent:35  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO find dest for source 1 for extent at index 35, mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO Extent 36 has flush number mismatch, : mend
Sep 22 23:15:03.646 INFO First source client ID for extent 36, mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO extent:36  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO extent:36  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO extent:36  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO find dest for source 1 for extent at index 36, mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO Extent 37 has flush number mismatch, : mend
Sep 22 23:15:03.646 INFO First source client ID for extent 37, mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO extent:37  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO extent:37  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO extent:37  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO find dest for source 1 for extent at index 37, mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO Extent 38 has flush number mismatch, : mend
Sep 22 23:15:03.646 INFO First source client ID for extent 38, mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO extent:38  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO extent:38  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO extent:38  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO find dest for source 1 for extent at index 38, mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO Extent 39 has flush number mismatch, : mend
Sep 22 23:15:03.646 INFO First source client ID for extent 39, mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO extent:39  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO extent:39  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO extent:39  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO find dest for source 1 for extent at index 39, mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO Extent 40 has flush number mismatch, : mend
Sep 22 23:15:03.646 INFO First source client ID for extent 40, mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO extent:40  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO extent:40  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO extent:40  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO find dest for source 1 for extent at index 40, mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO Extent 41 has flush number mismatch, : mend
Sep 22 23:15:03.646 INFO First source client ID for extent 41, mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO extent:41  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO extent:41  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO extent:41  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO find dest for source 1 for extent at index 41, mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO Extent 42 has flush number mismatch, : mend
Sep 22 23:15:03.646 INFO First source client ID for extent 42, mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO extent:42  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO extent:42  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO extent:42  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO find dest for source 1 for extent at index 42, mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO Extent 43 has flush number mismatch, : mend
Sep 22 23:15:03.646 INFO First source client ID for extent 43, mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO extent:43  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO extent:43  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO extent:43  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.646 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO find dest for source 1 for extent at index 43, mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO Extent 44 has flush number mismatch, : mend
Sep 22 23:15:03.647 INFO First source client ID for extent 44, mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO extent:44  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO extent:44  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO extent:44  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO find dest for source 1 for extent at index 44, mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO Extent 45 has flush number mismatch, : mend
Sep 22 23:15:03.647 INFO First source client ID for extent 45, mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO extent:45  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO extent:45  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO extent:45  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO find dest for source 1 for extent at index 45, mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO Extent 46 has flush number mismatch, : mend
Sep 22 23:15:03.647 INFO First source client ID for extent 46, mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO extent:46  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO extent:46  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO extent:46  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO find dest for source 1 for extent at index 46, mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO Extent 47 has flush number mismatch, : mend
Sep 22 23:15:03.647 INFO First source client ID for extent 47, mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO extent:47  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO extent:47  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO extent:47  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO find dest for source 1 for extent at index 47, mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO Extent 48 has flush number mismatch, : mend
Sep 22 23:15:03.647 INFO First source client ID for extent 48, mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO extent:48  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO extent:48  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO extent:48  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO find dest for source 1 for extent at index 48, mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO Extent 49 has flush number mismatch, : mend
Sep 22 23:15:03.647 INFO First source client ID for extent 49, mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO extent:49  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO extent:49  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO extent:49  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO find dest for source 1 for extent at index 49, mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO Extent 50 has flush number mismatch, : mend
Sep 22 23:15:03.647 INFO First source client ID for extent 50, mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO extent:50  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO extent:50  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO extent:50  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO find dest for source 1 for extent at index 50, mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO Extent 51 has flush number mismatch, : mend
Sep 22 23:15:03.647 INFO First source client ID for extent 51, mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO extent:51  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO extent:51  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO extent:51  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO find dest for source 1 for extent at index 51, mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO Extent 52 has flush number mismatch, : mend
Sep 22 23:15:03.647 INFO First source client ID for extent 52, mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO extent:52  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO extent:52  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO extent:52  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO find dest for source 1 for extent at index 52, mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO Extent 53 has flush number mismatch, : mend
Sep 22 23:15:03.647 INFO First source client ID for extent 53, mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO extent:53  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO extent:53  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO extent:53  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.647 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.648 INFO find dest for source 1 for extent at index 53, mrl: flush_mismatch, : mend
Sep 22 23:15:03.648 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.648 INFO Extent 54 has flush number mismatch, : mend
Sep 22 23:15:03.648 INFO First source client ID for extent 54, mrl: flush_mismatch, : mend
Sep 22 23:15:03.648 INFO extent:54  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.648 INFO extent:54  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.648 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.648 INFO extent:54  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.648 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.648 INFO find dest for source 1 for extent at index 54, mrl: flush_mismatch, : mend
Sep 22 23:15:03.648 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.648 INFO Extent 55 has flush number mismatch, : mend
Sep 22 23:15:03.648 INFO First source client ID for extent 55, mrl: flush_mismatch, : mend
Sep 22 23:15:03.648 INFO extent:55  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.648 INFO extent:55  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.648 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.648 INFO extent:55  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.648 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.648 INFO find dest for source 1 for extent at index 55, mrl: flush_mismatch, : mend
Sep 22 23:15:03.648 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.648 INFO Extent 56 has flush number mismatch, : mend
Sep 22 23:15:03.648 INFO First source client ID for extent 56, mrl: flush_mismatch, : mend
Sep 22 23:15:03.648 INFO extent:56  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.648 INFO extent:56  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.648 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.648 INFO extent:56  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.648 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.648 INFO find dest for source 1 for extent at index 56, mrl: flush_mismatch, : mend
Sep 22 23:15:03.648 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.648 INFO Extent 57 has flush number mismatch, : mend
Sep 22 23:15:03.648 INFO First source client ID for extent 57, mrl: flush_mismatch, : mend
Sep 22 23:15:03.648 INFO extent:57  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.648 INFO extent:57  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.648 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.648 INFO extent:57  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.648 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.648 INFO find dest for source 1 for extent at index 57, mrl: flush_mismatch, : mend
Sep 22 23:15:03.648 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.648 INFO Extent 58 has flush number mismatch, : mend
Sep 22 23:15:03.648 INFO First source client ID for extent 58, mrl: flush_mismatch, : mend
Sep 22 23:15:03.648 INFO extent:58  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.648 INFO extent:58  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.648 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.648 INFO extent:58  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.648 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.648 INFO find dest for source 1 for extent at index 58, mrl: flush_mismatch, : mend
Sep 22 23:15:03.648 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.648 INFO Extent 59 has flush number mismatch, : mend
Sep 22 23:15:03.648 INFO First source client ID for extent 59, mrl: flush_mismatch, : mend
Sep 22 23:15:03.648 INFO extent:59  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.648 INFO extent:59  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.648 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.648 INFO extent:59  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.648 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.648 INFO find dest for source 1 for extent at index 59, mrl: flush_mismatch, : mend
Sep 22 23:15:03.648 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.648 INFO Extent 60 has flush number mismatch, : mend
Sep 22 23:15:03.648 INFO First source client ID for extent 60, mrl: flush_mismatch, : mend
Sep 22 23:15:03.648 INFO extent:60  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.648 INFO extent:60  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.648 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.648 INFO extent:60  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.648 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.648 INFO find dest for source 1 for extent at index 60, mrl: flush_mismatch, : mend
Sep 22 23:15:03.648 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.648 INFO Extent 61 has flush number mismatch, : mend
Sep 22 23:15:03.648 INFO First source client ID for extent 61, mrl: flush_mismatch, : mend
Sep 22 23:15:03.648 INFO extent:61  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.648 INFO extent:61  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.648 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.648 INFO extent:61  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.648 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.648 INFO find dest for source 1 for extent at index 61, mrl: flush_mismatch, : mend
Sep 22 23:15:03.648 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.648 INFO Extent 62 has flush number mismatch, : mend
Sep 22 23:15:03.648 INFO First source client ID for extent 62, mrl: flush_mismatch, : mend
Sep 22 23:15:03.648 INFO extent:62  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.648 INFO extent:62  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.648 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.648 INFO extent:62  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.648 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.648 INFO find dest for source 1 for extent at index 62, mrl: flush_mismatch, : mend
Sep 22 23:15:03.648 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.648 INFO Extent 63 has flush number mismatch, : mend
Sep 22 23:15:03.648 INFO First source client ID for extent 63, mrl: flush_mismatch, : mend
Sep 22 23:15:03.648 INFO extent:63  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.648 INFO extent:63  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO extent:63  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO find dest for source 1 for extent at index 63, mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO Extent 64 has flush number mismatch, : mend
Sep 22 23:15:03.649 INFO First source client ID for extent 64, mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO extent:64  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO extent:64  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO extent:64  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO find dest for source 1 for extent at index 64, mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO Extent 65 has flush number mismatch, : mend
Sep 22 23:15:03.649 INFO First source client ID for extent 65, mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO extent:65  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO extent:65  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO extent:65  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO find dest for source 1 for extent at index 65, mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO Extent 66 has flush number mismatch, : mend
Sep 22 23:15:03.649 INFO First source client ID for extent 66, mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO extent:66  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO extent:66  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO extent:66  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO find dest for source 1 for extent at index 66, mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO Extent 67 has flush number mismatch, : mend
Sep 22 23:15:03.649 INFO First source client ID for extent 67, mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO extent:67  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO extent:67  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO extent:67  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO find dest for source 1 for extent at index 67, mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO Extent 68 has flush number mismatch, : mend
Sep 22 23:15:03.649 INFO First source client ID for extent 68, mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO extent:68  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO extent:68  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO extent:68  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO find dest for source 1 for extent at index 68, mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO Extent 69 has flush number mismatch, : mend
Sep 22 23:15:03.649 INFO First source client ID for extent 69, mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO extent:69  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO extent:69  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO extent:69  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO find dest for source 1 for extent at index 69, mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO Extent 70 has flush number mismatch, : mend
Sep 22 23:15:03.649 INFO First source client ID for extent 70, mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO extent:70  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO extent:70  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO extent:70  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO find dest for source 1 for extent at index 70, mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO Extent 71 has flush number mismatch, : mend
Sep 22 23:15:03.649 INFO First source client ID for extent 71, mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO extent:71  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO extent:71  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO extent:71  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO find dest for source 1 for extent at index 71, mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO Extent 72 has flush number mismatch, : mend
Sep 22 23:15:03.649 INFO First source client ID for extent 72, mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO extent:72  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO extent:72  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO extent:72  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO find dest for source 1 for extent at index 72, mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO Extent 73 has flush number mismatch, : mend
Sep 22 23:15:03.649 INFO First source client ID for extent 73, mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO extent:73  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.649 INFO extent:73  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO extent:73  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO find dest for source 1 for extent at index 73, mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO Extent 74 has flush number mismatch, : mend
Sep 22 23:15:03.650 INFO First source client ID for extent 74, mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO extent:74  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO extent:74  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO extent:74  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO find dest for source 1 for extent at index 74, mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO Extent 75 has flush number mismatch, : mend
Sep 22 23:15:03.650 INFO First source client ID for extent 75, mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO extent:75  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO extent:75  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO extent:75  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO find dest for source 1 for extent at index 75, mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO Extent 76 has flush number mismatch, : mend
Sep 22 23:15:03.650 INFO First source client ID for extent 76, mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO extent:76  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO extent:76  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO extent:76  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO find dest for source 1 for extent at index 76, mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO Extent 77 has flush number mismatch, : mend
Sep 22 23:15:03.650 INFO First source client ID for extent 77, mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO extent:77  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO extent:77  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO extent:77  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO find dest for source 1 for extent at index 77, mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO Extent 78 has flush number mismatch, : mend
Sep 22 23:15:03.650 INFO First source client ID for extent 78, mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO extent:78  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO extent:78  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO extent:78  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO find dest for source 1 for extent at index 78, mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO Extent 79 has flush number mismatch, : mend
Sep 22 23:15:03.650 INFO First source client ID for extent 79, mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO extent:79  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO extent:79  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO extent:79  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO find dest for source 1 for extent at index 79, mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO Extent 80 has flush number mismatch, : mend
Sep 22 23:15:03.650 INFO First source client ID for extent 80, mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO extent:80  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO extent:80  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO extent:80  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO find dest for source 1 for extent at index 80, mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO Extent 81 has flush number mismatch, : mend
Sep 22 23:15:03.650 INFO First source client ID for extent 81, mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO extent:81  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO extent:81  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO extent:81  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO find dest for source 1 for extent at index 81, mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO Extent 82 has flush number mismatch, : mend
Sep 22 23:15:03.650 INFO First source client ID for extent 82, mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO extent:82  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO extent:82  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO extent:82  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO find dest for source 1 for extent at index 82, mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO Extent 83 has flush number mismatch, : mend
Sep 22 23:15:03.650 INFO First source client ID for extent 83, mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO extent:83  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.650 INFO extent:83  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO extent:83  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO find dest for source 1 for extent at index 83, mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO Extent 84 has flush number mismatch, : mend
Sep 22 23:15:03.651 INFO First source client ID for extent 84, mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO extent:84  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO extent:84  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO extent:84  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO find dest for source 1 for extent at index 84, mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO Extent 85 has flush number mismatch, : mend
Sep 22 23:15:03.651 INFO First source client ID for extent 85, mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO extent:85  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO extent:85  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO extent:85  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO find dest for source 1 for extent at index 85, mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO Extent 86 has flush number mismatch, : mend
Sep 22 23:15:03.651 INFO First source client ID for extent 86, mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO extent:86  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO extent:86  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO extent:86  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO find dest for source 1 for extent at index 86, mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO Extent 87 has flush number mismatch, : mend
Sep 22 23:15:03.651 INFO First source client ID for extent 87, mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO extent:87  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO extent:87  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO extent:87  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO find dest for source 1 for extent at index 87, mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO Extent 88 has flush number mismatch, : mend
Sep 22 23:15:03.651 INFO First source client ID for extent 88, mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO extent:88  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO extent:88  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO extent:88  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO find dest for source 1 for extent at index 88, mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO Extent 89 has flush number mismatch, : mend
Sep 22 23:15:03.651 INFO First source client ID for extent 89, mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO extent:89  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO extent:89  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO extent:89  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO find dest for source 1 for extent at index 89, mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO Extent 90 has flush number mismatch, : mend
Sep 22 23:15:03.651 INFO First source client ID for extent 90, mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO extent:90  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO extent:90  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO extent:90  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO find dest for source 1 for extent at index 90, mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO Extent 91 has flush number mismatch, : mend
Sep 22 23:15:03.651 INFO First source client ID for extent 91, mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO extent:91  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO extent:91  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO extent:91  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO find dest for source 1 for extent at index 91, mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO Extent 92 has flush number mismatch, : mend
Sep 22 23:15:03.651 INFO First source client ID for extent 92, mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO extent:92  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO extent:92  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO extent:92  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO find dest for source 1 for extent at index 92, mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO Extent 93 has flush number mismatch, : mend
Sep 22 23:15:03.651 INFO First source client ID for extent 93, mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO extent:93  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.651 INFO extent:93  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO extent:93  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO find dest for source 1 for extent at index 93, mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO Extent 94 has flush number mismatch, : mend
Sep 22 23:15:03.652 INFO First source client ID for extent 94, mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO extent:94  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO extent:94  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO extent:94  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO find dest for source 1 for extent at index 94, mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO Extent 95 has flush number mismatch, : mend
Sep 22 23:15:03.652 INFO First source client ID for extent 95, mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO extent:95  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO extent:95  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO extent:95  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO find dest for source 1 for extent at index 95, mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO Extent 96 has flush number mismatch, : mend
Sep 22 23:15:03.652 INFO First source client ID for extent 96, mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO extent:96  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO extent:96  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO extent:96  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO find dest for source 1 for extent at index 96, mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO Extent 97 has flush number mismatch, : mend
Sep 22 23:15:03.652 INFO First source client ID for extent 97, mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO extent:97  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO extent:97  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO extent:97  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO find dest for source 1 for extent at index 97, mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO Extent 98 has flush number mismatch, : mend
Sep 22 23:15:03.652 INFO First source client ID for extent 98, mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO extent:98  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO extent:98  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO extent:98  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO find dest for source 1 for extent at index 98, mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO Extent 99 has flush number mismatch, : mend
Sep 22 23:15:03.652 INFO First source client ID for extent 99, mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO extent:99  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO extent:99  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO extent:99  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO find dest for source 1 for extent at index 99, mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO Extent 100 has flush number mismatch, : mend
Sep 22 23:15:03.652 INFO First source client ID for extent 100, mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO extent:100  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO extent:100  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO extent:100  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO find dest for source 1 for extent at index 100, mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO Extent 101 has flush number mismatch, : mend
Sep 22 23:15:03.652 INFO First source client ID for extent 101, mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO extent:101  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO extent:101  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO extent:101  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO find dest for source 1 for extent at index 101, mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO Extent 102 has flush number mismatch, : mend
Sep 22 23:15:03.652 INFO First source client ID for extent 102, mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO extent:102  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO extent:102  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO extent:102  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO find dest for source 1 for extent at index 102, mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO Extent 103 has flush number mismatch, : mend
Sep 22 23:15:03.652 INFO First source client ID for extent 103, mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO extent:103  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.652 INFO extent:103  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.653 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.653 INFO extent:103  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.653 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.653 INFO find dest for source 1 for extent at index 103, mrl: flush_mismatch, : mend
Sep 22 23:15:03.653 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.653 INFO Extent 104 has flush number mismatch, : mend
Sep 22 23:15:03.653 INFO First source client ID for extent 104, mrl: flush_mismatch, : mend
Sep 22 23:15:03.653 INFO extent:104  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.653 INFO extent:104  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.653 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.653 INFO extent:104  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.653 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.653 INFO find dest for source 1 for extent at index 104, mrl: flush_mismatch, : mend
Sep 22 23:15:03.653 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.653 INFO Extent 105 has flush number mismatch, : mend
Sep 22 23:15:03.653 INFO First source client ID for extent 105, mrl: flush_mismatch, : mend
Sep 22 23:15:03.653 INFO extent:105  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.653 INFO extent:105  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.653 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.653 INFO extent:105  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.653 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.653 INFO find dest for source 1 for extent at index 105, mrl: flush_mismatch, : mend
Sep 22 23:15:03.653 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.653 INFO Extent 106 has flush number mismatch, : mend
Sep 22 23:15:03.653 INFO First source client ID for extent 106, mrl: flush_mismatch, : mend
Sep 22 23:15:03.653 INFO extent:106  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.653 INFO extent:106  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.653 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.653 INFO extent:106  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.653 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.653 INFO find dest for source 1 for extent at index 106, mrl: flush_mismatch, : mend
Sep 22 23:15:03.653 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.653 INFO Extent 107 has flush number mismatch, : mend
Sep 22 23:15:03.653 INFO First source client ID for extent 107, mrl: flush_mismatch, : mend
Sep 22 23:15:03.653 INFO extent:107  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.653 INFO extent:107  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.653 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.653 INFO extent:107  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.653 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.653 INFO find dest for source 1 for extent at index 107, mrl: flush_mismatch, : mend
Sep 22 23:15:03.653 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.653 INFO Extent 108 has flush number mismatch, : mend
Sep 22 23:15:03.653 INFO First source client ID for extent 108, mrl: flush_mismatch, : mend
Sep 22 23:15:03.653 INFO extent:108  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.653 INFO extent:108  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.653 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.653 INFO extent:108  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.653 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.653 INFO find dest for source 1 for extent at index 108, mrl: flush_mismatch, : mend
Sep 22 23:15:03.653 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.653 INFO Extent 109 has flush number mismatch, : mend
Sep 22 23:15:03.653 INFO First source client ID for extent 109, mrl: flush_mismatch, : mend
Sep 22 23:15:03.653 INFO extent:109  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.653 INFO extent:109  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.653 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.653 INFO extent:109  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.653 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.653 INFO find dest for source 1 for extent at index 109, mrl: flush_mismatch, : mend
Sep 22 23:15:03.653 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.653 INFO Extent 110 has flush number mismatch, : mend
Sep 22 23:15:03.653 INFO First source client ID for extent 110, mrl: flush_mismatch, : mend
Sep 22 23:15:03.653 INFO extent:110  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.653 INFO extent:110  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.653 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.653 INFO extent:110  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.653 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.653 INFO find dest for source 1 for extent at index 110, mrl: flush_mismatch, : mend
Sep 22 23:15:03.653 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.653 INFO Extent 111 has flush number mismatch, : mend
Sep 22 23:15:03.653 INFO First source client ID for extent 111, mrl: flush_mismatch, : mend
Sep 22 23:15:03.653 INFO extent:111  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.653 INFO extent:111  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.653 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.653 INFO extent:111  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.653 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.653 INFO find dest for source 1 for extent at index 111, mrl: flush_mismatch, : mend
Sep 22 23:15:03.653 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.653 INFO Extent 112 has flush number mismatch, : mend
Sep 22 23:15:03.653 INFO First source client ID for extent 112, mrl: flush_mismatch, : mend
Sep 22 23:15:03.653 INFO extent:112  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.653 INFO extent:112  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.653 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.653 INFO extent:112  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.653 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.653 INFO find dest for source 1 for extent at index 112, mrl: flush_mismatch, : mend
Sep 22 23:15:03.653 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO Extent 113 has flush number mismatch, : mend
Sep 22 23:15:03.654 INFO First source client ID for extent 113, mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO extent:113  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO extent:113  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO extent:113  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO find dest for source 1 for extent at index 113, mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO Extent 114 has flush number mismatch, : mend
Sep 22 23:15:03.654 INFO First source client ID for extent 114, mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO extent:114  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO extent:114  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO extent:114  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO find dest for source 1 for extent at index 114, mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO Extent 115 has flush number mismatch, : mend
Sep 22 23:15:03.654 INFO First source client ID for extent 115, mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO extent:115  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO extent:115  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO extent:115  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO find dest for source 1 for extent at index 115, mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO Extent 116 has flush number mismatch, : mend
Sep 22 23:15:03.654 INFO First source client ID for extent 116, mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO extent:116  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO extent:116  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO extent:116  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO find dest for source 1 for extent at index 116, mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO Extent 117 has flush number mismatch, : mend
Sep 22 23:15:03.654 INFO First source client ID for extent 117, mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO extent:117  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO extent:117  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO extent:117  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO find dest for source 1 for extent at index 117, mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO Extent 118 has flush number mismatch, : mend
Sep 22 23:15:03.654 INFO First source client ID for extent 118, mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO extent:118  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO extent:118  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO extent:118  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO find dest for source 1 for extent at index 118, mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO Extent 119 has flush number mismatch, : mend
Sep 22 23:15:03.654 INFO First source client ID for extent 119, mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO extent:119  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO extent:119  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO extent:119  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO find dest for source 1 for extent at index 119, mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO Extent 120 has flush number mismatch, : mend
Sep 22 23:15:03.654 INFO First source client ID for extent 120, mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO extent:120  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO extent:120  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO extent:120  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO find dest for source 1 for extent at index 120, mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO Extent 121 has flush number mismatch, : mend
Sep 22 23:15:03.654 INFO First source client ID for extent 121, mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO extent:121  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO extent:121  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO extent:121  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO find dest for source 1 for extent at index 121, mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO Extent 122 has flush number mismatch, : mend
Sep 22 23:15:03.654 INFO First source client ID for extent 122, mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO extent:122  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO extent:122  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO extent:122  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO find dest for source 1 for extent at index 122, mrl: flush_mismatch, : mend
Sep 22 23:15:03.654 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO Extent 123 has flush number mismatch, : mend
Sep 22 23:15:03.655 INFO First source client ID for extent 123, mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO extent:123  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO extent:123  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO extent:123  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO find dest for source 1 for extent at index 123, mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO Extent 124 has flush number mismatch, : mend
Sep 22 23:15:03.655 INFO First source client ID for extent 124, mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO extent:124  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO extent:124  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO extent:124  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO find dest for source 1 for extent at index 124, mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO Extent 125 has flush number mismatch, : mend
Sep 22 23:15:03.655 INFO First source client ID for extent 125, mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO extent:125  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO extent:125  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO extent:125  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO find dest for source 1 for extent at index 125, mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO Extent 126 has flush number mismatch, : mend
Sep 22 23:15:03.655 INFO First source client ID for extent 126, mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO extent:126  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO extent:126  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO extent:126  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO find dest for source 1 for extent at index 126, mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO Extent 127 has flush number mismatch, : mend
Sep 22 23:15:03.655 INFO First source client ID for extent 127, mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO extent:127  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO extent:127  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO extent:127  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO find dest for source 1 for extent at index 127, mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO Extent 128 has flush number mismatch, : mend
Sep 22 23:15:03.655 INFO First source client ID for extent 128, mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO extent:128  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO extent:128  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO extent:128  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO find dest for source 1 for extent at index 128, mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO Extent 129 has flush number mismatch, : mend
Sep 22 23:15:03.655 INFO First source client ID for extent 129, mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO extent:129  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO extent:129  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO extent:129  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO find dest for source 1 for extent at index 129, mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO Extent 130 has flush number mismatch, : mend
Sep 22 23:15:03.655 INFO First source client ID for extent 130, mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO extent:130  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO extent:130  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO extent:130  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO find dest for source 1 for extent at index 130, mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO Extent 131 has flush number mismatch, : mend
Sep 22 23:15:03.655 INFO First source client ID for extent 131, mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO extent:131  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO extent:131  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO extent:131  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO find dest for source 1 for extent at index 131, mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO Extent 132 has flush number mismatch, : mend
Sep 22 23:15:03.655 INFO First source client ID for extent 132, mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO extent:132  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO extent:132  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO extent:132  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO find dest for source 1 for extent at index 132, mrl: flush_mismatch, : mend
Sep 22 23:15:03.655 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO Extent 133 has flush number mismatch, : mend
Sep 22 23:15:03.656 INFO First source client ID for extent 133, mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO extent:133  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO extent:133  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO extent:133  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO find dest for source 1 for extent at index 133, mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO Extent 134 has flush number mismatch, : mend
Sep 22 23:15:03.656 INFO First source client ID for extent 134, mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO extent:134  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO extent:134  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO extent:134  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO find dest for source 1 for extent at index 134, mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO Extent 135 has flush number mismatch, : mend
Sep 22 23:15:03.656 INFO First source client ID for extent 135, mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO extent:135  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO extent:135  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO extent:135  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO find dest for source 1 for extent at index 135, mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO Extent 136 has flush number mismatch, : mend
Sep 22 23:15:03.656 INFO First source client ID for extent 136, mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO extent:136  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO extent:136  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO extent:136  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO find dest for source 1 for extent at index 136, mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO Extent 137 has flush number mismatch, : mend
Sep 22 23:15:03.656 INFO First source client ID for extent 137, mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO extent:137  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO extent:137  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO extent:137  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO find dest for source 1 for extent at index 137, mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO Extent 138 has flush number mismatch, : mend
Sep 22 23:15:03.656 INFO First source client ID for extent 138, mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO extent:138  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO extent:138  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO extent:138  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO find dest for source 1 for extent at index 138, mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO Extent 139 has flush number mismatch, : mend
Sep 22 23:15:03.656 INFO First source client ID for extent 139, mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO extent:139  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO extent:139  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO extent:139  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO find dest for source 1 for extent at index 139, mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO Extent 140 has flush number mismatch, : mend
Sep 22 23:15:03.656 INFO First source client ID for extent 140, mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO extent:140  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO extent:140  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO extent:140  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO find dest for source 1 for extent at index 140, mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO Extent 141 has flush number mismatch, : mend
Sep 22 23:15:03.656 INFO First source client ID for extent 141, mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO extent:141  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO extent:141  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO extent:141  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO find dest for source 1 for extent at index 141, mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO Extent 142 has flush number mismatch, : mend
Sep 22 23:15:03.656 INFO First source client ID for extent 142, mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO extent:142  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO extent:142  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO extent:142  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO find dest for source 1 for extent at index 142, mrl: flush_mismatch, : mend
Sep 22 23:15:03.656 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO Extent 143 has flush number mismatch, : mend
Sep 22 23:15:03.657 INFO First source client ID for extent 143, mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO extent:143  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO extent:143  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO extent:143  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO find dest for source 1 for extent at index 143, mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO Extent 144 has flush number mismatch, : mend
Sep 22 23:15:03.657 INFO First source client ID for extent 144, mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO extent:144  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO extent:144  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO extent:144  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO find dest for source 1 for extent at index 144, mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO Extent 145 has flush number mismatch, : mend
Sep 22 23:15:03.657 INFO First source client ID for extent 145, mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO extent:145  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO extent:145  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO extent:145  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO find dest for source 1 for extent at index 145, mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO Extent 146 has flush number mismatch, : mend
Sep 22 23:15:03.657 INFO First source client ID for extent 146, mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO extent:146  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO extent:146  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO extent:146  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO find dest for source 1 for extent at index 146, mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO Extent 147 has flush number mismatch, : mend
Sep 22 23:15:03.657 INFO First source client ID for extent 147, mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO extent:147  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO extent:147  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO extent:147  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO find dest for source 1 for extent at index 147, mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO Extent 148 has flush number mismatch, : mend
Sep 22 23:15:03.657 INFO First source client ID for extent 148, mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO extent:148  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO extent:148  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO extent:148  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO find dest for source 1 for extent at index 148, mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO Extent 149 has flush number mismatch, : mend
Sep 22 23:15:03.657 INFO First source client ID for extent 149, mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO extent:149  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO extent:149  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO extent:149  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO find dest for source 1 for extent at index 149, mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO Extent 150 has flush number mismatch, : mend
Sep 22 23:15:03.657 INFO First source client ID for extent 150, mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO extent:150  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO extent:150  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO extent:150  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO find dest for source 1 for extent at index 150, mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO Extent 151 has flush number mismatch, : mend
Sep 22 23:15:03.657 INFO First source client ID for extent 151, mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO extent:151  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO extent:151  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO extent:151  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO find dest for source 1 for extent at index 151, mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO Extent 152 has flush number mismatch, : mend
Sep 22 23:15:03.657 INFO First source client ID for extent 152, mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO extent:152  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO extent:152  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO extent:152  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.657 INFO find dest for source 1 for extent at index 152, mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO Extent 153 has flush number mismatch, : mend
Sep 22 23:15:03.658 INFO First source client ID for extent 153, mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO extent:153  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO extent:153  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO extent:153  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO find dest for source 1 for extent at index 153, mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO Extent 154 has flush number mismatch, : mend
Sep 22 23:15:03.658 INFO First source client ID for extent 154, mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO extent:154  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO extent:154  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO extent:154  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO find dest for source 1 for extent at index 154, mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO Extent 155 has flush number mismatch, : mend
Sep 22 23:15:03.658 INFO First source client ID for extent 155, mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO extent:155  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO extent:155  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO extent:155  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO find dest for source 1 for extent at index 155, mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO Extent 156 has flush number mismatch, : mend
Sep 22 23:15:03.658 INFO First source client ID for extent 156, mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO extent:156  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO extent:156  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO extent:156  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO find dest for source 1 for extent at index 156, mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO Extent 157 has flush number mismatch, : mend
Sep 22 23:15:03.658 INFO First source client ID for extent 157, mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO extent:157  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO extent:157  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO extent:157  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO find dest for source 1 for extent at index 157, mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO Extent 158 has flush number mismatch, : mend
Sep 22 23:15:03.658 INFO First source client ID for extent 158, mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO extent:158  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO extent:158  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO extent:158  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO find dest for source 1 for extent at index 158, mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO Extent 159 has flush number mismatch, : mend
Sep 22 23:15:03.658 INFO First source client ID for extent 159, mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO extent:159  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO extent:159  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO extent:159  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO find dest for source 1 for extent at index 159, mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO Extent 160 has flush number mismatch, : mend
Sep 22 23:15:03.658 INFO First source client ID for extent 160, mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO extent:160  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO extent:160  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO extent:160  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO find dest for source 1 for extent at index 160, mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO Extent 161 has flush number mismatch, : mend
Sep 22 23:15:03.658 INFO First source client ID for extent 161, mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO extent:161  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO extent:161  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO extent:161  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO find dest for source 1 for extent at index 161, mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO Extent 162 has flush number mismatch, : mend
Sep 22 23:15:03.658 INFO First source client ID for extent 162, mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO extent:162  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO extent:162  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO extent:162  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO find dest for source 1 for extent at index 162, mrl: flush_mismatch, : mend
Sep 22 23:15:03.658 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO Extent 163 has flush number mismatch, : mend
Sep 22 23:15:03.659 INFO First source client ID for extent 163, mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO extent:163  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO extent:163  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO extent:163  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO find dest for source 1 for extent at index 163, mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO Extent 164 has flush number mismatch, : mend
Sep 22 23:15:03.659 INFO First source client ID for extent 164, mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO extent:164  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO extent:164  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO extent:164  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO find dest for source 1 for extent at index 164, mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO Extent 165 has flush number mismatch, : mend
Sep 22 23:15:03.659 INFO First source client ID for extent 165, mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO extent:165  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO extent:165  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO extent:165  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO find dest for source 1 for extent at index 165, mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO Extent 166 has flush number mismatch, : mend
Sep 22 23:15:03.659 INFO First source client ID for extent 166, mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO extent:166  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO extent:166  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO extent:166  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO find dest for source 1 for extent at index 166, mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO Extent 167 has flush number mismatch, : mend
Sep 22 23:15:03.659 INFO First source client ID for extent 167, mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO extent:167  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO extent:167  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO extent:167  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO find dest for source 1 for extent at index 167, mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO Extent 168 has flush number mismatch, : mend
Sep 22 23:15:03.659 INFO First source client ID for extent 168, mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO extent:168  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO extent:168  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO extent:168  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO find dest for source 1 for extent at index 168, mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO Extent 169 has flush number mismatch, : mend
Sep 22 23:15:03.659 INFO First source client ID for extent 169, mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO extent:169  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO extent:169  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO extent:169  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO find dest for source 1 for extent at index 169, mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO Extent 170 has flush number mismatch, : mend
Sep 22 23:15:03.659 INFO First source client ID for extent 170, mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO extent:170  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO extent:170  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO extent:170  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO find dest for source 1 for extent at index 170, mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO Extent 171 has flush number mismatch, : mend
Sep 22 23:15:03.659 INFO First source client ID for extent 171, mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO extent:171  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO extent:171  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO extent:171  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO find dest for source 1 for extent at index 171, mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO Extent 172 has flush number mismatch, : mend
Sep 22 23:15:03.659 INFO First source client ID for extent 172, mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO extent:172  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO extent:172  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO extent:172  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO find dest for source 1 for extent at index 172, mrl: flush_mismatch, : mend
Sep 22 23:15:03.659 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO Extent 173 has flush number mismatch, : mend
Sep 22 23:15:03.660 INFO First source client ID for extent 173, mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO extent:173  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO extent:173  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO extent:173  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO find dest for source 1 for extent at index 173, mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO Extent 174 has flush number mismatch, : mend
Sep 22 23:15:03.660 INFO First source client ID for extent 174, mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO extent:174  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO extent:174  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO extent:174  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO find dest for source 1 for extent at index 174, mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO Extent 175 has flush number mismatch, : mend
Sep 22 23:15:03.660 INFO First source client ID for extent 175, mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO extent:175  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO extent:175  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO extent:175  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO find dest for source 1 for extent at index 175, mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO Extent 176 has flush number mismatch, : mend
Sep 22 23:15:03.660 INFO First source client ID for extent 176, mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO extent:176  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO extent:176  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO extent:176  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO find dest for source 1 for extent at index 176, mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO Extent 177 has flush number mismatch, : mend
Sep 22 23:15:03.660 INFO First source client ID for extent 177, mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO extent:177  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO extent:177  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO extent:177  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO find dest for source 1 for extent at index 177, mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO Extent 178 has flush number mismatch, : mend
Sep 22 23:15:03.660 INFO First source client ID for extent 178, mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO extent:178  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO extent:178  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO extent:178  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO find dest for source 1 for extent at index 178, mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO Extent 179 has flush number mismatch, : mend
Sep 22 23:15:03.660 INFO First source client ID for extent 179, mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO extent:179  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO extent:179  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO extent:179  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO find dest for source 1 for extent at index 179, mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO Extent 180 has flush number mismatch, : mend
Sep 22 23:15:03.660 INFO First source client ID for extent 180, mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO extent:180  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO extent:180  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO extent:180  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO find dest for source 1 for extent at index 180, mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO Extent 181 has flush number mismatch, : mend
Sep 22 23:15:03.660 INFO First source client ID for extent 181, mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO extent:181  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO extent:181  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO extent:181  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO find dest for source 1 for extent at index 181, mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO Extent 182 has flush number mismatch, : mend
Sep 22 23:15:03.660 INFO First source client ID for extent 182, mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO extent:182  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO extent:182  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO extent:182  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.660 INFO find dest for source 1 for extent at index 182, mrl: flush_mismatch, : mend
Sep 22 23:15:03.661 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.661 INFO Extent 183 has flush number mismatch, : mend
Sep 22 23:15:03.661 INFO First source client ID for extent 183, mrl: flush_mismatch, : mend
Sep 22 23:15:03.661 INFO extent:183  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.661 INFO extent:183  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.661 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.661 INFO extent:183  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.661 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.661 INFO find dest for source 1 for extent at index 183, mrl: flush_mismatch, : mend
Sep 22 23:15:03.661 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.661 INFO Extent 184 has flush number mismatch, : mend
Sep 22 23:15:03.661 INFO First source client ID for extent 184, mrl: flush_mismatch, : mend
Sep 22 23:15:03.661 INFO extent:184  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.661 INFO extent:184  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.661 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.661 INFO extent:184  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.661 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.661 INFO find dest for source 1 for extent at index 184, mrl: flush_mismatch, : mend
Sep 22 23:15:03.661 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.661 INFO Extent 185 has flush number mismatch, : mend
Sep 22 23:15:03.661 INFO First source client ID for extent 185, mrl: flush_mismatch, : mend
Sep 22 23:15:03.661 INFO extent:185  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.661 INFO extent:185  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.661 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.661 INFO extent:185  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.661 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.661 INFO find dest for source 1 for extent at index 185, mrl: flush_mismatch, : mend
Sep 22 23:15:03.661 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.661 INFO Extent 186 has flush number mismatch, : mend
Sep 22 23:15:03.661 INFO First source client ID for extent 186, mrl: flush_mismatch, : mend
Sep 22 23:15:03.661 INFO extent:186  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.661 INFO extent:186  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.661 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.661 INFO extent:186  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.661 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.661 INFO find dest for source 1 for extent at index 186, mrl: flush_mismatch, : mend
Sep 22 23:15:03.661 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.661 INFO Extent 187 has flush number mismatch, : mend
Sep 22 23:15:03.661 INFO First source client ID for extent 187, mrl: flush_mismatch, : mend
Sep 22 23:15:03.661 INFO extent:187  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:15:03.661 INFO extent:187  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.661 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.661 INFO extent:187  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:15:03.661 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:15:03.661 INFO find dest for source 1 for extent at index 187, mrl: flush_mismatch, : mend
Sep 22 23:15:03.661 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:15:03.661 INFO Transition from WaitQuorum to Repair
Sep 22 23:15:03.661 INFO Transition from WaitQuorum to Repair
Sep 22 23:15:03.661 INFO Transition from WaitQuorum to Repair
Sep 22 23:15:03.661 INFO Found 188 extents that need repair
Sep 22 23:15:03.661 INFO Full repair list: {115: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 0: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 143: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 151: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 169: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 185: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 11: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 16: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 67: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 137: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 146: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 105: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 4: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 70: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 113: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 85: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 69: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 102: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 73: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 76: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 98: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 65: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 45: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 75: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 68: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 35: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 152: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 30: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 95: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 133: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 174: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 158: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 106: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 103: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 80: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 177: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 84: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 91: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 2: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 99: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 110: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 121: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 157: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 57: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 32: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 60: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 7: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 139: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 163: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 165: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 71: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 141: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 114: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 107: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 150: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 164: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 6: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 52: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 19: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 79: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 97: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 112: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 118: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 130: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 25: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 47: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 3: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 54: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 58: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 18: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 138: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 100: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 123: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 96: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 28: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 178: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 36: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 92: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 147: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 111: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 187: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 149: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 160: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 122: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 77: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 136: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 176: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 125: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 90: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 144: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 156: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 33: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 48: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 78: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 88: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 82: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 86: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 94: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 145: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 168: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 127: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 26: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 34: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 12: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 172: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 53: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 175: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 140: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 167: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 142: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 171: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 42: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 43: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 179: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 120: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 66: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 132: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 173: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 180: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 184: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 24: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 50: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 117: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 38: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 129: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 153: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 181: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 128: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 72: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 21: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 61: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 134: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 46: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 41: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 22: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 56: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 23: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 10: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 44: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 59: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 154: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 159: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 31: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 87: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 148: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 162: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 51: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 39: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 155: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 29: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 135: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 161: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 15: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 64: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 93: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 40: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 49: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 81: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 170: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 13: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 27: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 8: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 1: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 9: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 116: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 131: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 17: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 182: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 183: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 20: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 63: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 166: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 14: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 37: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 124: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 5: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 55: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 62: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 89: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 126: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 186: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 108: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 109: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 104: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 101: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 74: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 83: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 119: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }}, : downstairs
Sep 22 23:15:03.662 INFO Task list: [ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: 115, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: 115 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: 115, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: 115 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: 0, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: 0 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: 0, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: 0 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(8), op: ExtentFlush { repair_id: ReconciliationId(8), extent_id: 143, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(9), op: ExtentClose { repair_id: ReconciliationId(9), extent_id: 143 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(10), op: ExtentRepair { repair_id: ReconciliationId(10), extent_id: 143, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(11), op: ExtentReopen { repair_id: ReconciliationId(11), extent_id: 143 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(12), op: ExtentFlush { repair_id: ReconciliationId(12), extent_id: 151, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(13), op: ExtentClose { repair_id: ReconciliationId(13), extent_id: 151 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(14), op: ExtentRepair { repair_id: ReconciliationId(14), extent_id: 151, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(15), op: ExtentReopen { repair_id: ReconciliationId(15), extent_id: 151 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(16), op: ExtentFlush { repair_id: ReconciliationId(16), extent_id: 169, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(17), op: ExtentClose { repair_id: ReconciliationId(17), extent_id: 169 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(18), op: ExtentRepair { repair_id: ReconciliationId(18), extent_id: 169, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(19), op: ExtentReopen { repair_id: ReconciliationId(19), extent_id: 169 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(20), op: ExtentFlush { repair_id: ReconciliationId(20), extent_id: 185, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(21), op: ExtentClose { repair_id: ReconciliationId(21), extent_id: 185 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(22), op: ExtentRepair { repair_id: ReconciliationId(22), extent_id: 185, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(23), op: ExtentReopen { repair_id: ReconciliationId(23), extent_id: 185 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(24), op: ExtentFlush { repair_id: ReconciliationId(24), extent_id: 11, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(25), op: ExtentClose { repair_id: ReconciliationId(25), extent_id: 11 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(26), op: ExtentRepair { repair_id: ReconciliationId(26), extent_id: 11, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(27), op: ExtentReopen { repair_id: ReconciliationId(27), extent_id: 11 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(28), op: ExtentFlush { repair_id: ReconciliationId(28), extent_id: 16, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(29), op: ExtentClose { repair_id: ReconciliationId(29), extent_id: 16 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(30), op: ExtentRepair { repair_id: ReconciliationId(30), extent_id: 16, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(31), op: ExtentReopen { repair_id: ReconciliationId(31), extent_id: 16 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(32), op: ExtentFlush { repair_id: ReconciliationId(32), extent_id: 67, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(33), op: ExtentClose { repair_id: ReconciliationId(33), extent_id: 67 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(34), op: ExtentRepair { repair_id: ReconciliationId(34), extent_id: 67, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(35), op: ExtentReopen { repair_id: ReconciliationId(35), extent_id: 67 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(36), op: ExtentFlush { repair_id: ReconciliationId(36), extent_id: 137, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(37), op: ExtentClose { repair_id: ReconciliationId(37), extent_id: 137 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(38), op: ExtentRepair { repair_id: ReconciliationId(38), extent_id: 137, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(39), op: ExtentReopen { repair_id: ReconciliationId(39), extent_id: 137 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(40), op: ExtentFlush { repair_id: ReconciliationId(40), extent_id: 146, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(41), op: ExtentClose { repair_id: ReconciliationId(41), extent_id: 146 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(42), op: ExtentRepair { repair_id: ReconciliationId(42), extent_id: 146, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(43), op: ExtentReopen { repair_id: ReconciliationId(43), extent_id: 146 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(44), op: ExtentFlush { repair_id: ReconciliationId(44), extent_id: 105, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(45), op: ExtentClose { repair_id: ReconciliationId(45), extent_id: 105 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(46), op: ExtentRepair { repair_id: ReconciliationId(46), extent_id: 105, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(47), op: ExtentReopen { repair_id: ReconciliationId(47), extent_id: 105 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(48), op: ExtentFlush { repair_id: ReconciliationId(48), extent_id: 4, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(49), op: ExtentClose { repair_id: ReconciliationId(49), extent_id: 4 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(50), op: ExtentRepair { repair_id: ReconciliationId(50), extent_id: 4, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(51), op: ExtentReopen { repair_id: ReconciliationId(51), extent_id: 4 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(52), op: ExtentFlush { repair_id: ReconciliationId(52), extent_id: 70, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(53), op: ExtentClose { repair_id: ReconciliationId(53), extent_id: 70 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(54), op: ExtentRepair { repair_id: ReconciliationId(54), extent_id: 70, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(55), op: ExtentReopen { repair_id: ReconciliationId(55), extent_id: 70 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(56), op: ExtentFlush { repair_id: ReconciliationId(56), extent_id: 113, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(57), op: ExtentClose { repair_id: ReconciliationId(57), extent_id: 113 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(58), op: ExtentRepair { repair_id: ReconciliationId(58), extent_id: 113, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(59), op: ExtentReopen { repair_id: ReconciliationId(59), extent_id: 113 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(60), op: ExtentFlush { repair_id: ReconciliationId(60), extent_id: 85, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(61), op: ExtentClose { repair_id: ReconciliationId(61), extent_id: 85 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(62), op: ExtentRepair { repair_id: ReconciliationId(62), extent_id: 85, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(63), op: ExtentReopen { repair_id: ReconciliationId(63), extent_id: 85 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(64), op: ExtentFlush { repair_id: ReconciliationId(64), extent_id: 69, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(65), op: ExtentClose { repair_id: ReconciliationId(65), extent_id: 69 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(66), op: ExtentRepair { repair_id: ReconciliationId(66), extent_id: 69, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(67), op: ExtentReopen { repair_id: ReconciliationId(67), extent_id: 69 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(68), op: ExtentFlush { repair_id: ReconciliationId(68), extent_id: 102, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(69), op: ExtentClose { repair_id: ReconciliationId(69), extent_id: 102 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(70), op: ExtentRepair { repair_id: ReconciliationId(70), extent_id: 102, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(71), op: ExtentReopen { repair_id: ReconciliationId(71), extent_id: 102 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(72), op: ExtentFlush { repair_id: ReconciliationId(72), extent_id: 73, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(73), op: ExtentClose { repair_id: ReconciliationId(73), extent_id: 73 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(74), op: ExtentRepair { repair_id: ReconciliationId(74), extent_id: 73, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(75), op: ExtentReopen { repair_id: ReconciliationId(75), extent_id: 73 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(76), op: ExtentFlush { repair_id: ReconciliationId(76), extent_id: 76, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(77), op: ExtentClose { repair_id: ReconciliationId(77), extent_id: 76 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(78), op: ExtentRepair { repair_id: ReconciliationId(78), extent_id: 76, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(79), op: ExtentReopen { repair_id: ReconciliationId(79), extent_id: 76 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(80), op: ExtentFlush { repair_id: ReconciliationId(80), extent_id: 98, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(81), op: ExtentClose { repair_id: ReconciliationId(81), extent_id: 98 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(82), op: ExtentRepair { repair_id: ReconciliationId(82), extent_id: 98, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(83), op: ExtentReopen { repair_id: ReconciliationId(83), extent_id: 98 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(84), op: ExtentFlush { repair_id: ReconciliationId(84), extent_id: 65, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(85), op: ExtentClose { repair_id: ReconciliationId(85), extent_id: 65 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(86), op: ExtentRepair { repair_id: ReconciliationId(86), extent_id: 65, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(87), op: ExtentReopen { repair_id: ReconciliationId(87), extent_id: 65 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(88), op: ExtentFlush { repair_id: ReconciliationId(88), extent_id: 45, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(89), op: ExtentClose { repair_id: ReconciliationId(89), extent_id: 45 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(90), op: ExtentRepair { repair_id: ReconciliationId(90), extent_id: 45, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(91), op: ExtentReopen { repair_id: ReconciliationId(91), extent_id: 45 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(92), op: ExtentFlush { repair_id: ReconciliationId(92), extent_id: 75, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(93), op: ExtentClose { repair_id: ReconciliationId(93), extent_id: 75 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(94), op: ExtentRepair { repair_id: ReconciliationId(94), extent_id: 75, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(95), op: ExtentReopen { repair_id: ReconciliationId(95), extent_id: 75 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(96), op: ExtentFlush { repair_id: ReconciliationId(96), extent_id: 68, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(97), op: ExtentClose { repair_id: ReconciliationId(97), extent_id: 68 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(98), op: ExtentRepair { repair_id: ReconciliationId(98), extent_id: 68, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(99), op: ExtentReopen { repair_id: ReconciliationId(99), extent_id: 68 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(100), op: ExtentFlush { repair_id: ReconciliationId(100), extent_id: 35, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(101), op: ExtentClose { repair_id: ReconciliationId(101), extent_id: 35 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(102), op: ExtentRepair { repair_id: ReconciliationId(102), extent_id: 35, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(103), op: ExtentReopen { repair_id: ReconciliationId(103), extent_id: 35 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(104), op: ExtentFlush { repair_id: ReconciliationId(104), extent_id: 152, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(105), op: ExtentClose { repair_id: ReconciliationId(105), extent_id: 152 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(106), op: ExtentRepair { repair_id: ReconciliationId(106), extent_id: 152, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(107), op: ExtentReopen { repair_id: ReconciliationId(107), extent_id: 152 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(108), op: ExtentFlush { repair_id: ReconciliationId(108), extent_id: 30, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(109), op: ExtentClose { repair_id: ReconciliationId(109), extent_id: 30 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(110), op: ExtentRepair { repair_id: ReconciliationId(110), extent_id: 30, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(111), op: ExtentReopen { repair_id: ReconciliationId(111), extent_id: 30 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(112), op: ExtentFlush { repair_id: ReconciliationId(112), extent_id: 95, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(113), op: ExtentClose { repair_id: ReconciliationId(113), extent_id: 95 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(114), op: ExtentRepair { repair_id: ReconciliationId(114), extent_id: 95, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(115), op: ExtentReopen { repair_id: ReconciliationId(115), extent_id: 95 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(116), op: ExtentFlush { repair_id: ReconciliationId(116), extent_id: 133, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(117), op: ExtentClose { repair_id: ReconciliationId(117), extent_id: 133 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(118), op: ExtentRepair { repair_id: ReconciliationId(118), extent_id: 133, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(119), op: ExtentReopen { repair_id: ReconciliationId(119), extent_id: 133 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(120), op: ExtentFlush { repair_id: ReconciliationId(120), extent_id: 174, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(121), op: ExtentClose { repair_id: ReconciliationId(121), extent_id: 174 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(122), op: ExtentRepair { repair_id: ReconciliationId(122), extent_id: 174, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(123), op: ExtentReopen { repair_id: ReconciliationId(123), extent_id: 174 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(124), op: ExtentFlush { repair_id: ReconciliationId(124), extent_id: 158, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(125), op: ExtentClose { repair_id: ReconciliationId(125), extent_id: 158 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(126), op: ExtentRepair { repair_id: ReconciliationId(126), extent_id: 158, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(127), op: ExtentReopen { repair_id: ReconciliationId(127), extent_id: 158 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(128), op: ExtentFlush { repair_id: ReconciliationId(128), extent_id: 106, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(129), op: ExtentClose { repair_id: ReconciliationId(129), extent_id: 106 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(130), op: ExtentRepair { repair_id: ReconciliationId(130), extent_id: 106, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(131), op: ExtentReopen { repair_id: ReconciliationId(131), extent_id: 106 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(132), op: ExtentFlush { repair_id: ReconciliationId(132), extent_id: 103, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(133), op: ExtentClose { repair_id: ReconciliationId(133), extent_id: 103 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(134), op: ExtentRepair { repair_id: ReconciliationId(134), extent_id: 103, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(135), op: ExtentReopen { repair_id: ReconciliationId(135), extent_id: 103 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(136), op: ExtentFlush { repair_id: ReconciliationId(136), extent_id: 80, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(137), op: ExtentClose { repair_id: ReconciliationId(137), extent_id: 80 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(138), op: ExtentRepair { repair_id: ReconciliationId(138), extent_id: 80, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(139), op: ExtentReopen { repair_id: ReconciliationId(139), extent_id: 80 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(140), op: ExtentFlush { repair_id: ReconciliationId(140), extent_id: 177, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(141), op: ExtentClose { repair_id: ReconciliationId(141), extent_id: 177 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(142), op: ExtentRepair { repair_id: ReconciliationId(142), extent_id: 177, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(143), op: ExtentReopen { repair_id: ReconciliationId(143), extent_id: 177 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(144), op: ExtentFlush { repair_id: ReconciliationId(144), extent_id: 84, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(145), op: ExtentClose { repair_id: ReconciliationId(145), extent_id: 84 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(146), op: ExtentRepair { repair_id: ReconciliationId(146), extent_id: 84, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(147), op: ExtentReopen { repair_id: ReconciliationId(147), extent_id: 84 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(148), op: ExtentFlush { repair_id: ReconciliationId(148), extent_id: 91, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(149), op: ExtentClose { repair_id: ReconciliationId(149), extent_id: 91 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(150), op: ExtentRepair { repair_id: ReconciliationId(150), extent_id: 91, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(151), op: ExtentReopen { repair_id: ReconciliationId(151), extent_id: 91 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(152), op: ExtentFlush { repair_id: ReconciliationId(152), extent_id: 2, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(153), op: ExtentClose { repair_id: ReconciliationId(153), extent_id: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(154), op: ExtentRepair { repair_id: ReconciliationId(154), extent_id: 2, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(155), op: ExtentReopen { repair_id: ReconciliationId(155), extent_id: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(156), op: ExtentFlush { repair_id: ReconciliationId(156), extent_id: 99, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(157), op: ExtentClose { repair_id: ReconciliationId(157), extent_id: 99 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(158), op: ExtentRepair { repair_id: ReconciliationId(158), extent_id: 99, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(159), op: ExtentReopen { repair_id: ReconciliationId(159), extent_id: 99 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(160), op: ExtentFlush { repair_id: ReconciliationId(160), extent_id: 110, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(161), op: ExtentClose { repair_id: ReconciliationId(161), extent_id: 110 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(162), op: ExtentRepair { repair_id: ReconciliationId(162), extent_id: 110, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(163), op: ExtentReopen { repair_id: ReconciliationId(163), extent_id: 110 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(164), op: ExtentFlush { repair_id: ReconciliationId(164), extent_id: 121, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(165), op: ExtentClose { repair_id: ReconciliationId(165), extent_id: 121 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(166), op: ExtentRepair { repair_id: ReconciliationId(166), extent_id: 121, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(167), op: ExtentReopen { repair_id: ReconciliationId(167), extent_id: 121 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(168), op: ExtentFlush { repair_id: ReconciliationId(168), extent_id: 157, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(169), op: ExtentClose { repair_id: ReconciliationId(169), extent_id: 157 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(170), op: ExtentRepair { repair_id: ReconciliationId(170), extent_id: 157, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(171), op: ExtentReopen { repair_id: ReconciliationId(171), extent_id: 157 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(172), op: ExtentFlush { repair_id: ReconciliationId(172), extent_id: 57, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(173), op: ExtentClose { repair_id: ReconciliationId(173), extent_id: 57 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(174), op: ExtentRepair { repair_id: ReconciliationId(174), extent_id: 57, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(175), op: ExtentReopen { repair_id: ReconciliationId(175), extent_id: 57 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(176), op: ExtentFlush { repair_id: ReconciliationId(176), extent_id: 32, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(177), op: ExtentClose { repair_id: ReconciliationId(177), extent_id: 32 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(178), op: ExtentRepair { repair_id: ReconciliationId(178), extent_id: 32, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(179), op: ExtentReopen { repair_id: ReconciliationId(179), extent_id: 32 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(180), op: ExtentFlush { repair_id: ReconciliationId(180), extent_id: 60, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(181), op: ExtentClose { repair_id: ReconciliationId(181), extent_id: 60 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(182), op: ExtentRepair { repair_id: ReconciliationId(182), extent_id: 60, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(183), op: ExtentReopen { repair_id: ReconciliationId(183), extent_id: 60 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(184), op: ExtentFlush { repair_id: ReconciliationId(184), extent_id: 7, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(185), op: ExtentClose { repair_id: ReconciliationId(185), extent_id: 7 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(186), op: ExtentRepair { repair_id: ReconciliationId(186), extent_id: 7, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(187), op: ExtentReopen { repair_id: ReconciliationId(187), extent_id: 7 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(188), op: ExtentFlush { repair_id: ReconciliationId(188), extent_id: 139, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(189), op: ExtentClose { repair_id: ReconciliationId(189), extent_id: 139 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(190), op: ExtentRepair { repair_id: ReconciliationId(190), extent_id: 139, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(191), op: ExtentReopen { repair_id: ReconciliationId(191), extent_id: 139 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(192), op: ExtentFlush { repair_id: ReconciliationId(192), extent_id: 163, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(193), op: ExtentClose { repair_id: ReconciliationId(193), extent_id: 163 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(194), op: ExtentRepair { repair_id: ReconciliationId(194), extent_id: 163, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(195), op: ExtentReopen { repair_id: ReconciliationId(195), extent_id: 163 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(196), op: ExtentFlush { repair_id: ReconciliationId(196), extent_id: 165, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(197), op: ExtentClose { repair_id: ReconciliationId(197), extent_id: 165 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(198), op: ExtentRepair { repair_id: ReconciliationId(198), extent_id: 165, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(199), op: ExtentReopen { repair_id: ReconciliationId(199), extent_id: 165 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(200), op: ExtentFlush { repair_id: ReconciliationId(200), extent_id: 71, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(201), op: ExtentClose { repair_id: ReconciliationId(201), extent_id: 71 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(202), op: ExtentRepair { repair_id: ReconciliationId(202), extent_id: 71, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(203), op: ExtentReopen { repair_id: ReconciliationId(203), extent_id: 71 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(204), op: ExtentFlush { repair_id: ReconciliationId(204), extent_id: 141, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(205), op: ExtentClose { repair_id: ReconciliationId(205), extent_id: 141 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(206), op: ExtentRepair { repair_id: ReconciliationId(206), extent_id: 141, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(207), op: ExtentReopen { repair_id: ReconciliationId(207), extent_id: 141 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(208), op: ExtentFlush { repair_id: ReconciliationId(208), extent_id: 114, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(209), op: ExtentClose { repair_id: ReconciliationId(209), extent_id: 114 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(210), op: ExtentRepair { repair_id: ReconciliationId(210), extent_id: 114, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(211), op: ExtentReopen { repair_id: ReconciliationId(211), extent_id: 114 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(212), op: ExtentFlush { repair_id: ReconciliationId(212), extent_id: 107, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(213), op: ExtentClose { repair_id: ReconciliationId(213), extent_id: 107 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(214), op: ExtentRepair { repair_id: ReconciliationId(214), extent_id: 107, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(215), op: ExtentReopen { repair_id: ReconciliationId(215), extent_id: 107 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(216), op: ExtentFlush { repair_id: ReconciliationId(216), extent_id: 150, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(217), op: ExtentClose { repair_id: ReconciliationId(217), extent_id: 150 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(218), op: ExtentRepair { repair_id: ReconciliationId(218), extent_id: 150, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(219), op: ExtentReopen { repair_id: ReconciliationId(219), extent_id: 150 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(220), op: ExtentFlush { repair_id: ReconciliationId(220), extent_id: 164, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(221), op: ExtentClose { repair_id: ReconciliationId(221), extent_id: 164 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(222), op: ExtentRepair { repair_id: ReconciliationId(222), extent_id: 164, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(223), op: ExtentReopen { repair_id: ReconciliationId(223), extent_id: 164 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(224), op: ExtentFlush { repair_id: ReconciliationId(224), extent_id: 6, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(225), op: ExtentClose { repair_id: ReconciliationId(225), extent_id: 6 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(226), op: ExtentRepair { repair_id: ReconciliationId(226), extent_id: 6, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(227), op: ExtentReopen { repair_id: ReconciliationId(227), extent_id: 6 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(228), op: ExtentFlush { repair_id: ReconciliationId(228), extent_id: 52, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(229), op: ExtentClose { repair_id: ReconciliationId(229), extent_id: 52 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(230), op: ExtentRepair { repair_id: ReconciliationId(230), extent_id: 52, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(231), op: ExtentReopen { repair_id: ReconciliationId(231), extent_id: 52 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(232), op: ExtentFlush { repair_id: ReconciliationId(232), extent_id: 19, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(233), op: ExtentClose { repair_id: ReconciliationId(233), extent_id: 19 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(234), op: ExtentRepair { repair_id: ReconciliationId(234), extent_id: 19, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(235), op: ExtentReopen { repair_id: ReconciliationId(235), extent_id: 19 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(236), op: ExtentFlush { repair_id: ReconciliationId(236), extent_id: 79, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(237), op: ExtentClose { repair_id: ReconciliationId(237), extent_id: 79 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(238), op: ExtentRepair { repair_id: ReconciliationId(238), extent_id: 79, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(239), op: ExtentReopen { repair_id: ReconciliationId(239), extent_id: 79 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(240), op: ExtentFlush { repair_id: ReconciliationId(240), extent_id: 97, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(241), op: ExtentClose { repair_id: ReconciliationId(241), extent_id: 97 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(242), op: ExtentRepair { repair_id: ReconciliationId(242), extent_id: 97, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(243), op: ExtentReopen { repair_id: ReconciliationId(243), extent_id: 97 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(244), op: ExtentFlush { repair_id: ReconciliationId(244), extent_id: 112, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(245), op: ExtentClose { repair_id: ReconciliationId(245), extent_id: 112 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(246), op: ExtentRepair { repair_id: ReconciliationId(246), extent_id: 112, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(247), op: ExtentReopen { repair_id: ReconciliationId(247), extent_id: 112 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(248), op: ExtentFlush { repair_id: ReconciliationId(248), extent_id: 118, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(249), op: ExtentClose { repair_id: ReconciliationId(249), extent_id: 118 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(250), op: ExtentRepair { repair_id: ReconciliationId(250), extent_id: 118, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(251), op: ExtentReopen { repair_id: ReconciliationId(251), extent_id: 118 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(252), op: ExtentFlush { repair_id: ReconciliationId(252), extent_id: 130, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(253), op: ExtentClose { repair_id: ReconciliationId(253), extent_id: 130 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(254), op: ExtentRepair { repair_id: ReconciliationId(254), extent_id: 130, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(255), op: ExtentReopen { repair_id: ReconciliationId(255), extent_id: 130 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(256), op: ExtentFlush { repair_id: ReconciliationId(256), extent_id: 25, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(257), op: ExtentClose { repair_id: ReconciliationId(257), extent_id: 25 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(258), op: ExtentRepair { repair_id: ReconciliationId(258), extent_id: 25, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(259), op: ExtentReopen { repair_id: ReconciliationId(259), extent_id: 25 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(260), op: ExtentFlush { repair_id: ReconciliationId(260), extent_id: 47, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(261), op: ExtentClose { repair_id: ReconciliationId(261), extent_id: 47 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(262), op: ExtentRepair { repair_id: ReconciliationId(262), extent_id: 47, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(263), op: ExtentReopen { repair_id: ReconciliationId(263), extent_id: 47 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(264), op: ExtentFlush { repair_id: ReconciliationId(264), extent_id: 3, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(265), op: ExtentClose { repair_id: ReconciliationId(265), extent_id: 3 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(266), op: ExtentRepair { repair_id: ReconciliationId(266), extent_id: 3, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(267), op: ExtentReopen { repair_id: ReconciliationId(267), extent_id: 3 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(268), op: ExtentFlush { repair_id: ReconciliationId(268), extent_id: 54, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(269), op: ExtentClose { repair_id: ReconciliationId(269), extent_id: 54 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(270), op: ExtentRepair { repair_id: ReconciliationId(270), extent_id: 54, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(271), op: ExtentReopen { repair_id: ReconciliationId(271), extent_id: 54 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(272), op: ExtentFlush { repair_id: ReconciliationId(272), extent_id: 58, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(273), op: ExtentClose { repair_id: ReconciliationId(273), extent_id: 58 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(274), op: ExtentRepair { repair_id: ReconciliationId(274), extent_id: 58, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(275), op: ExtentReopen { repair_id: ReconciliationId(275), extent_id: 58 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(276), op: ExtentFlush { repair_id: ReconciliationId(276), extent_id: 18, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(277), op: ExtentClose { repair_id: ReconciliationId(277), extent_id: 18 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(278), op: ExtentRepair { repair_id: ReconciliationId(278), extent_id: 18, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(279), op: ExtentReopen { repair_id: ReconciliationId(279), extent_id: 18 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(280), op: ExtentFlush { repair_id: ReconciliationId(280), extent_id: 138, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(281), op: ExtentClose { repair_id: ReconciliationId(281), extent_id: 138 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(282), op: ExtentRepair { repair_id: ReconciliationId(282), extent_id: 138, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(283), op: ExtentReopen { repair_id: ReconciliationId(283), extent_id: 138 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(284), op: ExtentFlush { repair_id: ReconciliationId(284), extent_id: 100, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(285), op: ExtentClose { repair_id: ReconciliationId(285), extent_id: 100 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(286), op: ExtentRepair { repair_id: ReconciliationId(286), extent_id: 100, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(287), op: ExtentReopen { repair_id: ReconciliationId(287), extent_id: 100 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(288), op: ExtentFlush { repair_id: ReconciliationId(288), extent_id: 123, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(289), op: ExtentClose { repair_id: ReconciliationId(289), extent_id: 123 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(290), op: ExtentRepair { repair_id: ReconciliationId(290), extent_id: 123, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(291), op: ExtentReopen { repair_id: ReconciliationId(291), extent_id: 123 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(292), op: ExtentFlush { repair_id: ReconciliationId(292), extent_id: 96, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(293), op: ExtentClose { repair_id: ReconciliationId(293), extent_id: 96 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(294), op: ExtentRepair { repair_id: ReconciliationId(294), extent_id: 96, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(295), op: ExtentReopen { repair_id: ReconciliationId(295), extent_id: 96 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(296), op: ExtentFlush { repair_id: ReconciliationId(296), extent_id: 28, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(297), op: ExtentClose { repair_id: ReconciliationId(297), extent_id: 28 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(298), op: ExtentRepair { repair_id: ReconciliationId(298), extent_id: 28, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(299), op: ExtentReopen { repair_id: ReconciliationId(299), extent_id: 28 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(300), op: ExtentFlush { repair_id: ReconciliationId(300), extent_id: 178, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(301), op: ExtentClose { repair_id: ReconciliationId(301), extent_id: 178 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(302), op: ExtentRepair { repair_id: ReconciliationId(302), extent_id: 178, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(303), op: ExtentReopen { repair_id: ReconciliationId(303), extent_id: 178 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(304), op: ExtentFlush { repair_id: ReconciliationId(304), extent_id: 36, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(305), op: ExtentClose { repair_id: ReconciliationId(305), extent_id: 36 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(306), op: ExtentRepair { repair_id: ReconciliationId(306), extent_id: 36, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(307), op: ExtentReopen { repair_id: ReconciliationId(307), extent_id: 36 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(308), op: ExtentFlush { repair_id: ReconciliationId(308), extent_id: 92, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(309), op: ExtentClose { repair_id: ReconciliationId(309), extent_id: 92 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(310), op: ExtentRepair { repair_id: ReconciliationId(310), extent_id: 92, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(311), op: ExtentReopen { repair_id: ReconciliationId(311), extent_id: 92 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(312), op: ExtentFlush { repair_id: ReconciliationId(312), extent_id: 147, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(313), op: ExtentClose { repair_id: ReconciliationId(313), extent_id: 147 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(314), op: ExtentRepair { repair_id: ReconciliationId(314), extent_id: 147, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(315), op: ExtentReopen { repair_id: ReconciliationId(315), extent_id: 147 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(316), op: ExtentFlush { repair_id: ReconciliationId(316), extent_id: 111, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(317), op: ExtentClose { repair_id: ReconciliationId(317), extent_id: 111 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(318), op: ExtentRepair { repair_id: ReconciliationId(318), extent_id: 111, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(319), op: ExtentReopen { repair_id: ReconciliationId(319), extent_id: 111 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(320), op: ExtentFlush { repair_id: ReconciliationId(320), extent_id: 187, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(321), op: ExtentClose { repair_id: ReconciliationId(321), extent_id: 187 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(322), op: ExtentRepair { repair_id: ReconciliationId(322), extent_id: 187, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(323), op: ExtentReopen { repair_id: ReconciliationId(323), extent_id: 187 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(324), op: ExtentFlush { repair_id: ReconciliationId(324), extent_id: 149, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(325), op: ExtentClose { repair_id: ReconciliationId(325), extent_id: 149 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(326), op: ExtentRepair { repair_id: ReconciliationId(326), extent_id: 149, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(327), op: ExtentReopen { repair_id: ReconciliationId(327), extent_id: 149 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(328), op: ExtentFlush { repair_id: ReconciliationId(328), extent_id: 160, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(329), op: ExtentClose { repair_id: ReconciliationId(329), extent_id: 160 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(330), op: ExtentRepair { repair_id: ReconciliationId(330), extent_id: 160, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(331), op: ExtentReopen { repair_id: ReconciliationId(331), extent_id: 160 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(332), op: ExtentFlush { repair_id: ReconciliationId(332), extent_id: 122, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(333), op: ExtentClose { repair_id: ReconciliationId(333), extent_id: 122 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(334), op: ExtentRepair { repair_id: ReconciliationId(334), extent_id: 122, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(335), op: ExtentReopen { repair_id: ReconciliationId(335), extent_id: 122 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(336), op: ExtentFlush { repair_id: ReconciliationId(336), extent_id: 77, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(337), op: ExtentClose { repair_id: ReconciliationId(337), extent_id: 77 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(338), op: ExtentRepair { repair_id: ReconciliationId(338), extent_id: 77, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(339), op: ExtentReopen { repair_id: ReconciliationId(339), extent_id: 77 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(340), op: ExtentFlush { repair_id: ReconciliationId(340), extent_id: 136, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(341), op: ExtentClose { repair_id: ReconciliationId(341), extent_id: 136 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(342), op: ExtentRepair { repair_id: ReconciliationId(342), extent_id: 136, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(343), op: ExtentReopen { repair_id: ReconciliationId(343), extent_id: 136 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(344), op: ExtentFlush { repair_id: ReconciliationId(344), extent_id: 176, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(345), op: ExtentClose { repair_id: ReconciliationId(345), extent_id: 176 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(346), op: ExtentRepair { repair_id: ReconciliationId(346), extent_id: 176, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(347), op: ExtentReopen { repair_id: ReconciliationId(347), extent_id: 176 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(348), op: ExtentFlush { repair_id: ReconciliationId(348), extent_id: 125, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(349), op: ExtentClose { repair_id: ReconciliationId(349), extent_id: 125 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(350), op: ExtentRepair { repair_id: ReconciliationId(350), extent_id: 125, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(351), op: ExtentReopen { repair_id: ReconciliationId(351), extent_id: 125 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(352), op: ExtentFlush { repair_id: ReconciliationId(352), extent_id: 90, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(353), op: ExtentClose { repair_id: ReconciliationId(353), extent_id: 90 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(354), op: ExtentRepair { repair_id: ReconciliationId(354), extent_id: 90, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(355), op: ExtentReopen { repair_id: ReconciliationId(355), extent_id: 90 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(356), op: ExtentFlush { repair_id: ReconciliationId(356), extent_id: 144, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(357), op: ExtentClose { repair_id: ReconciliationId(357), extent_id: 144 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(358), op: ExtentRepair { repair_id: ReconciliationId(358), extent_id: 144, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(359), op: ExtentReopen { repair_id: ReconciliationId(359), extent_id: 144 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(360), op: ExtentFlush { repair_id: ReconciliationId(360), extent_id: 156, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(361), op: ExtentClose { repair_id: ReconciliationId(361), extent_id: 156 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(362), op: ExtentRepair { repair_id: ReconciliationId(362), extent_id: 156, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(363), op: ExtentReopen { repair_id: ReconciliationId(363), extent_id: 156 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(364), op: ExtentFlush { repair_id: ReconciliationId(364), extent_id: 33, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(365), op: ExtentClose { repair_id: ReconciliationId(365), extent_id: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(366), op: ExtentRepair { repair_id: ReconciliationId(366), extent_id: 33, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(367), op: ExtentReopen { repair_id: ReconciliationId(367), extent_id: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(368), op: ExtentFlush { repair_id: ReconciliationId(368), extent_id: 48, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(369), op: ExtentClose { repair_id: ReconciliationId(369), extent_id: 48 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(370), op: ExtentRepair { repair_id: ReconciliationId(370), extent_id: 48, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(371), op: ExtentReopen { repair_id: ReconciliationId(371), extent_id: 48 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(372), op: ExtentFlush { repair_id: ReconciliationId(372), extent_id: 78, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(373), op: ExtentClose { repair_id: ReconciliationId(373), extent_id: 78 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(374), op: ExtentRepair { repair_id: ReconciliationId(374), extent_id: 78, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(375), op: ExtentReopen { repair_id: ReconciliationId(375), extent_id: 78 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(376), op: ExtentFlush { repair_id: ReconciliationId(376), extent_id: 88, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(377), op: ExtentClose { repair_id: ReconciliationId(377), extent_id: 88 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(378), op: ExtentRepair { repair_id: ReconciliationId(378), extent_id: 88, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(379), op: ExtentReopen { repair_id: ReconciliationId(379), extent_id: 88 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(380), op: ExtentFlush { repair_id: ReconciliationId(380), extent_id: 82, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(381), op: ExtentClose { repair_id: ReconciliationId(381), extent_id: 82 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(382), op: ExtentRepair { repair_id: ReconciliationId(382), extent_id: 82, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(383), op: ExtentReopen { repair_id: ReconciliationId(383), extent_id: 82 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(384), op: ExtentFlush { repair_id: ReconciliationId(384), extent_id: 86, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(385), op: ExtentClose { repair_id: ReconciliationId(385), extent_id: 86 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(386), op: ExtentRepair { repair_id: ReconciliationId(386), extent_id: 86, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(387), op: ExtentReopen { repair_id: ReconciliationId(387), extent_id: 86 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(388), op: ExtentFlush { repair_id: ReconciliationId(388), extent_id: 94, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(389), op: ExtentClose { repair_id: ReconciliationId(389), extent_id: 94 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(390), op: ExtentRepair { repair_id: ReconciliationId(390), extent_id: 94, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(391), op: ExtentReopen { repair_id: ReconciliationId(391), extent_id: 94 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(392), op: ExtentFlush { repair_id: ReconciliationId(392), extent_id: 145, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(393), op: ExtentClose { repair_id: ReconciliationId(393), extent_id: 145 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(394), op: ExtentRepair { repair_id: ReconciliationId(394), extent_id: 145, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(395), op: ExtentReopen { repair_id: ReconciliationId(395), extent_id: 145 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(396), op: ExtentFlush { repair_id: ReconciliationId(396), extent_id: 168, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(397), op: ExtentClose { repair_id: ReconciliationId(397), extent_id: 168 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(398), op: ExtentRepair { repair_id: ReconciliationId(398), extent_id: 168, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(399), op: ExtentReopen { repair_id: ReconciliationId(399), extent_id: 168 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(400), op: ExtentFlush { repair_id: ReconciliationId(400), extent_id: 127, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(401), op: ExtentClose { repair_id: ReconciliationId(401), extent_id: 127 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(402), op: ExtentRepair { repair_id: ReconciliationId(402), extent_id: 127, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(403), op: ExtentReopen { repair_id: ReconciliationId(403), extent_id: 127 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(404), op: ExtentFlush { repair_id: ReconciliationId(404), extent_id: 26, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(405), op: ExtentClose { repair_id: ReconciliationId(405), extent_id: 26 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(406), op: ExtentRepair { repair_id: ReconciliationId(406), extent_id: 26, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(407), op: ExtentReopen { repair_id: ReconciliationId(407), extent_id: 26 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(408), op: ExtentFlush { repair_id: ReconciliationId(408), extent_id: 34, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(409), op: ExtentClose { repair_id: ReconciliationId(409), extent_id: 34 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(410), op: ExtentRepair { repair_id: ReconciliationId(410), extent_id: 34, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(411), op: ExtentReopen { repair_id: ReconciliationId(411), extent_id: 34 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(412), op: ExtentFlush { repair_id: ReconciliationId(412), extent_id: 12, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(413), op: ExtentClose { repair_id: ReconciliationId(413), extent_id: 12 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(414), op: ExtentRepair { repair_id: ReconciliationId(414), extent_id: 12, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(415), op: ExtentReopen { repair_id: ReconciliationId(415), extent_id: 12 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(416), op: ExtentFlush { repair_id: ReconciliationId(416), extent_id: 172, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(417), op: ExtentClose { repair_id: ReconciliationId(417), extent_id: 172 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(418), op: ExtentRepair { repair_id: ReconciliationId(418), extent_id: 172, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(419), op: ExtentReopen { repair_id: ReconciliationId(419), extent_id: 172 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(420), op: ExtentFlush { repair_id: ReconciliationId(420), extent_id: 53, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(421), op: ExtentClose { repair_id: ReconciliationId(421), extent_id: 53 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(422), op: ExtentRepair { repair_id: ReconciliationId(422), extent_id: 53, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(423), op: ExtentReopen { repair_id: ReconciliationId(423), extent_id: 53 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(424), op: ExtentFlush { repair_id: ReconciliationId(424), extent_id: 175, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(425), op: ExtentClose { repair_id: ReconciliationId(425), extent_id: 175 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(426), op: ExtentRepair { repair_id: ReconciliationId(426), extent_id: 175, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(427), op: ExtentReopen { repair_id: ReconciliationId(427), extent_id: 175 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(428), op: ExtentFlush { repair_id: ReconciliationId(428), extent_id: 140, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(429), op: ExtentClose { repair_id: ReconciliationId(429), extent_id: 140 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(430), op: ExtentRepair { repair_id: ReconciliationId(430), extent_id: 140, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(431), op: ExtentReopen { repair_id: ReconciliationId(431), extent_id: 140 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(432), op: ExtentFlush { repair_id: ReconciliationId(432), extent_id: 167, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(433), op: ExtentClose { repair_id: ReconciliationId(433), extent_id: 167 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(434), op: ExtentRepair { repair_id: ReconciliationId(434), extent_id: 167, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(435), op: ExtentReopen { repair_id: ReconciliationId(435), extent_id: 167 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(436), op: ExtentFlush { repair_id: ReconciliationId(436), extent_id: 142, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(437), op: ExtentClose { repair_id: ReconciliationId(437), extent_id: 142 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(438), op: ExtentRepair { repair_id: ReconciliationId(438), extent_id: 142, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(439), op: ExtentReopen { repair_id: ReconciliationId(439), extent_id: 142 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(440), op: ExtentFlush { repair_id: ReconciliationId(440), extent_id: 171, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(441), op: ExtentClose { repair_id: ReconciliationId(441), extent_id: 171 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(442), op: ExtentRepair { repair_id: ReconciliationId(442), extent_id: 171, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(443), op: ExtentReopen { repair_id: ReconciliationId(443), extent_id: 171 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(444), op: ExtentFlush { repair_id: ReconciliationId(444), extent_id: 42, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(445), op: ExtentClose { repair_id: ReconciliationId(445), extent_id: 42 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(446), op: ExtentRepair { repair_id: ReconciliationId(446), extent_id: 42, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(447), op: ExtentReopen { repair_id: ReconciliationId(447), extent_id: 42 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(448), op: ExtentFlush { repair_id: ReconciliationId(448), extent_id: 43, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(449), op: ExtentClose { repair_id: ReconciliationId(449), extent_id: 43 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(450), op: ExtentRepair { repair_id: ReconciliationId(450), extent_id: 43, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(451), op: ExtentReopen { repair_id: ReconciliationId(451), extent_id: 43 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(452), op: ExtentFlush { repair_id: ReconciliationId(452), extent_id: 179, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(453), op: ExtentClose { repair_id: ReconciliationId(453), extent_id: 179 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(454), op: ExtentRepair { repair_id: ReconciliationId(454), extent_id: 179, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(455), op: ExtentReopen { repair_id: ReconciliationId(455), extent_id: 179 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(456), op: ExtentFlush { repair_id: ReconciliationId(456), extent_id: 120, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(457), op: ExtentClose { repair_id: ReconciliationId(457), extent_id: 120 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(458), op: ExtentRepair { repair_id: ReconciliationId(458), extent_id: 120, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(459), op: ExtentReopen { repair_id: ReconciliationId(459), extent_id: 120 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(460), op: ExtentFlush { repair_id: ReconciliationId(460), extent_id: 66, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(461), op: ExtentClose { repair_id: ReconciliationId(461), extent_id: 66 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(462), op: ExtentRepair { repair_id: ReconciliationId(462), extent_id: 66, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(463), op: ExtentReopen { repair_id: ReconciliationId(463), extent_id: 66 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(464), op: ExtentFlush { repair_id: ReconciliationId(464), extent_id: 132, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(465), op: ExtentClose { repair_id: ReconciliationId(465), extent_id: 132 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(466), op: ExtentRepair { repair_id: ReconciliationId(466), extent_id: 132, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(467), op: ExtentReopen { repair_id: ReconciliationId(467), extent_id: 132 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(468), op: ExtentFlush { repair_id: ReconciliationId(468), extent_id: 173, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(469), op: ExtentClose { repair_id: ReconciliationId(469), extent_id: 173 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(470), op: ExtentRepair { repair_id: ReconciliationId(470), extent_id: 173, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(471), op: ExtentReopen { repair_id: ReconciliationId(471), extent_id: 173 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(472), op: ExtentFlush { repair_id: ReconciliationId(472), extent_id: 180, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(473), op: ExtentClose { repair_id: ReconciliationId(473), extent_id: 180 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(474), op: ExtentRepair { repair_id: ReconciliationId(474), extent_id: 180, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(475), op: ExtentReopen { repair_id: ReconciliationId(475), extent_id: 180 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(476), op: ExtentFlush { repair_id: ReconciliationId(476), extent_id: 184, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(477), op: ExtentClose { repair_id: ReconciliationId(477), extent_id: 184 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(478), op: ExtentRepair { repair_id: ReconciliationId(478), extent_id: 184, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(479), op: ExtentReopen { repair_id: ReconciliationId(479), extent_id: 184 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(480), op: ExtentFlush { repair_id: ReconciliationId(480), extent_id: 24, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(481), op: ExtentClose { repair_id: ReconciliationId(481), extent_id: 24 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(482), op: ExtentRepair { repair_id: ReconciliationId(482), extent_id: 24, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(483), op: ExtentReopen { repair_id: ReconciliationId(483), extent_id: 24 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(484), op: ExtentFlush { repair_id: ReconciliationId(484), extent_id: 50, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(485), op: ExtentClose { repair_id: ReconciliationId(485), extent_id: 50 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(486), op: ExtentRepair { repair_id: ReconciliationId(486), extent_id: 50, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(487), op: ExtentReopen { repair_id: ReconciliationId(487), extent_id: 50 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(488), op: ExtentFlush { repair_id: ReconciliationId(488), extent_id: 117, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(489), op: ExtentClose { repair_id: ReconciliationId(489), extent_id: 117 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(490), op: ExtentRepair { repair_id: ReconciliationId(490), extent_id: 117, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(491), op: ExtentReopen { repair_id: ReconciliationId(491), extent_id: 117 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(492), op: ExtentFlush { repair_id: ReconciliationId(492), extent_id: 38, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(493), op: ExtentClose { repair_id: ReconciliationId(493), extent_id: 38 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(494), op: ExtentRepair { repair_id: ReconciliationId(494), extent_id: 38, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(495), op: ExtentReopen { repair_id: ReconciliationId(495), extent_id: 38 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(496), op: ExtentFlush { repair_id: ReconciliationId(496), extent_id: 129, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(497), op: ExtentClose { repair_id: ReconciliationId(497), extent_id: 129 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(498), op: ExtentRepair { repair_id: ReconciliationId(498), extent_id: 129, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(499), op: ExtentReopen { repair_id: ReconciliationId(499), extent_id: 129 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(500), op: ExtentFlush { repair_id: ReconciliationId(500), extent_id: 153, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(501), op: ExtentClose { repair_id: ReconciliationId(501), extent_id: 153 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(502), op: ExtentRepair { repair_id: ReconciliationId(502), extent_id: 153, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(503), op: ExtentReopen { repair_id: ReconciliationId(503), extent_id: 153 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(504), op: ExtentFlush { repair_id: ReconciliationId(504), extent_id: 181, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(505), op: ExtentClose { repair_id: ReconciliationId(505), extent_id: 181 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(506), op: ExtentRepair { repair_id: ReconciliationId(506), extent_id: 181, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(507), op: ExtentReopen { repair_id: ReconciliationId(507), extent_id: 181 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(508), op: ExtentFlush { repair_id: ReconciliationId(508), extent_id: 128, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(509), op: ExtentClose { repair_id: ReconciliationId(509), extent_id: 128 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(510), op: ExtentRepair { repair_id: ReconciliationId(510), extent_id: 128, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(511), op: ExtentReopen { repair_id: ReconciliationId(511), extent_id: 128 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(512), op: ExtentFlush { repair_id: ReconciliationId(512), extent_id: 72, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(513), op: ExtentClose { repair_id: ReconciliationId(513), extent_id: 72 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(514), op: ExtentRepair { repair_id: ReconciliationId(514), extent_id: 72, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(515), op: ExtentReopen { repair_id: ReconciliationId(515), extent_id: 72 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(516), op: ExtentFlush { repair_id: ReconciliationId(516), extent_id: 21, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(517), op: ExtentClose { repair_id: ReconciliationId(517), extent_id: 21 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(518), op: ExtentRepair { repair_id: ReconciliationId(518), extent_id: 21, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(519), op: ExtentReopen { repair_id: ReconciliationId(519), extent_id: 21 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(520), op: ExtentFlush { repair_id: ReconciliationId(520), extent_id: 61, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(521), op: ExtentClose { repair_id: ReconciliationId(521), extent_id: 61 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(522), op: ExtentRepair { repair_id: ReconciliationId(522), extent_id: 61, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(523), op: ExtentReopen { repair_id: ReconciliationId(523), extent_id: 61 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(524), op: ExtentFlush { repair_id: ReconciliationId(524), extent_id: 134, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(525), op: ExtentClose { repair_id: ReconciliationId(525), extent_id: 134 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(526), op: ExtentRepair { repair_id: ReconciliationId(526), extent_id: 134, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(527), op: ExtentReopen { repair_id: ReconciliationId(527), extent_id: 134 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(528), op: ExtentFlush { repair_id: ReconciliationId(528), extent_id: 46, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(529), op: ExtentClose { repair_id: ReconciliationId(529), extent_id: 46 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(530), op: ExtentRepair { repair_id: ReconciliationId(530), extent_id: 46, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(531), op: ExtentReopen { repair_id: ReconciliationId(531), extent_id: 46 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(532), op: ExtentFlush { repair_id: ReconciliationId(532), extent_id: 41, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(533), op: ExtentClose { repair_id: ReconciliationId(533), extent_id: 41 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(534), op: ExtentRepair { repair_id: ReconciliationId(534), extent_id: 41, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(535), op: ExtentReopen { repair_id: ReconciliationId(535), extent_id: 41 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(536), op: ExtentFlush { repair_id: ReconciliationId(536), extent_id: 22, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(537), op: ExtentClose { repair_id: ReconciliationId(537), extent_id: 22 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(538), op: ExtentRepair { repair_id: ReconciliationId(538), extent_id: 22, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(539), op: ExtentReopen { repair_id: ReconciliationId(539), extent_id: 22 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(540), op: ExtentFlush { repair_id: ReconciliationId(540), extent_id: 56, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(541), op: ExtentClose { repair_id: ReconciliationId(541), extent_id: 56 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(542), op: ExtentRepair { repair_id: ReconciliationId(542), extent_id: 56, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(543), op: ExtentReopen { repair_id: ReconciliationId(543), extent_id: 56 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(544), op: ExtentFlush { repair_id: ReconciliationId(544), extent_id: 23, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(545), op: ExtentClose { repair_id: ReconciliationId(545), extent_id: 23 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(546), op: ExtentRepair { repair_id: ReconciliationId(546), extent_id: 23, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(547), op: ExtentReopen { repair_id: ReconciliationId(547), extent_id: 23 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(548), op: ExtentFlush { repair_id: ReconciliationId(548), extent_id: 10, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(549), op: ExtentClose { repair_id: ReconciliationId(549), extent_id: 10 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(550), op: ExtentRepair { repair_id: ReconciliationId(550), extent_id: 10, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(551), op: ExtentReopen { repair_id: ReconciliationId(551), extent_id: 10 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(552), op: ExtentFlush { repair_id: ReconciliationId(552), extent_id: 44, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(553), op: ExtentClose { repair_id: ReconciliationId(553), extent_id: 44 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(554), op: ExtentRepair { repair_id: ReconciliationId(554), extent_id: 44, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(555), op: ExtentReopen { repair_id: ReconciliationId(555), extent_id: 44 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(556), op: ExtentFlush { repair_id: ReconciliationId(556), extent_id: 59, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(557), op: ExtentClose { repair_id: ReconciliationId(557), extent_id: 59 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(558), op: ExtentRepair { repair_id: ReconciliationId(558), extent_id: 59, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(559), op: ExtentReopen { repair_id: ReconciliationId(559), extent_id: 59 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(560), op: ExtentFlush { repair_id: ReconciliationId(560), extent_id: 154, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(561), op: ExtentClose { repair_id: ReconciliationId(561), extent_id: 154 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(562), op: ExtentRepair { repair_id: ReconciliationId(562), extent_id: 154, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(563), op: ExtentReopen { repair_id: ReconciliationId(563), extent_id: 154 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(564), op: ExtentFlush { repair_id: ReconciliationId(564), extent_id: 159, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(565), op: ExtentClose { repair_id: ReconciliationId(565), extent_id: 159 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(566), op: ExtentRepair { repair_id: ReconciliationId(566), extent_id: 159, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(567), op: ExtentReopen { repair_id: ReconciliationId(567), extent_id: 159 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(568), op: ExtentFlush { repair_id: ReconciliationId(568), extent_id: 31, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(569), op: ExtentClose { repair_id: ReconciliationId(569), extent_id: 31 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(570), op: ExtentRepair { repair_id: ReconciliationId(570), extent_id: 31, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(571), op: ExtentReopen { repair_id: ReconciliationId(571), extent_id: 31 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(572), op: ExtentFlush { repair_id: ReconciliationId(572), extent_id: 87, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(573), op: ExtentClose { repair_id: ReconciliationId(573), extent_id: 87 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(574), op: ExtentRepair { repair_id: ReconciliationId(574), extent_id: 87, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(575), op: ExtentReopen { repair_id: ReconciliationId(575), extent_id: 87 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(576), op: ExtentFlush { repair_id: ReconciliationId(576), extent_id: 148, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(577), op: ExtentClose { repair_id: ReconciliationId(577), extent_id: 148 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(578), op: ExtentRepair { repair_id: ReconciliationId(578), extent_id: 148, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(579), op: ExtentReopen { repair_id: ReconciliationId(579), extent_id: 148 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(580), op: ExtentFlush { repair_id: ReconciliationId(580), extent_id: 162, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(581), op: ExtentClose { repair_id: ReconciliationId(581), extent_id: 162 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(582), op: ExtentRepair { repair_id: ReconciliationId(582), extent_id: 162, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(583), op: ExtentReopen { repair_id: ReconciliationId(583), extent_id: 162 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(584), op: ExtentFlush { repair_id: ReconciliationId(584), extent_id: 51, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(585), op: ExtentClose { repair_id: ReconciliationId(585), extent_id: 51 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(586), op: ExtentRepair { repair_id: ReconciliationId(586), extent_id: 51, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(587), op: ExtentReopen { repair_id: ReconciliationId(587), extent_id: 51 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(588), op: ExtentFlush { repair_id: ReconciliationId(588), extent_id: 39, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(589), op: ExtentClose { repair_id: ReconciliationId(589), extent_id: 39 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(590), op: ExtentRepair { repair_id: ReconciliationId(590), extent_id: 39, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(591), op: ExtentReopen { repair_id: ReconciliationId(591), extent_id: 39 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(592), op: ExtentFlush { repair_id: ReconciliationId(592), extent_id: 155, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(593), op: ExtentClose { repair_id: ReconciliationId(593), extent_id: 155 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(594), op: ExtentRepair { repair_id: ReconciliationId(594), extent_id: 155, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(595), op: ExtentReopen { repair_id: ReconciliationId(595), extent_id: 155 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(596), op: ExtentFlush { repair_id: ReconciliationId(596), extent_id: 29, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(597), op: ExtentClose { repair_id: ReconciliationId(597), extent_id: 29 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(598), op: ExtentRepair { repair_id: ReconciliationId(598), extent_id: 29, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(599), op: ExtentReopen { repair_id: ReconciliationId(599), extent_id: 29 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(600), op: ExtentFlush { repair_id: ReconciliationId(600), extent_id: 135, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(601), op: ExtentClose { repair_id: ReconciliationId(601), extent_id: 135 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(602), op: ExtentRepair { repair_id: ReconciliationId(602), extent_id: 135, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(603), op: ExtentReopen { repair_id: ReconciliationId(603), extent_id: 135 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(604), op: ExtentFlush { repair_id: ReconciliationId(604), extent_id: 161, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(605), op: ExtentClose { repair_id: ReconciliationId(605), extent_id: 161 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(606), op: ExtentRepair { repair_id: ReconciliationId(606), extent_id: 161, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(607), op: ExtentReopen { repair_id: ReconciliationId(607), extent_id: 161 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(608), op: ExtentFlush { repair_id: ReconciliationId(608), extent_id: 15, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(609), op: ExtentClose { repair_id: ReconciliationId(609), extent_id: 15 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(610), op: ExtentRepair { repair_id: ReconciliationId(610), extent_id: 15, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(611), op: ExtentReopen { repair_id: ReconciliationId(611), extent_id: 15 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(612), op: ExtentFlush { repair_id: ReconciliationId(612), extent_id: 64, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(613), op: ExtentClose { repair_id: ReconciliationId(613), extent_id: 64 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(614), op: ExtentRepair { repair_id: ReconciliationId(614), extent_id: 64, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(615), op: ExtentReopen { repair_id: ReconciliationId(615), extent_id: 64 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(616), op: ExtentFlush { repair_id: ReconciliationId(616), extent_id: 93, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(617), op: ExtentClose { repair_id: ReconciliationId(617), extent_id: 93 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(618), op: ExtentRepair { repair_id: ReconciliationId(618), extent_id: 93, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(619), op: ExtentReopen { repair_id: ReconciliationId(619), extent_id: 93 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(620), op: ExtentFlush { repair_id: ReconciliationId(620), extent_id: 40, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(621), op: ExtentClose { repair_id: ReconciliationId(621), extent_id: 40 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(622), op: ExtentRepair { repair_id: ReconciliationId(622), extent_id: 40, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(623), op: ExtentReopen { repair_id: ReconciliationId(623), extent_id: 40 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(624), op: ExtentFlush { repair_id: ReconciliationId(624), extent_id: 49, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(625), op: ExtentClose { repair_id: ReconciliationId(625), extent_id: 49 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(626), op: ExtentRepair { repair_id: ReconciliationId(626), extent_id: 49, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(627), op: ExtentReopen { repair_id: ReconciliationId(627), extent_id: 49 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(628), op: ExtentFlush { repair_id: ReconciliationId(628), extent_id: 81, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(629), op: ExtentClose { repair_id: ReconciliationId(629), extent_id: 81 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(630), op: ExtentRepair { repair_id: ReconciliationId(630), extent_id: 81, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(631), op: ExtentReopen { repair_id: ReconciliationId(631), extent_id: 81 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(632), op: ExtentFlush { repair_id: ReconciliationId(632), extent_id: 170, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(633), op: ExtentClose { repair_id: ReconciliationId(633), extent_id: 170 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(634), op: ExtentRepair { repair_id: ReconciliationId(634), extent_id: 170, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(635), op: ExtentReopen { repair_id: ReconciliationId(635), extent_id: 170 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(636), op: ExtentFlush { repair_id: ReconciliationId(636), extent_id: 13, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(637), op: ExtentClose { repair_id: ReconciliationId(637), extent_id: 13 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(638), op: ExtentRepair { repair_id: ReconciliationId(638), extent_id: 13, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(639), op: ExtentReopen { repair_id: ReconciliationId(639), extent_id: 13 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(640), op: ExtentFlush { repair_id: ReconciliationId(640), extent_id: 27, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(641), op: ExtentClose { repair_id: ReconciliationId(641), extent_id: 27 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(642), op: ExtentRepair { repair_id: ReconciliationId(642), extent_id: 27, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(643), op: ExtentReopen { repair_id: ReconciliationId(643), extent_id: 27 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(644), op: ExtentFlush { repair_id: ReconciliationId(644), extent_id: 8, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(645), op: ExtentClose { repair_id: ReconciliationId(645), extent_id: 8 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(646), op: ExtentRepair { repair_id: ReconciliationId(646), extent_id: 8, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(647), op: ExtentReopen { repair_id: ReconciliationId(647), extent_id: 8 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(648), op: ExtentFlush { repair_id: ReconciliationId(648), extent_id: 1, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(649), op: ExtentClose { repair_id: ReconciliationId(649), extent_id: 1 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(650), op: ExtentRepair { repair_id: ReconciliationId(650), extent_id: 1, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(651), op: ExtentReopen { repair_id: ReconciliationId(651), extent_id: 1 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(652), op: ExtentFlush { repair_id: ReconciliationId(652), extent_id: 9, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(653), op: ExtentClose { repair_id: ReconciliationId(653), extent_id: 9 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(654), op: ExtentRepair { repair_id: ReconciliationId(654), extent_id: 9, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(655), op: ExtentReopen { repair_id: ReconciliationId(655), extent_id: 9 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(656), op: ExtentFlush { repair_id: ReconciliationId(656), extent_id: 116, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(657), op: ExtentClose { repair_id: ReconciliationId(657), extent_id: 116 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(658), op: ExtentRepair { repair_id: ReconciliationId(658), extent_id: 116, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(659), op: ExtentReopen { repair_id: ReconciliationId(659), extent_id: 116 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(660), op: ExtentFlush { repair_id: ReconciliationId(660), extent_id: 131, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(661), op: ExtentClose { repair_id: ReconciliationId(661), extent_id: 131 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(662), op: ExtentRepair { repair_id: ReconciliationId(662), extent_id: 131, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(663), op: ExtentReopen { repair_id: ReconciliationId(663), extent_id: 131 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(664), op: ExtentFlush { repair_id: ReconciliationId(664), extent_id: 17, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(665), op: ExtentClose { repair_id: ReconciliationId(665), extent_id: 17 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(666), op: ExtentRepair { repair_id: ReconciliationId(666), extent_id: 17, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(667), op: ExtentReopen { repair_id: ReconciliationId(667), extent_id: 17 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(668), op: ExtentFlush { repair_id: ReconciliationId(668), extent_id: 182, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(669), op: ExtentClose { repair_id: ReconciliationId(669), extent_id: 182 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(670), op: ExtentRepair { repair_id: ReconciliationId(670), extent_id: 182, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(671), op: ExtentReopen { repair_id: ReconciliationId(671), extent_id: 182 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(672), op: ExtentFlush { repair_id: ReconciliationId(672), extent_id: 183, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(673), op: ExtentClose { repair_id: ReconciliationId(673), extent_id: 183 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(674), op: ExtentRepair { repair_id: ReconciliationId(674), extent_id: 183, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(675), op: ExtentReopen { repair_id: ReconciliationId(675), extent_id: 183 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(676), op: ExtentFlush { repair_id: ReconciliationId(676), extent_id: 20, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(677), op: ExtentClose { repair_id: ReconciliationId(677), extent_id: 20 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(678), op: ExtentRepair { repair_id: ReconciliationId(678), extent_id: 20, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(679), op: ExtentReopen { repair_id: ReconciliationId(679), extent_id: 20 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(680), op: ExtentFlush { repair_id: ReconciliationId(680), extent_id: 63, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(681), op: ExtentClose { repair_id: ReconciliationId(681), extent_id: 63 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(682), op: ExtentRepair { repair_id: ReconciliationId(682), extent_id: 63, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(683), op: ExtentReopen { repair_id: ReconciliationId(683), extent_id: 63 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(684), op: ExtentFlush { repair_id: ReconciliationId(684), extent_id: 166, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(685), op: ExtentClose { repair_id: ReconciliationId(685), extent_id: 166 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(686), op: ExtentRepair { repair_id: ReconciliationId(686), extent_id: 166, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(687), op: ExtentReopen { repair_id: ReconciliationId(687), extent_id: 166 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(688), op: ExtentFlush { repair_id: ReconciliationId(688), extent_id: 14, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(689), op: ExtentClose { repair_id: ReconciliationId(689), extent_id: 14 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(690), op: ExtentRepair { repair_id: ReconciliationId(690), extent_id: 14, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(691), op: ExtentReopen { repair_id: ReconciliationId(691), extent_id: 14 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(692), op: ExtentFlush { repair_id: ReconciliationId(692), extent_id: 37, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(693), op: ExtentClose { repair_id: ReconciliationId(693), extent_id: 37 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(694), op: ExtentRepair { repair_id: ReconciliationId(694), extent_id: 37, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(695), op: ExtentReopen { repair_id: ReconciliationId(695), extent_id: 37 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(696), op: ExtentFlush { repair_id: ReconciliationId(696), extent_id: 124, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(697), op: ExtentClose { repair_id: ReconciliationId(697), extent_id: 124 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(698), op: ExtentRepair { repair_id: ReconciliationId(698), extent_id: 124, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(699), op: ExtentReopen { repair_id: ReconciliationId(699), extent_id: 124 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(700), op: ExtentFlush { repair_id: ReconciliationId(700), extent_id: 5, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(701), op: ExtentClose { repair_id: ReconciliationId(701), extent_id: 5 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(702), op: ExtentRepair { repair_id: ReconciliationId(702), extent_id: 5, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(703), op: ExtentReopen { repair_id: ReconciliationId(703), extent_id: 5 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(704), op: ExtentFlush { repair_id: ReconciliationId(704), extent_id: 55, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(705), op: ExtentClose { repair_id: ReconciliationId(705), extent_id: 55 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(706), op: ExtentRepair { repair_id: ReconciliationId(706), extent_id: 55, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(707), op: ExtentReopen { repair_id: ReconciliationId(707), extent_id: 55 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(708), op: ExtentFlush { repair_id: ReconciliationId(708), extent_id: 62, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(709), op: ExtentClose { repair_id: ReconciliationId(709), extent_id: 62 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(710), op: ExtentRepair { repair_id: ReconciliationId(710), extent_id: 62, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(711), op: ExtentReopen { repair_id: ReconciliationId(711), extent_id: 62 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(712), op: ExtentFlush { repair_id: ReconciliationId(712), extent_id: 89, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(713), op: ExtentClose { repair_id: ReconciliationId(713), extent_id: 89 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(714), op: ExtentRepair { repair_id: ReconciliationId(714), extent_id: 89, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(715), op: ExtentReopen { repair_id: ReconciliationId(715), extent_id: 89 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(716), op: ExtentFlush { repair_id: ReconciliationId(716), extent_id: 126, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(717), op: ExtentClose { repair_id: ReconciliationId(717), extent_id: 126 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(718), op: ExtentRepair { repair_id: ReconciliationId(718), extent_id: 126, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(719), op: ExtentReopen { repair_id: ReconciliationId(719), extent_id: 126 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(720), op: ExtentFlush { repair_id: ReconciliationId(720), extent_id: 186, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(721), op: ExtentClose { repair_id: ReconciliationId(721), extent_id: 186 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(722), op: ExtentRepair { repair_id: ReconciliationId(722), extent_id: 186, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(723), op: ExtentReopen { repair_id: ReconciliationId(723), extent_id: 186 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(724), op: ExtentFlush { repair_id: ReconciliationId(724), extent_id: 108, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(725), op: ExtentClose { repair_id: ReconciliationId(725), extent_id: 108 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(726), op: ExtentRepair { repair_id: ReconciliationId(726), extent_id: 108, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(727), op: ExtentReopen { repair_id: ReconciliationId(727), extent_id: 108 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(728), op: ExtentFlush { repair_id: ReconciliationId(728), extent_id: 109, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(729), op: ExtentClose { repair_id: ReconciliationId(729), extent_id: 109 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(730), op: ExtentRepair { repair_id: ReconciliationId(730), extent_id: 109, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(731), op: ExtentReopen { repair_id: ReconciliationId(731), extent_id: 109 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(732), op: ExtentFlush { repair_id: ReconciliationId(732), extent_id: 104, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(733), op: ExtentClose { repair_id: ReconciliationId(733), extent_id: 104 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(734), op: ExtentRepair { repair_id: ReconciliationId(734), extent_id: 104, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(735), op: ExtentReopen { repair_id: ReconciliationId(735), extent_id: 104 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(736), op: ExtentFlush { repair_id: ReconciliationId(736), extent_id: 101, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(737), op: ExtentClose { repair_id: ReconciliationId(737), extent_id: 101 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(738), op: ExtentRepair { repair_id: ReconciliationId(738), extent_id: 101, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(739), op: ExtentReopen { repair_id: ReconciliationId(739), extent_id: 101 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(740), op: ExtentFlush { repair_id: ReconciliationId(740), extent_id: 74, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(741), op: ExtentClose { repair_id: ReconciliationId(741), extent_id: 74 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(742), op: ExtentRepair { repair_id: ReconciliationId(742), extent_id: 74, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(743), op: ExtentReopen { repair_id: ReconciliationId(743), extent_id: 74 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(744), op: ExtentFlush { repair_id: ReconciliationId(744), extent_id: 83, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(745), op: ExtentClose { repair_id: ReconciliationId(745), extent_id: 83 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(746), op: ExtentRepair { repair_id: ReconciliationId(746), extent_id: 83, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(747), op: ExtentReopen { repair_id: ReconciliationId(747), extent_id: 83 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(748), op: ExtentFlush { repair_id: ReconciliationId(748), extent_id: 119, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(749), op: ExtentClose { repair_id: ReconciliationId(749), extent_id: 119 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(750), op: ExtentRepair { repair_id: ReconciliationId(750), extent_id: 119, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(751), op: ExtentReopen { repair_id: ReconciliationId(751), extent_id: 119 }, state: ClientData([New, New, New]) }], : downstairs
Sep 22 23:15:03.666 INFO Begin repair with 752 commands
Sep 22 23:15:03.666 INFO Pop front: ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: 115, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:03.666 INFO Sent repair work, now wait for resp
Sep 22 23:15:03.666 INFO [0] received reconcile message
Sep 22 23:15:03.666 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: 115, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:03.666 INFO [0] client ExtentFlush { repair_id: ReconciliationId(0), extent_id: 115, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:03.666 INFO [1] received reconcile message
Sep 22 23:15:03.666 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: 115, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:03.666 INFO [1] client ExtentFlush { repair_id: ReconciliationId(0), extent_id: 115, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:03.666 INFO [2] received reconcile message
Sep 22 23:15:03.666 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: 115, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:03.666 INFO [2] client ExtentFlush { repair_id: ReconciliationId(0), extent_id: 115, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:03.666 DEBG 0 Flush extent 115 with f:2 g:2
Sep 22 23:15:03.666 DEBG Flush just extent 115 with f:2 and g:2
Sep 22 23:15:03.666 DEBG [1] It's time to notify for 0
Sep 22 23:15:03.666 INFO Completion from [1] id:0 status:true
Sep 22 23:15:03.667 INFO [1/752] Repair commands completed
Sep 22 23:15:03.667 INFO Pop front: ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: 115 }, state: ClientData([New, New, New]) }
Sep 22 23:15:03.667 INFO Sent repair work, now wait for resp
Sep 22 23:15:03.667 INFO [0] received reconcile message
Sep 22 23:15:03.667 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: 115 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:03.667 INFO [0] client ExtentClose { repair_id: ReconciliationId(1), extent_id: 115 }
Sep 22 23:15:03.667 INFO [1] received reconcile message
Sep 22 23:15:03.667 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: 115 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:03.667 INFO [1] client ExtentClose { repair_id: ReconciliationId(1), extent_id: 115 }
Sep 22 23:15:03.667 INFO [2] received reconcile message
Sep 22 23:15:03.667 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: 115 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:03.667 INFO [2] client ExtentClose { repair_id: ReconciliationId(1), extent_id: 115 }
Sep 22 23:15:03.667 DEBG 1 Close extent 115
Sep 22 23:15:03.667 DEBG 1 Close extent 115
Sep 22 23:15:03.668 DEBG 1 Close extent 115
Sep 22 23:15:03.668 DEBG [2] It's time to notify for 1
Sep 22 23:15:03.668 INFO Completion from [2] id:1 status:true
Sep 22 23:15:03.668 INFO [2/752] Repair commands completed
Sep 22 23:15:03.668 INFO Pop front: ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: 115, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:03.668 INFO Sent repair work, now wait for resp
Sep 22 23:15:03.668 INFO [0] received reconcile message
Sep 22 23:15:03.668 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: 115, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:03.668 INFO [0] client ExtentRepair { repair_id: ReconciliationId(2), extent_id: 115, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:03.668 INFO [0] Sending repair request ReconciliationId(2)
Sep 22 23:15:03.668 INFO [1] received reconcile message
Sep 22 23:15:03.668 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: 115, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:03.668 INFO [1] client ExtentRepair { repair_id: ReconciliationId(2), extent_id: 115, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:03.668 INFO [1] No action required ReconciliationId(2)
Sep 22 23:15:03.668 INFO [2] received reconcile message
Sep 22 23:15:03.668 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: 115, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:03.668 INFO [2] client ExtentRepair { repair_id: ReconciliationId(2), extent_id: 115, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:03.668 INFO [2] No action required ReconciliationId(2)
Sep 22 23:15:03.668 DEBG 2 Repair extent 115 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:03.668 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/073.copy"
Sep 22 23:15:03.733 INFO accepted connection, remote_addr: 127.0.0.1:60228, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:03.734 TRCE incoming request, uri: /extent/115/files, method: GET, req_id: fc4aa63c-5d77-4b7d-9d5a-6f38c0bfc191, remote_addr: 127.0.0.1:60228, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:03.734 INFO request completed, latency_us: 247, response_code: 200, uri: /extent/115/files, method: GET, req_id: fc4aa63c-5d77-4b7d-9d5a-6f38c0bfc191, remote_addr: 127.0.0.1:60228, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:03.734 INFO eid:115 Found repair files: ["073", "073.db"]
Sep 22 23:15:03.734 TRCE incoming request, uri: /newextent/115/data, method: GET, req_id: d59ca312-fa66-4fb5-84df-ae6f84c20c0e, remote_addr: 127.0.0.1:60228, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:03.735 INFO request completed, latency_us: 473, response_code: 200, uri: /newextent/115/data, method: GET, req_id: d59ca312-fa66-4fb5-84df-ae6f84c20c0e, remote_addr: 127.0.0.1:60228, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:03.740 TRCE incoming request, uri: /newextent/115/db, method: GET, req_id: a1ec65b7-9644-49e4-876e-f6d91b61ee23, remote_addr: 127.0.0.1:60228, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:03.740 INFO request completed, latency_us: 292, response_code: 200, uri: /newextent/115/db, method: GET, req_id: a1ec65b7-9644-49e4-876e-f6d91b61ee23, remote_addr: 127.0.0.1:60228, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:03.742 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/073.copy" to "/tmp/downstairs-vrx8aK6L/00/000/073.replace"
Sep 22 23:15:03.742 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:03.743 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/073.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:03.743 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/073"
Sep 22 23:15:03.743 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/073.db"
Sep 22 23:15:03.743 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:03.743 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/073.replace" to "/tmp/downstairs-vrx8aK6L/00/000/073.completed"
Sep 22 23:15:03.743 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:03.743 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:03.743 DEBG [0] It's time to notify for 2
Sep 22 23:15:03.744 INFO Completion from [0] id:2 status:true
Sep 22 23:15:03.744 INFO [3/752] Repair commands completed
Sep 22 23:15:03.744 INFO Pop front: ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: 115 }, state: ClientData([New, New, New]) }
Sep 22 23:15:03.744 INFO Sent repair work, now wait for resp
Sep 22 23:15:03.744 INFO [0] received reconcile message
Sep 22 23:15:03.744 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: 115 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:03.744 INFO [0] client ExtentReopen { repair_id: ReconciliationId(3), extent_id: 115 }
Sep 22 23:15:03.744 INFO [1] received reconcile message
Sep 22 23:15:03.744 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: 115 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:03.744 INFO [1] client ExtentReopen { repair_id: ReconciliationId(3), extent_id: 115 }
Sep 22 23:15:03.744 INFO [2] received reconcile message
Sep 22 23:15:03.744 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: 115 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:03.744 INFO [2] client ExtentReopen { repair_id: ReconciliationId(3), extent_id: 115 }
Sep 22 23:15:03.744 DEBG 3 Reopen extent 115
Sep 22 23:15:03.745 DEBG 3 Reopen extent 115
Sep 22 23:15:03.745 DEBG 3 Reopen extent 115
Sep 22 23:15:03.746 DEBG [2] It's time to notify for 3
Sep 22 23:15:03.746 INFO Completion from [2] id:3 status:true
Sep 22 23:15:03.746 INFO [4/752] Repair commands completed
Sep 22 23:15:03.746 INFO Pop front: ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: 0, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:03.746 INFO Sent repair work, now wait for resp
Sep 22 23:15:03.746 INFO [0] received reconcile message
Sep 22 23:15:03.746 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: 0, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:03.746 INFO [0] client ExtentFlush { repair_id: ReconciliationId(4), extent_id: 0, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:03.746 INFO [1] received reconcile message
Sep 22 23:15:03.746 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: 0, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:03.746 INFO [1] client ExtentFlush { repair_id: ReconciliationId(4), extent_id: 0, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:03.746 INFO [2] received reconcile message
Sep 22 23:15:03.746 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: 0, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:03.746 INFO [2] client ExtentFlush { repair_id: ReconciliationId(4), extent_id: 0, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:03.746 DEBG 4 Flush extent 0 with f:2 g:2
Sep 22 23:15:03.746 DEBG Flush just extent 0 with f:2 and g:2
Sep 22 23:15:03.746 DEBG [1] It's time to notify for 4
Sep 22 23:15:03.746 INFO Completion from [1] id:4 status:true
Sep 22 23:15:03.746 INFO [5/752] Repair commands completed
Sep 22 23:15:03.746 INFO Pop front: ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: 0 }, state: ClientData([New, New, New]) }
Sep 22 23:15:03.746 INFO Sent repair work, now wait for resp
Sep 22 23:15:03.746 INFO [0] received reconcile message
Sep 22 23:15:03.747 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: 0 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:03.747 INFO [0] client ExtentClose { repair_id: ReconciliationId(5), extent_id: 0 }
Sep 22 23:15:03.747 INFO [1] received reconcile message
Sep 22 23:15:03.747 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: 0 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:03.747 INFO [1] client ExtentClose { repair_id: ReconciliationId(5), extent_id: 0 }
Sep 22 23:15:03.747 INFO [2] received reconcile message
Sep 22 23:15:03.747 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: 0 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:03.747 INFO [2] client ExtentClose { repair_id: ReconciliationId(5), extent_id: 0 }
Sep 22 23:15:03.747 DEBG 5 Close extent 0
Sep 22 23:15:03.747 DEBG 5 Close extent 0
Sep 22 23:15:03.747 DEBG 5 Close extent 0
Sep 22 23:15:03.748 DEBG [2] It's time to notify for 5
Sep 22 23:15:03.748 INFO Completion from [2] id:5 status:true
Sep 22 23:15:03.748 INFO [6/752] Repair commands completed
Sep 22 23:15:03.748 INFO Pop front: ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: 0, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:03.748 INFO Sent repair work, now wait for resp
Sep 22 23:15:03.748 INFO [0] received reconcile message
Sep 22 23:15:03.748 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: 0, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:03.748 INFO [0] client ExtentRepair { repair_id: ReconciliationId(6), extent_id: 0, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:03.748 INFO [0] Sending repair request ReconciliationId(6)
Sep 22 23:15:03.748 INFO [1] received reconcile message
Sep 22 23:15:03.748 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: 0, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:03.748 INFO [1] client ExtentRepair { repair_id: ReconciliationId(6), extent_id: 0, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:03.748 INFO [1] No action required ReconciliationId(6)
Sep 22 23:15:03.748 INFO [2] received reconcile message
Sep 22 23:15:03.748 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: 0, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:03.748 INFO [2] client ExtentRepair { repair_id: ReconciliationId(6), extent_id: 0, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:03.748 INFO [2] No action required ReconciliationId(6)
Sep 22 23:15:03.748 DEBG 6 Repair extent 0 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:03.748 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/000.copy"
Sep 22 23:15:03.813 INFO accepted connection, remote_addr: 127.0.0.1:59635, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:03.813 TRCE incoming request, uri: /extent/0/files, method: GET, req_id: 0d8be5b1-9b4c-484f-a8aa-f45fc2ffbf62, remote_addr: 127.0.0.1:59635, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:03.814 INFO request completed, latency_us: 224, response_code: 200, uri: /extent/0/files, method: GET, req_id: 0d8be5b1-9b4c-484f-a8aa-f45fc2ffbf62, remote_addr: 127.0.0.1:59635, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:03.814 INFO eid:0 Found repair files: ["000", "000.db"]
Sep 22 23:15:03.814 TRCE incoming request, uri: /newextent/0/data, method: GET, req_id: 3bc4bfd9-1d32-4eab-b20a-0541a1f03211, remote_addr: 127.0.0.1:59635, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:03.815 INFO request completed, latency_us: 322, response_code: 200, uri: /newextent/0/data, method: GET, req_id: 3bc4bfd9-1d32-4eab-b20a-0541a1f03211, remote_addr: 127.0.0.1:59635, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:03.820 TRCE incoming request, uri: /newextent/0/db, method: GET, req_id: 17282a65-af12-4b9f-8647-66c0701d13f1, remote_addr: 127.0.0.1:59635, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:03.820 INFO request completed, latency_us: 292, response_code: 200, uri: /newextent/0/db, method: GET, req_id: 17282a65-af12-4b9f-8647-66c0701d13f1, remote_addr: 127.0.0.1:59635, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:03.821 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/000.copy" to "/tmp/downstairs-vrx8aK6L/00/000/000.replace"
Sep 22 23:15:03.821 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:03.822 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/000.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:03.822 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/000"
Sep 22 23:15:03.822 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/000.db"
Sep 22 23:15:03.823 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:03.823 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/000.replace" to "/tmp/downstairs-vrx8aK6L/00/000/000.completed"
Sep 22 23:15:03.823 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:03.823 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:03.823 DEBG [0] It's time to notify for 6
Sep 22 23:15:03.823 INFO Completion from [0] id:6 status:true
Sep 22 23:15:03.823 INFO [7/752] Repair commands completed
Sep 22 23:15:03.823 INFO Pop front: ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: 0 }, state: ClientData([New, New, New]) }
Sep 22 23:15:03.823 INFO Sent repair work, now wait for resp
Sep 22 23:15:03.823 INFO [0] received reconcile message
Sep 22 23:15:03.823 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: 0 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:03.823 INFO [0] client ExtentReopen { repair_id: ReconciliationId(7), extent_id: 0 }
Sep 22 23:15:03.823 INFO [1] received reconcile message
Sep 22 23:15:03.823 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: 0 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:03.823 INFO [1] client ExtentReopen { repair_id: ReconciliationId(7), extent_id: 0 }
Sep 22 23:15:03.823 INFO [2] received reconcile message
Sep 22 23:15:03.823 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: 0 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:03.823 INFO [2] client ExtentReopen { repair_id: ReconciliationId(7), extent_id: 0 }
Sep 22 23:15:03.823 DEBG 7 Reopen extent 0
Sep 22 23:15:03.824 DEBG 7 Reopen extent 0
Sep 22 23:15:03.825 DEBG 7 Reopen extent 0
Sep 22 23:15:03.825 DEBG [2] It's time to notify for 7
Sep 22 23:15:03.825 INFO Completion from [2] id:7 status:true
Sep 22 23:15:03.825 INFO [8/752] Repair commands completed
Sep 22 23:15:03.825 INFO Pop front: ReconcileIO { id: ReconciliationId(8), op: ExtentFlush { repair_id: ReconciliationId(8), extent_id: 143, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:03.825 INFO Sent repair work, now wait for resp
Sep 22 23:15:03.825 INFO [0] received reconcile message
Sep 22 23:15:03.825 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(8), op: ExtentFlush { repair_id: ReconciliationId(8), extent_id: 143, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:03.825 INFO [0] client ExtentFlush { repair_id: ReconciliationId(8), extent_id: 143, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:03.825 INFO [1] received reconcile message
Sep 22 23:15:03.825 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(8), op: ExtentFlush { repair_id: ReconciliationId(8), extent_id: 143, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:03.825 INFO [1] client ExtentFlush { repair_id: ReconciliationId(8), extent_id: 143, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:03.825 INFO [2] received reconcile message
Sep 22 23:15:03.826 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(8), op: ExtentFlush { repair_id: ReconciliationId(8), extent_id: 143, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:03.826 INFO [2] client ExtentFlush { repair_id: ReconciliationId(8), extent_id: 143, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:03.826 DEBG 8 Flush extent 143 with f:2 g:2
Sep 22 23:15:03.826 DEBG Flush just extent 143 with f:2 and g:2
Sep 22 23:15:03.826 DEBG [1] It's time to notify for 8
Sep 22 23:15:03.826 INFO Completion from [1] id:8 status:true
Sep 22 23:15:03.826 INFO [9/752] Repair commands completed
Sep 22 23:15:03.826 INFO Pop front: ReconcileIO { id: ReconciliationId(9), op: ExtentClose { repair_id: ReconciliationId(9), extent_id: 143 }, state: ClientData([New, New, New]) }
Sep 22 23:15:03.826 INFO Sent repair work, now wait for resp
Sep 22 23:15:03.826 INFO [0] received reconcile message
Sep 22 23:15:03.826 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(9), op: ExtentClose { repair_id: ReconciliationId(9), extent_id: 143 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:03.826 INFO [0] client ExtentClose { repair_id: ReconciliationId(9), extent_id: 143 }
Sep 22 23:15:03.826 INFO [1] received reconcile message
Sep 22 23:15:03.826 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(9), op: ExtentClose { repair_id: ReconciliationId(9), extent_id: 143 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:03.826 INFO [1] client ExtentClose { repair_id: ReconciliationId(9), extent_id: 143 }
Sep 22 23:15:03.826 INFO [2] received reconcile message
Sep 22 23:15:03.826 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(9), op: ExtentClose { repair_id: ReconciliationId(9), extent_id: 143 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:03.826 INFO [2] client ExtentClose { repair_id: ReconciliationId(9), extent_id: 143 }
Sep 22 23:15:03.826 DEBG 9 Close extent 143
Sep 22 23:15:03.826 DEBG 9 Close extent 143
Sep 22 23:15:03.827 DEBG 9 Close extent 143
Sep 22 23:15:03.827 DEBG [2] It's time to notify for 9
Sep 22 23:15:03.827 INFO Completion from [2] id:9 status:true
Sep 22 23:15:03.827 INFO [10/752] Repair commands completed
Sep 22 23:15:03.827 INFO Pop front: ReconcileIO { id: ReconciliationId(10), op: ExtentRepair { repair_id: ReconciliationId(10), extent_id: 143, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:03.827 INFO Sent repair work, now wait for resp
Sep 22 23:15:03.827 INFO [0] received reconcile message
Sep 22 23:15:03.827 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(10), op: ExtentRepair { repair_id: ReconciliationId(10), extent_id: 143, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:03.827 INFO [0] client ExtentRepair { repair_id: ReconciliationId(10), extent_id: 143, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:03.827 INFO [0] Sending repair request ReconciliationId(10)
Sep 22 23:15:03.827 INFO [1] received reconcile message
Sep 22 23:15:03.827 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(10), op: ExtentRepair { repair_id: ReconciliationId(10), extent_id: 143, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:03.827 INFO [1] client ExtentRepair { repair_id: ReconciliationId(10), extent_id: 143, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:03.827 INFO [1] No action required ReconciliationId(10)
Sep 22 23:15:03.827 INFO [2] received reconcile message
Sep 22 23:15:03.828 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(10), op: ExtentRepair { repair_id: ReconciliationId(10), extent_id: 143, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:03.828 INFO [2] client ExtentRepair { repair_id: ReconciliationId(10), extent_id: 143, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:03.828 INFO [2] No action required ReconciliationId(10)
Sep 22 23:15:03.828 DEBG 10 Repair extent 143 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:03.828 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/08F.copy"
Sep 22 23:15:03.891 INFO accepted connection, remote_addr: 127.0.0.1:48510, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:03.891 TRCE incoming request, uri: /extent/143/files, method: GET, req_id: af04195d-46c5-4493-8f92-caf2bd07b9b5, remote_addr: 127.0.0.1:48510, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:03.892 INFO request completed, latency_us: 192, response_code: 200, uri: /extent/143/files, method: GET, req_id: af04195d-46c5-4493-8f92-caf2bd07b9b5, remote_addr: 127.0.0.1:48510, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:03.892 INFO eid:143 Found repair files: ["08F", "08F.db"]
Sep 22 23:15:03.892 TRCE incoming request, uri: /newextent/143/data, method: GET, req_id: a7ca02ea-0a2a-4689-8953-a295b7a3447a, remote_addr: 127.0.0.1:48510, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:03.892 INFO request completed, latency_us: 250, response_code: 200, uri: /newextent/143/data, method: GET, req_id: a7ca02ea-0a2a-4689-8953-a295b7a3447a, remote_addr: 127.0.0.1:48510, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:03.898 TRCE incoming request, uri: /newextent/143/db, method: GET, req_id: d1471026-4a00-483b-8e58-7d87331b1162, remote_addr: 127.0.0.1:48510, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:03.898 INFO request completed, latency_us: 292, response_code: 200, uri: /newextent/143/db, method: GET, req_id: d1471026-4a00-483b-8e58-7d87331b1162, remote_addr: 127.0.0.1:48510, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:03.899 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/08F.copy" to "/tmp/downstairs-vrx8aK6L/00/000/08F.replace"
Sep 22 23:15:03.899 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:03.900 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/08F.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:03.900 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/08F"
Sep 22 23:15:03.900 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/08F.db"
Sep 22 23:15:03.900 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:03.900 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/08F.replace" to "/tmp/downstairs-vrx8aK6L/00/000/08F.completed"
Sep 22 23:15:03.900 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:03.900 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:03.901 DEBG [0] It's time to notify for 10
Sep 22 23:15:03.901 INFO Completion from [0] id:10 status:true
Sep 22 23:15:03.901 INFO [11/752] Repair commands completed
Sep 22 23:15:03.901 INFO Pop front: ReconcileIO { id: ReconciliationId(11), op: ExtentReopen { repair_id: ReconciliationId(11), extent_id: 143 }, state: ClientData([New, New, New]) }
Sep 22 23:15:03.901 INFO Sent repair work, now wait for resp
Sep 22 23:15:03.901 INFO [0] received reconcile message
Sep 22 23:15:03.901 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(11), op: ExtentReopen { repair_id: ReconciliationId(11), extent_id: 143 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:03.901 INFO [0] client ExtentReopen { repair_id: ReconciliationId(11), extent_id: 143 }
Sep 22 23:15:03.901 INFO [1] received reconcile message
Sep 22 23:15:03.901 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(11), op: ExtentReopen { repair_id: ReconciliationId(11), extent_id: 143 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:03.901 INFO [1] client ExtentReopen { repair_id: ReconciliationId(11), extent_id: 143 }
Sep 22 23:15:03.901 INFO [2] received reconcile message
Sep 22 23:15:03.901 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(11), op: ExtentReopen { repair_id: ReconciliationId(11), extent_id: 143 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:03.901 INFO [2] client ExtentReopen { repair_id: ReconciliationId(11), extent_id: 143 }
Sep 22 23:15:03.901 DEBG 11 Reopen extent 143
Sep 22 23:15:03.902 DEBG 11 Reopen extent 143
Sep 22 23:15:03.902 DEBG 11 Reopen extent 143
Sep 22 23:15:03.903 DEBG [2] It's time to notify for 11
Sep 22 23:15:03.903 INFO Completion from [2] id:11 status:true
Sep 22 23:15:03.903 INFO [12/752] Repair commands completed
Sep 22 23:15:03.903 INFO Pop front: ReconcileIO { id: ReconciliationId(12), op: ExtentFlush { repair_id: ReconciliationId(12), extent_id: 151, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:03.903 INFO Sent repair work, now wait for resp
Sep 22 23:15:03.903 INFO [0] received reconcile message
Sep 22 23:15:03.903 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(12), op: ExtentFlush { repair_id: ReconciliationId(12), extent_id: 151, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:03.903 INFO [0] client ExtentFlush { repair_id: ReconciliationId(12), extent_id: 151, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:03.903 INFO [1] received reconcile message
Sep 22 23:15:03.903 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(12), op: ExtentFlush { repair_id: ReconciliationId(12), extent_id: 151, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:03.903 INFO [1] client ExtentFlush { repair_id: ReconciliationId(12), extent_id: 151, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:03.903 INFO [2] received reconcile message
Sep 22 23:15:03.903 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(12), op: ExtentFlush { repair_id: ReconciliationId(12), extent_id: 151, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:03.903 INFO [2] client ExtentFlush { repair_id: ReconciliationId(12), extent_id: 151, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:03.903 DEBG 12 Flush extent 151 with f:2 g:2
Sep 22 23:15:03.903 DEBG Flush just extent 151 with f:2 and g:2
Sep 22 23:15:03.903 DEBG [1] It's time to notify for 12
Sep 22 23:15:03.903 INFO Completion from [1] id:12 status:true
Sep 22 23:15:03.903 INFO [13/752] Repair commands completed
Sep 22 23:15:03.903 INFO Pop front: ReconcileIO { id: ReconciliationId(13), op: ExtentClose { repair_id: ReconciliationId(13), extent_id: 151 }, state: ClientData([New, New, New]) }
Sep 22 23:15:03.903 INFO Sent repair work, now wait for resp
Sep 22 23:15:03.903 INFO [0] received reconcile message
Sep 22 23:15:03.903 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(13), op: ExtentClose { repair_id: ReconciliationId(13), extent_id: 151 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:03.903 INFO [0] client ExtentClose { repair_id: ReconciliationId(13), extent_id: 151 }
Sep 22 23:15:03.904 INFO [1] received reconcile message
Sep 22 23:15:03.904 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(13), op: ExtentClose { repair_id: ReconciliationId(13), extent_id: 151 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:03.904 INFO [1] client ExtentClose { repair_id: ReconciliationId(13), extent_id: 151 }
Sep 22 23:15:03.904 INFO [2] received reconcile message
Sep 22 23:15:03.904 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(13), op: ExtentClose { repair_id: ReconciliationId(13), extent_id: 151 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:03.904 INFO [2] client ExtentClose { repair_id: ReconciliationId(13), extent_id: 151 }
Sep 22 23:15:03.904 DEBG 13 Close extent 151
Sep 22 23:15:03.904 DEBG 13 Close extent 151
Sep 22 23:15:03.904 DEBG 13 Close extent 151
Sep 22 23:15:03.905 DEBG [2] It's time to notify for 13
Sep 22 23:15:03.905 INFO Completion from [2] id:13 status:true
Sep 22 23:15:03.905 INFO [14/752] Repair commands completed
Sep 22 23:15:03.905 INFO Pop front: ReconcileIO { id: ReconciliationId(14), op: ExtentRepair { repair_id: ReconciliationId(14), extent_id: 151, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:03.905 INFO Sent repair work, now wait for resp
Sep 22 23:15:03.905 INFO [0] received reconcile message
Sep 22 23:15:03.905 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(14), op: ExtentRepair { repair_id: ReconciliationId(14), extent_id: 151, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:03.905 INFO [0] client ExtentRepair { repair_id: ReconciliationId(14), extent_id: 151, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:03.905 INFO [0] Sending repair request ReconciliationId(14)
Sep 22 23:15:03.905 INFO [1] received reconcile message
Sep 22 23:15:03.905 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(14), op: ExtentRepair { repair_id: ReconciliationId(14), extent_id: 151, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:03.905 INFO [1] client ExtentRepair { repair_id: ReconciliationId(14), extent_id: 151, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:03.905 INFO [1] No action required ReconciliationId(14)
Sep 22 23:15:03.905 INFO [2] received reconcile message
Sep 22 23:15:03.905 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(14), op: ExtentRepair { repair_id: ReconciliationId(14), extent_id: 151, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:03.905 INFO [2] client ExtentRepair { repair_id: ReconciliationId(14), extent_id: 151, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:03.905 INFO [2] No action required ReconciliationId(14)
Sep 22 23:15:03.905 DEBG 14 Repair extent 151 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:03.905 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/097.copy"
Sep 22 23:15:03.913 DEBG [2] Read AckReady 1072, : downstairs
Sep 22 23:15:03.914 DEBG up_ds_listen was notified
Sep 22 23:15:03.914 DEBG up_ds_listen process 1072
Sep 22 23:15:03.914 DEBG [A] ack job 1072:73, : downstairs
Sep 22 23:15:03.968 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:03.969 INFO accepted connection, remote_addr: 127.0.0.1:56341, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:03.969 TRCE incoming request, uri: /extent/151/files, method: GET, req_id: b3d5e4bc-8f05-4b62-bb0b-9062dd73f389, remote_addr: 127.0.0.1:56341, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:03.969 INFO request completed, latency_us: 196, response_code: 200, uri: /extent/151/files, method: GET, req_id: b3d5e4bc-8f05-4b62-bb0b-9062dd73f389, remote_addr: 127.0.0.1:56341, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:03.969 INFO eid:151 Found repair files: ["097", "097.db"]
Sep 22 23:15:03.970 TRCE incoming request, uri: /newextent/151/data, method: GET, req_id: ad576779-3026-445d-8b4b-fe704a09030f, remote_addr: 127.0.0.1:56341, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:03.970 INFO request completed, latency_us: 307, response_code: 200, uri: /newextent/151/data, method: GET, req_id: ad576779-3026-445d-8b4b-fe704a09030f, remote_addr: 127.0.0.1:56341, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:03.973 DEBG Flush     :1071 extent_limit None deps:[JobId(1070), JobId(1069)] res:true f:25 g:1
Sep 22 23:15:03.973 INFO [lossy] skipping 1072
Sep 22 23:15:03.973 WARN 1073 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:15:03.975 TRCE incoming request, uri: /newextent/151/db, method: GET, req_id: b7ec581b-28f6-4e46-b0f7-43d84228c035, remote_addr: 127.0.0.1:56341, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:03.975 INFO request completed, latency_us: 291, response_code: 200, uri: /newextent/151/db, method: GET, req_id: b7ec581b-28f6-4e46-b0f7-43d84228c035, remote_addr: 127.0.0.1:56341, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:03.976 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/097.copy" to "/tmp/downstairs-vrx8aK6L/00/000/097.replace"
Sep 22 23:15:03.977 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:03.977 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/097.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:03.978 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/097"
Sep 22 23:15:03.978 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/097.db"
Sep 22 23:15:03.978 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:03.978 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/097.replace" to "/tmp/downstairs-vrx8aK6L/00/000/097.completed"
Sep 22 23:15:03.978 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:03.978 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:03.978 DEBG [0] It's time to notify for 14
Sep 22 23:15:03.978 INFO Completion from [0] id:14 status:true
Sep 22 23:15:03.978 INFO [15/752] Repair commands completed
Sep 22 23:15:03.978 INFO Pop front: ReconcileIO { id: ReconciliationId(15), op: ExtentReopen { repair_id: ReconciliationId(15), extent_id: 151 }, state: ClientData([New, New, New]) }
Sep 22 23:15:03.978 INFO Sent repair work, now wait for resp
Sep 22 23:15:03.978 INFO [0] received reconcile message
Sep 22 23:15:03.978 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(15), op: ExtentReopen { repair_id: ReconciliationId(15), extent_id: 151 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:03.978 INFO [0] client ExtentReopen { repair_id: ReconciliationId(15), extent_id: 151 }
Sep 22 23:15:03.978 INFO [1] received reconcile message
Sep 22 23:15:03.978 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(15), op: ExtentReopen { repair_id: ReconciliationId(15), extent_id: 151 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:03.979 INFO [1] client ExtentReopen { repair_id: ReconciliationId(15), extent_id: 151 }
Sep 22 23:15:03.979 INFO [2] received reconcile message
Sep 22 23:15:03.979 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(15), op: ExtentReopen { repair_id: ReconciliationId(15), extent_id: 151 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:03.979 INFO [2] client ExtentReopen { repair_id: ReconciliationId(15), extent_id: 151 }
Sep 22 23:15:03.979 DEBG 15 Reopen extent 151
Sep 22 23:15:03.979 DEBG Read      :1072 deps:[JobId(1071)] res:true
Sep 22 23:15:03.979 DEBG 15 Reopen extent 151
Sep 22 23:15:03.980 DEBG 15 Reopen extent 151
Sep 22 23:15:03.980 DEBG [2] It's time to notify for 15
Sep 22 23:15:03.980 INFO Completion from [2] id:15 status:true
Sep 22 23:15:03.980 INFO [16/752] Repair commands completed
Sep 22 23:15:03.980 INFO Pop front: ReconcileIO { id: ReconciliationId(16), op: ExtentFlush { repair_id: ReconciliationId(16), extent_id: 169, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:03.980 INFO Sent repair work, now wait for resp
Sep 22 23:15:03.981 INFO [0] received reconcile message
Sep 22 23:15:03.981 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(16), op: ExtentFlush { repair_id: ReconciliationId(16), extent_id: 169, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:03.981 INFO [0] client ExtentFlush { repair_id: ReconciliationId(16), extent_id: 169, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:03.981 INFO [1] received reconcile message
Sep 22 23:15:03.981 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(16), op: ExtentFlush { repair_id: ReconciliationId(16), extent_id: 169, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:03.981 INFO [1] client ExtentFlush { repair_id: ReconciliationId(16), extent_id: 169, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:03.981 INFO [2] received reconcile message
Sep 22 23:15:03.981 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(16), op: ExtentFlush { repair_id: ReconciliationId(16), extent_id: 169, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:03.981 INFO [2] client ExtentFlush { repair_id: ReconciliationId(16), extent_id: 169, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:03.981 DEBG 16 Flush extent 169 with f:2 g:2
Sep 22 23:15:03.981 DEBG Flush just extent 169 with f:2 and g:2
Sep 22 23:15:03.981 DEBG [1] It's time to notify for 16
Sep 22 23:15:03.981 INFO Completion from [1] id:16 status:true
Sep 22 23:15:03.981 INFO [17/752] Repair commands completed
Sep 22 23:15:03.981 INFO Pop front: ReconcileIO { id: ReconciliationId(17), op: ExtentClose { repair_id: ReconciliationId(17), extent_id: 169 }, state: ClientData([New, New, New]) }
Sep 22 23:15:03.981 INFO Sent repair work, now wait for resp
Sep 22 23:15:03.981 INFO [0] received reconcile message
Sep 22 23:15:03.981 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(17), op: ExtentClose { repair_id: ReconciliationId(17), extent_id: 169 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:03.981 INFO [0] client ExtentClose { repair_id: ReconciliationId(17), extent_id: 169 }
Sep 22 23:15:03.981 INFO [1] received reconcile message
Sep 22 23:15:03.981 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(17), op: ExtentClose { repair_id: ReconciliationId(17), extent_id: 169 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:03.981 INFO [1] client ExtentClose { repair_id: ReconciliationId(17), extent_id: 169 }
Sep 22 23:15:03.981 INFO [2] received reconcile message
Sep 22 23:15:03.981 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(17), op: ExtentClose { repair_id: ReconciliationId(17), extent_id: 169 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:03.981 INFO [2] client ExtentClose { repair_id: ReconciliationId(17), extent_id: 169 }
Sep 22 23:15:03.981 DEBG 17 Close extent 169
Sep 22 23:15:03.982 DEBG 17 Close extent 169
Sep 22 23:15:03.982 DEBG 17 Close extent 169
Sep 22 23:15:03.982 DEBG [2] It's time to notify for 17
Sep 22 23:15:03.982 INFO Completion from [2] id:17 status:true
Sep 22 23:15:03.982 INFO [18/752] Repair commands completed
Sep 22 23:15:03.982 INFO Pop front: ReconcileIO { id: ReconciliationId(18), op: ExtentRepair { repair_id: ReconciliationId(18), extent_id: 169, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:03.982 INFO Sent repair work, now wait for resp
Sep 22 23:15:03.982 INFO [0] received reconcile message
Sep 22 23:15:03.982 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(18), op: ExtentRepair { repair_id: ReconciliationId(18), extent_id: 169, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:03.982 INFO [0] client ExtentRepair { repair_id: ReconciliationId(18), extent_id: 169, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:03.983 INFO [0] Sending repair request ReconciliationId(18)
Sep 22 23:15:03.983 INFO [1] received reconcile message
Sep 22 23:15:03.983 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(18), op: ExtentRepair { repair_id: ReconciliationId(18), extent_id: 169, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:03.983 INFO [1] client ExtentRepair { repair_id: ReconciliationId(18), extent_id: 169, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:03.983 INFO [1] No action required ReconciliationId(18)
Sep 22 23:15:03.983 INFO [2] received reconcile message
Sep 22 23:15:03.983 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(18), op: ExtentRepair { repair_id: ReconciliationId(18), extent_id: 169, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:03.983 INFO [2] client ExtentRepair { repair_id: ReconciliationId(18), extent_id: 169, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:03.983 INFO [2] No action required ReconciliationId(18)
Sep 22 23:15:03.983 DEBG 18 Repair extent 169 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:03.983 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/0A9.copy"
Sep 22 23:15:04.001 DEBG IO Read  1074 has deps [JobId(1073)]
Sep 22 23:15:04.007 DEBG up_ds_listen was notified
Sep 22 23:15:04.007 DEBG up_ds_listen process 1071
Sep 22 23:15:04.007 DEBG [A] ack job 1071:72, : downstairs
Sep 22 23:15:04.007 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:04.016 WARN returning error on flush!
Sep 22 23:15:04.016 DEBG Flush     :1073 extent_limit None deps:[JobId(1072), JobId(1071)] res:false f:26 g:1
Sep 22 23:15:04.016 INFO [lossy] skipping 1074
Sep 22 23:15:04.016 WARN returning error on flush!
Sep 22 23:15:04.016 DEBG Flush     :1073 extent_limit None deps:[JobId(1072), JobId(1071)] res:false f:26 g:1
Sep 22 23:15:04.016 DEBG Flush     :1073 extent_limit None deps:[JobId(1072), JobId(1071)] res:true f:26 g:1
Sep 22 23:15:04.016 INFO [lossy] skipping 1074
Sep 22 23:15:04.016 INFO [lossy] skipping 1074
Sep 22 23:15:04.016 WARN returning error on read!
Sep 22 23:15:04.016 DEBG Read      :1074 deps:[JobId(1073)] res:false
Sep 22 23:15:04.016 WARN returning error on read!
Sep 22 23:15:04.016 DEBG Read      :1074 deps:[JobId(1073)] res:false
Sep 22 23:15:04.016 INFO [lossy] skipping 1074
Sep 22 23:15:04.016 INFO [lossy] skipping 1074
Sep 22 23:15:04.016 INFO [lossy] skipping 1074
Sep 22 23:15:04.022 DEBG Read      :1074 deps:[JobId(1073)] res:true
Sep 22 23:15:04.045 INFO accepted connection, remote_addr: 127.0.0.1:56054, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.045 TRCE incoming request, uri: /extent/169/files, method: GET, req_id: 399c34c7-e683-431a-95ad-4dd2f649d7c1, remote_addr: 127.0.0.1:56054, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.045 INFO request completed, latency_us: 238, response_code: 200, uri: /extent/169/files, method: GET, req_id: 399c34c7-e683-431a-95ad-4dd2f649d7c1, remote_addr: 127.0.0.1:56054, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.045 INFO eid:169 Found repair files: ["0A9", "0A9.db"]
Sep 22 23:15:04.046 TRCE incoming request, uri: /newextent/169/data, method: GET, req_id: f20cbb11-b32b-4c1e-a03e-8f710b489dd0, remote_addr: 127.0.0.1:56054, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.046 INFO request completed, latency_us: 341, response_code: 200, uri: /newextent/169/data, method: GET, req_id: f20cbb11-b32b-4c1e-a03e-8f710b489dd0, remote_addr: 127.0.0.1:56054, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.051 TRCE incoming request, uri: /newextent/169/db, method: GET, req_id: e2689a95-b2f4-4ac1-a4a3-b49b73593ab9, remote_addr: 127.0.0.1:56054, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.051 INFO request completed, latency_us: 290, response_code: 200, uri: /newextent/169/db, method: GET, req_id: e2689a95-b2f4-4ac1-a4a3-b49b73593ab9, remote_addr: 127.0.0.1:56054, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.053 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/0A9.copy" to "/tmp/downstairs-vrx8aK6L/00/000/0A9.replace"
Sep 22 23:15:04.053 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:04.054 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/0A9.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:04.054 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/0A9"
Sep 22 23:15:04.054 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/0A9.db"
Sep 22 23:15:04.054 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:04.054 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/0A9.replace" to "/tmp/downstairs-vrx8aK6L/00/000/0A9.completed"
Sep 22 23:15:04.054 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:04.054 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:04.054 DEBG [0] It's time to notify for 18
Sep 22 23:15:04.054 INFO Completion from [0] id:18 status:true
Sep 22 23:15:04.054 INFO [19/752] Repair commands completed
Sep 22 23:15:04.054 INFO Pop front: ReconcileIO { id: ReconciliationId(19), op: ExtentReopen { repair_id: ReconciliationId(19), extent_id: 169 }, state: ClientData([New, New, New]) }
Sep 22 23:15:04.054 INFO Sent repair work, now wait for resp
Sep 22 23:15:04.055 INFO [0] received reconcile message
Sep 22 23:15:04.055 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(19), op: ExtentReopen { repair_id: ReconciliationId(19), extent_id: 169 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:04.055 INFO [0] client ExtentReopen { repair_id: ReconciliationId(19), extent_id: 169 }
Sep 22 23:15:04.055 INFO [1] received reconcile message
Sep 22 23:15:04.055 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(19), op: ExtentReopen { repair_id: ReconciliationId(19), extent_id: 169 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:04.055 INFO [1] client ExtentReopen { repair_id: ReconciliationId(19), extent_id: 169 }
Sep 22 23:15:04.055 INFO [2] received reconcile message
Sep 22 23:15:04.055 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(19), op: ExtentReopen { repair_id: ReconciliationId(19), extent_id: 169 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:04.055 INFO [2] client ExtentReopen { repair_id: ReconciliationId(19), extent_id: 169 }
Sep 22 23:15:04.055 DEBG 19 Reopen extent 169
Sep 22 23:15:04.056 DEBG 19 Reopen extent 169
Sep 22 23:15:04.056 DEBG 19 Reopen extent 169
Sep 22 23:15:04.057 DEBG [2] It's time to notify for 19
Sep 22 23:15:04.057 INFO Completion from [2] id:19 status:true
Sep 22 23:15:04.057 INFO [20/752] Repair commands completed
Sep 22 23:15:04.057 INFO Pop front: ReconcileIO { id: ReconciliationId(20), op: ExtentFlush { repair_id: ReconciliationId(20), extent_id: 185, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:04.057 INFO Sent repair work, now wait for resp
Sep 22 23:15:04.057 INFO [0] received reconcile message
Sep 22 23:15:04.057 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(20), op: ExtentFlush { repair_id: ReconciliationId(20), extent_id: 185, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:04.057 INFO [0] client ExtentFlush { repair_id: ReconciliationId(20), extent_id: 185, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:04.057 INFO [1] received reconcile message
Sep 22 23:15:04.057 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(20), op: ExtentFlush { repair_id: ReconciliationId(20), extent_id: 185, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:04.057 INFO [1] client ExtentFlush { repair_id: ReconciliationId(20), extent_id: 185, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:04.057 INFO [2] received reconcile message
Sep 22 23:15:04.057 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(20), op: ExtentFlush { repair_id: ReconciliationId(20), extent_id: 185, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:04.057 INFO [2] client ExtentFlush { repair_id: ReconciliationId(20), extent_id: 185, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:04.057 DEBG 20 Flush extent 185 with f:2 g:2
Sep 22 23:15:04.057 DEBG Flush just extent 185 with f:2 and g:2
Sep 22 23:15:04.057 DEBG [1] It's time to notify for 20
Sep 22 23:15:04.057 INFO Completion from [1] id:20 status:true
Sep 22 23:15:04.057 INFO [21/752] Repair commands completed
Sep 22 23:15:04.057 INFO Pop front: ReconcileIO { id: ReconciliationId(21), op: ExtentClose { repair_id: ReconciliationId(21), extent_id: 185 }, state: ClientData([New, New, New]) }
Sep 22 23:15:04.057 INFO Sent repair work, now wait for resp
Sep 22 23:15:04.057 INFO [0] received reconcile message
Sep 22 23:15:04.057 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(21), op: ExtentClose { repair_id: ReconciliationId(21), extent_id: 185 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:04.057 INFO [0] client ExtentClose { repair_id: ReconciliationId(21), extent_id: 185 }
Sep 22 23:15:04.057 INFO [1] received reconcile message
Sep 22 23:15:04.057 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(21), op: ExtentClose { repair_id: ReconciliationId(21), extent_id: 185 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:04.057 INFO [1] client ExtentClose { repair_id: ReconciliationId(21), extent_id: 185 }
Sep 22 23:15:04.057 INFO [2] received reconcile message
Sep 22 23:15:04.057 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(21), op: ExtentClose { repair_id: ReconciliationId(21), extent_id: 185 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:04.057 INFO [2] client ExtentClose { repair_id: ReconciliationId(21), extent_id: 185 }
Sep 22 23:15:04.058 DEBG 21 Close extent 185
Sep 22 23:15:04.058 DEBG 21 Close extent 185
Sep 22 23:15:04.058 DEBG 21 Close extent 185
Sep 22 23:15:04.059 DEBG [2] It's time to notify for 21
Sep 22 23:15:04.059 INFO Completion from [2] id:21 status:true
Sep 22 23:15:04.059 INFO [22/752] Repair commands completed
Sep 22 23:15:04.059 INFO Pop front: ReconcileIO { id: ReconciliationId(22), op: ExtentRepair { repair_id: ReconciliationId(22), extent_id: 185, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:04.059 INFO Sent repair work, now wait for resp
Sep 22 23:15:04.059 INFO [0] received reconcile message
Sep 22 23:15:04.059 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(22), op: ExtentRepair { repair_id: ReconciliationId(22), extent_id: 185, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:04.059 INFO [0] client ExtentRepair { repair_id: ReconciliationId(22), extent_id: 185, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:04.059 INFO [0] Sending repair request ReconciliationId(22)
Sep 22 23:15:04.059 INFO [1] received reconcile message
Sep 22 23:15:04.059 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(22), op: ExtentRepair { repair_id: ReconciliationId(22), extent_id: 185, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:04.059 INFO [1] client ExtentRepair { repair_id: ReconciliationId(22), extent_id: 185, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:04.059 INFO [1] No action required ReconciliationId(22)
Sep 22 23:15:04.059 INFO [2] received reconcile message
Sep 22 23:15:04.059 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(22), op: ExtentRepair { repair_id: ReconciliationId(22), extent_id: 185, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:04.059 INFO [2] client ExtentRepair { repair_id: ReconciliationId(22), extent_id: 185, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:04.059 INFO [2] No action required ReconciliationId(22)
Sep 22 23:15:04.059 DEBG 22 Repair extent 185 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:04.059 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/0B9.copy"
Sep 22 23:15:04.120 INFO accepted connection, remote_addr: 127.0.0.1:56651, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.120 TRCE incoming request, uri: /extent/185/files, method: GET, req_id: b191d31f-80ce-431d-a61f-c83308a9b1fb, remote_addr: 127.0.0.1:56651, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.120 INFO request completed, latency_us: 189, response_code: 200, uri: /extent/185/files, method: GET, req_id: b191d31f-80ce-431d-a61f-c83308a9b1fb, remote_addr: 127.0.0.1:56651, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.120 INFO eid:185 Found repair files: ["0B9", "0B9.db"]
Sep 22 23:15:04.121 TRCE incoming request, uri: /newextent/185/data, method: GET, req_id: 59dab808-59f2-4be0-905f-fb02a78b7119, remote_addr: 127.0.0.1:56651, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.121 INFO request completed, latency_us: 301, response_code: 200, uri: /newextent/185/data, method: GET, req_id: 59dab808-59f2-4be0-905f-fb02a78b7119, remote_addr: 127.0.0.1:56651, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.126 TRCE incoming request, uri: /newextent/185/db, method: GET, req_id: 54da01a1-4b13-4299-9283-5b14fbf75b16, remote_addr: 127.0.0.1:56651, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.126 INFO request completed, latency_us: 291, response_code: 200, uri: /newextent/185/db, method: GET, req_id: 54da01a1-4b13-4299-9283-5b14fbf75b16, remote_addr: 127.0.0.1:56651, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.128 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/0B9.copy" to "/tmp/downstairs-vrx8aK6L/00/000/0B9.replace"
Sep 22 23:15:04.128 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:04.128 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/0B9.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:04.129 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/0B9"
Sep 22 23:15:04.129 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/0B9.db"
Sep 22 23:15:04.129 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:04.129 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/0B9.replace" to "/tmp/downstairs-vrx8aK6L/00/000/0B9.completed"
Sep 22 23:15:04.129 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:04.129 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:04.129 DEBG [0] It's time to notify for 22
Sep 22 23:15:04.129 INFO Completion from [0] id:22 status:true
Sep 22 23:15:04.129 INFO [23/752] Repair commands completed
Sep 22 23:15:04.129 INFO Pop front: ReconcileIO { id: ReconciliationId(23), op: ExtentReopen { repair_id: ReconciliationId(23), extent_id: 185 }, state: ClientData([New, New, New]) }
Sep 22 23:15:04.129 INFO Sent repair work, now wait for resp
Sep 22 23:15:04.129 INFO [0] received reconcile message
Sep 22 23:15:04.129 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(23), op: ExtentReopen { repair_id: ReconciliationId(23), extent_id: 185 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:04.129 INFO [0] client ExtentReopen { repair_id: ReconciliationId(23), extent_id: 185 }
Sep 22 23:15:04.129 INFO [1] received reconcile message
Sep 22 23:15:04.129 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(23), op: ExtentReopen { repair_id: ReconciliationId(23), extent_id: 185 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:04.129 INFO [1] client ExtentReopen { repair_id: ReconciliationId(23), extent_id: 185 }
Sep 22 23:15:04.129 INFO [2] received reconcile message
Sep 22 23:15:04.129 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(23), op: ExtentReopen { repair_id: ReconciliationId(23), extent_id: 185 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:04.129 INFO [2] client ExtentReopen { repair_id: ReconciliationId(23), extent_id: 185 }
Sep 22 23:15:04.130 DEBG 23 Reopen extent 185
Sep 22 23:15:04.130 DEBG 23 Reopen extent 185
Sep 22 23:15:04.131 DEBG 23 Reopen extent 185
Sep 22 23:15:04.131 DEBG [2] It's time to notify for 23
Sep 22 23:15:04.131 INFO Completion from [2] id:23 status:true
Sep 22 23:15:04.131 INFO [24/752] Repair commands completed
Sep 22 23:15:04.131 INFO Pop front: ReconcileIO { id: ReconciliationId(24), op: ExtentFlush { repair_id: ReconciliationId(24), extent_id: 11, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:04.131 INFO Sent repair work, now wait for resp
Sep 22 23:15:04.131 INFO [0] received reconcile message
Sep 22 23:15:04.131 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(24), op: ExtentFlush { repair_id: ReconciliationId(24), extent_id: 11, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:04.131 INFO [0] client ExtentFlush { repair_id: ReconciliationId(24), extent_id: 11, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:04.131 INFO [1] received reconcile message
Sep 22 23:15:04.131 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(24), op: ExtentFlush { repair_id: ReconciliationId(24), extent_id: 11, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:04.131 INFO [1] client ExtentFlush { repair_id: ReconciliationId(24), extent_id: 11, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:04.131 INFO [2] received reconcile message
Sep 22 23:15:04.131 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(24), op: ExtentFlush { repair_id: ReconciliationId(24), extent_id: 11, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:04.131 INFO [2] client ExtentFlush { repair_id: ReconciliationId(24), extent_id: 11, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:04.132 DEBG 24 Flush extent 11 with f:2 g:2
Sep 22 23:15:04.132 DEBG Flush just extent 11 with f:2 and g:2
Sep 22 23:15:04.132 DEBG [1] It's time to notify for 24
Sep 22 23:15:04.132 INFO Completion from [1] id:24 status:true
Sep 22 23:15:04.132 INFO [25/752] Repair commands completed
Sep 22 23:15:04.132 INFO Pop front: ReconcileIO { id: ReconciliationId(25), op: ExtentClose { repair_id: ReconciliationId(25), extent_id: 11 }, state: ClientData([New, New, New]) }
Sep 22 23:15:04.132 INFO Sent repair work, now wait for resp
Sep 22 23:15:04.132 INFO [0] received reconcile message
Sep 22 23:15:04.132 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(25), op: ExtentClose { repair_id: ReconciliationId(25), extent_id: 11 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:04.132 INFO [0] client ExtentClose { repair_id: ReconciliationId(25), extent_id: 11 }
Sep 22 23:15:04.132 INFO [1] received reconcile message
Sep 22 23:15:04.132 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(25), op: ExtentClose { repair_id: ReconciliationId(25), extent_id: 11 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:04.132 INFO [1] client ExtentClose { repair_id: ReconciliationId(25), extent_id: 11 }
Sep 22 23:15:04.132 INFO [2] received reconcile message
Sep 22 23:15:04.132 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(25), op: ExtentClose { repair_id: ReconciliationId(25), extent_id: 11 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:04.132 INFO [2] client ExtentClose { repair_id: ReconciliationId(25), extent_id: 11 }
Sep 22 23:15:04.132 DEBG 25 Close extent 11
Sep 22 23:15:04.132 DEBG 25 Close extent 11
Sep 22 23:15:04.133 DEBG 25 Close extent 11
Sep 22 23:15:04.133 DEBG [2] It's time to notify for 25
Sep 22 23:15:04.133 INFO Completion from [2] id:25 status:true
Sep 22 23:15:04.133 INFO [26/752] Repair commands completed
Sep 22 23:15:04.133 INFO Pop front: ReconcileIO { id: ReconciliationId(26), op: ExtentRepair { repair_id: ReconciliationId(26), extent_id: 11, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:04.133 INFO Sent repair work, now wait for resp
Sep 22 23:15:04.133 INFO [0] received reconcile message
Sep 22 23:15:04.133 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(26), op: ExtentRepair { repair_id: ReconciliationId(26), extent_id: 11, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:04.133 INFO [0] client ExtentRepair { repair_id: ReconciliationId(26), extent_id: 11, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:04.133 INFO [0] Sending repair request ReconciliationId(26)
Sep 22 23:15:04.133 INFO [1] received reconcile message
Sep 22 23:15:04.133 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(26), op: ExtentRepair { repair_id: ReconciliationId(26), extent_id: 11, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:04.133 INFO [1] client ExtentRepair { repair_id: ReconciliationId(26), extent_id: 11, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:04.133 INFO [1] No action required ReconciliationId(26)
Sep 22 23:15:04.133 INFO [2] received reconcile message
Sep 22 23:15:04.133 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(26), op: ExtentRepair { repair_id: ReconciliationId(26), extent_id: 11, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:04.133 INFO [2] client ExtentRepair { repair_id: ReconciliationId(26), extent_id: 11, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:04.134 INFO [2] No action required ReconciliationId(26)
Sep 22 23:15:04.134 DEBG 26 Repair extent 11 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:04.134 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/00B.copy"
Sep 22 23:15:04.197 INFO accepted connection, remote_addr: 127.0.0.1:51187, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.197 TRCE incoming request, uri: /extent/11/files, method: GET, req_id: 7be2ad5a-a85f-4792-8154-5b18504ceefc, remote_addr: 127.0.0.1:51187, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.197 INFO request completed, latency_us: 198, response_code: 200, uri: /extent/11/files, method: GET, req_id: 7be2ad5a-a85f-4792-8154-5b18504ceefc, remote_addr: 127.0.0.1:51187, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.197 INFO eid:11 Found repair files: ["00B", "00B.db"]
Sep 22 23:15:04.198 TRCE incoming request, uri: /newextent/11/data, method: GET, req_id: 71e899e6-2917-4aad-ae54-74e9c21fccbd, remote_addr: 127.0.0.1:51187, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.198 INFO request completed, latency_us: 324, response_code: 200, uri: /newextent/11/data, method: GET, req_id: 71e899e6-2917-4aad-ae54-74e9c21fccbd, remote_addr: 127.0.0.1:51187, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.203 TRCE incoming request, uri: /newextent/11/db, method: GET, req_id: 2d389c87-3b92-4ac1-a8bd-9e76b06b1c1c, remote_addr: 127.0.0.1:51187, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.203 INFO request completed, latency_us: 290, response_code: 200, uri: /newextent/11/db, method: GET, req_id: 2d389c87-3b92-4ac1-a8bd-9e76b06b1c1c, remote_addr: 127.0.0.1:51187, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.204 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/00B.copy" to "/tmp/downstairs-vrx8aK6L/00/000/00B.replace"
Sep 22 23:15:04.204 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:04.205 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/00B.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:04.206 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/00B"
Sep 22 23:15:04.206 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/00B.db"
Sep 22 23:15:04.206 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:04.206 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/00B.replace" to "/tmp/downstairs-vrx8aK6L/00/000/00B.completed"
Sep 22 23:15:04.206 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:04.206 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:04.206 DEBG [0] It's time to notify for 26
Sep 22 23:15:04.206 INFO Completion from [0] id:26 status:true
Sep 22 23:15:04.206 INFO [27/752] Repair commands completed
Sep 22 23:15:04.206 INFO Pop front: ReconcileIO { id: ReconciliationId(27), op: ExtentReopen { repair_id: ReconciliationId(27), extent_id: 11 }, state: ClientData([New, New, New]) }
Sep 22 23:15:04.206 INFO Sent repair work, now wait for resp
Sep 22 23:15:04.206 INFO [0] received reconcile message
Sep 22 23:15:04.206 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(27), op: ExtentReopen { repair_id: ReconciliationId(27), extent_id: 11 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:04.206 INFO [0] client ExtentReopen { repair_id: ReconciliationId(27), extent_id: 11 }
Sep 22 23:15:04.206 INFO [1] received reconcile message
Sep 22 23:15:04.206 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(27), op: ExtentReopen { repair_id: ReconciliationId(27), extent_id: 11 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:04.206 INFO [1] client ExtentReopen { repair_id: ReconciliationId(27), extent_id: 11 }
Sep 22 23:15:04.206 INFO [2] received reconcile message
Sep 22 23:15:04.206 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(27), op: ExtentReopen { repair_id: ReconciliationId(27), extent_id: 11 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:04.206 INFO [2] client ExtentReopen { repair_id: ReconciliationId(27), extent_id: 11 }
Sep 22 23:15:04.207 DEBG 27 Reopen extent 11
Sep 22 23:15:04.207 DEBG 27 Reopen extent 11
Sep 22 23:15:04.208 DEBG 27 Reopen extent 11
Sep 22 23:15:04.208 DEBG [2] It's time to notify for 27
Sep 22 23:15:04.208 INFO Completion from [2] id:27 status:true
Sep 22 23:15:04.208 INFO [28/752] Repair commands completed
Sep 22 23:15:04.208 INFO Pop front: ReconcileIO { id: ReconciliationId(28), op: ExtentFlush { repair_id: ReconciliationId(28), extent_id: 16, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:04.208 INFO Sent repair work, now wait for resp
Sep 22 23:15:04.208 INFO [0] received reconcile message
Sep 22 23:15:04.208 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(28), op: ExtentFlush { repair_id: ReconciliationId(28), extent_id: 16, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:04.208 INFO [0] client ExtentFlush { repair_id: ReconciliationId(28), extent_id: 16, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:04.208 INFO [1] received reconcile message
Sep 22 23:15:04.208 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(28), op: ExtentFlush { repair_id: ReconciliationId(28), extent_id: 16, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:04.208 INFO [1] client ExtentFlush { repair_id: ReconciliationId(28), extent_id: 16, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:04.209 INFO [2] received reconcile message
Sep 22 23:15:04.209 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(28), op: ExtentFlush { repair_id: ReconciliationId(28), extent_id: 16, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:04.209 INFO [2] client ExtentFlush { repair_id: ReconciliationId(28), extent_id: 16, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:04.209 DEBG 28 Flush extent 16 with f:2 g:2
Sep 22 23:15:04.209 DEBG Flush just extent 16 with f:2 and g:2
Sep 22 23:15:04.209 DEBG [1] It's time to notify for 28
Sep 22 23:15:04.209 INFO Completion from [1] id:28 status:true
Sep 22 23:15:04.209 INFO [29/752] Repair commands completed
Sep 22 23:15:04.209 INFO Pop front: ReconcileIO { id: ReconciliationId(29), op: ExtentClose { repair_id: ReconciliationId(29), extent_id: 16 }, state: ClientData([New, New, New]) }
Sep 22 23:15:04.209 INFO Sent repair work, now wait for resp
Sep 22 23:15:04.209 INFO [0] received reconcile message
Sep 22 23:15:04.209 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(29), op: ExtentClose { repair_id: ReconciliationId(29), extent_id: 16 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:04.209 INFO [0] client ExtentClose { repair_id: ReconciliationId(29), extent_id: 16 }
Sep 22 23:15:04.209 INFO [1] received reconcile message
Sep 22 23:15:04.209 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(29), op: ExtentClose { repair_id: ReconciliationId(29), extent_id: 16 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:04.209 INFO [1] client ExtentClose { repair_id: ReconciliationId(29), extent_id: 16 }
Sep 22 23:15:04.209 INFO [2] received reconcile message
Sep 22 23:15:04.209 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(29), op: ExtentClose { repair_id: ReconciliationId(29), extent_id: 16 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:04.209 INFO [2] client ExtentClose { repair_id: ReconciliationId(29), extent_id: 16 }
Sep 22 23:15:04.209 DEBG 29 Close extent 16
Sep 22 23:15:04.209 DEBG 29 Close extent 16
Sep 22 23:15:04.210 DEBG 29 Close extent 16
Sep 22 23:15:04.210 DEBG [2] It's time to notify for 29
Sep 22 23:15:04.210 INFO Completion from [2] id:29 status:true
Sep 22 23:15:04.210 INFO [30/752] Repair commands completed
Sep 22 23:15:04.210 INFO Pop front: ReconcileIO { id: ReconciliationId(30), op: ExtentRepair { repair_id: ReconciliationId(30), extent_id: 16, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:04.210 INFO Sent repair work, now wait for resp
Sep 22 23:15:04.210 INFO [0] received reconcile message
Sep 22 23:15:04.210 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(30), op: ExtentRepair { repair_id: ReconciliationId(30), extent_id: 16, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:04.210 INFO [0] client ExtentRepair { repair_id: ReconciliationId(30), extent_id: 16, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:04.210 INFO [0] Sending repair request ReconciliationId(30)
Sep 22 23:15:04.210 INFO [1] received reconcile message
Sep 22 23:15:04.210 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(30), op: ExtentRepair { repair_id: ReconciliationId(30), extent_id: 16, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:04.210 INFO [1] client ExtentRepair { repair_id: ReconciliationId(30), extent_id: 16, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:04.210 INFO [1] No action required ReconciliationId(30)
Sep 22 23:15:04.211 INFO [2] received reconcile message
Sep 22 23:15:04.211 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(30), op: ExtentRepair { repair_id: ReconciliationId(30), extent_id: 16, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:04.211 INFO [2] client ExtentRepair { repair_id: ReconciliationId(30), extent_id: 16, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:04.211 INFO [2] No action required ReconciliationId(30)
Sep 22 23:15:04.211 DEBG 30 Repair extent 16 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:04.211 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/010.copy"
Sep 22 23:15:04.275 INFO accepted connection, remote_addr: 127.0.0.1:41743, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.275 TRCE incoming request, uri: /extent/16/files, method: GET, req_id: 70672ca2-f83d-4706-8879-e6dab8b2c87b, remote_addr: 127.0.0.1:41743, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.275 INFO request completed, latency_us: 203, response_code: 200, uri: /extent/16/files, method: GET, req_id: 70672ca2-f83d-4706-8879-e6dab8b2c87b, remote_addr: 127.0.0.1:41743, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.275 INFO eid:16 Found repair files: ["010", "010.db"]
Sep 22 23:15:04.276 TRCE incoming request, uri: /newextent/16/data, method: GET, req_id: c4c672da-1bc0-4b33-843f-a79992d3a7ec, remote_addr: 127.0.0.1:41743, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.276 INFO request completed, latency_us: 306, response_code: 200, uri: /newextent/16/data, method: GET, req_id: c4c672da-1bc0-4b33-843f-a79992d3a7ec, remote_addr: 127.0.0.1:41743, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.281 TRCE incoming request, uri: /newextent/16/db, method: GET, req_id: 34bfd9ff-fbb1-4ae5-bf6d-0c17fdd2a155, remote_addr: 127.0.0.1:41743, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.281 INFO request completed, latency_us: 292, response_code: 200, uri: /newextent/16/db, method: GET, req_id: 34bfd9ff-fbb1-4ae5-bf6d-0c17fdd2a155, remote_addr: 127.0.0.1:41743, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.282 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/010.copy" to "/tmp/downstairs-vrx8aK6L/00/000/010.replace"
Sep 22 23:15:04.282 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:04.283 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/010.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:04.283 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/010"
Sep 22 23:15:04.284 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/010.db"
Sep 22 23:15:04.284 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:04.284 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/010.replace" to "/tmp/downstairs-vrx8aK6L/00/000/010.completed"
Sep 22 23:15:04.284 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:04.284 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:04.284 DEBG [0] It's time to notify for 30
Sep 22 23:15:04.284 INFO Completion from [0] id:30 status:true
Sep 22 23:15:04.284 INFO [31/752] Repair commands completed
Sep 22 23:15:04.284 INFO Pop front: ReconcileIO { id: ReconciliationId(31), op: ExtentReopen { repair_id: ReconciliationId(31), extent_id: 16 }, state: ClientData([New, New, New]) }
Sep 22 23:15:04.284 INFO Sent repair work, now wait for resp
Sep 22 23:15:04.284 INFO [0] received reconcile message
Sep 22 23:15:04.284 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(31), op: ExtentReopen { repair_id: ReconciliationId(31), extent_id: 16 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:04.284 INFO [0] client ExtentReopen { repair_id: ReconciliationId(31), extent_id: 16 }
Sep 22 23:15:04.284 INFO [1] received reconcile message
Sep 22 23:15:04.284 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(31), op: ExtentReopen { repair_id: ReconciliationId(31), extent_id: 16 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:04.284 INFO [1] client ExtentReopen { repair_id: ReconciliationId(31), extent_id: 16 }
Sep 22 23:15:04.284 INFO [2] received reconcile message
Sep 22 23:15:04.284 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(31), op: ExtentReopen { repair_id: ReconciliationId(31), extent_id: 16 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:04.284 INFO [2] client ExtentReopen { repair_id: ReconciliationId(31), extent_id: 16 }
Sep 22 23:15:04.284 DEBG 31 Reopen extent 16
Sep 22 23:15:04.285 DEBG 31 Reopen extent 16
Sep 22 23:15:04.286 DEBG 31 Reopen extent 16
Sep 22 23:15:04.286 DEBG [2] It's time to notify for 31
Sep 22 23:15:04.286 INFO Completion from [2] id:31 status:true
Sep 22 23:15:04.286 INFO [32/752] Repair commands completed
Sep 22 23:15:04.286 INFO Pop front: ReconcileIO { id: ReconciliationId(32), op: ExtentFlush { repair_id: ReconciliationId(32), extent_id: 67, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:04.286 INFO Sent repair work, now wait for resp
Sep 22 23:15:04.286 INFO [0] received reconcile message
Sep 22 23:15:04.286 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(32), op: ExtentFlush { repair_id: ReconciliationId(32), extent_id: 67, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:04.286 INFO [0] client ExtentFlush { repair_id: ReconciliationId(32), extent_id: 67, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:04.286 INFO [1] received reconcile message
Sep 22 23:15:04.286 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(32), op: ExtentFlush { repair_id: ReconciliationId(32), extent_id: 67, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:04.286 INFO [1] client ExtentFlush { repair_id: ReconciliationId(32), extent_id: 67, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:04.286 INFO [2] received reconcile message
Sep 22 23:15:04.286 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(32), op: ExtentFlush { repair_id: ReconciliationId(32), extent_id: 67, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:04.286 INFO [2] client ExtentFlush { repair_id: ReconciliationId(32), extent_id: 67, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:04.287 DEBG 32 Flush extent 67 with f:2 g:2
Sep 22 23:15:04.287 DEBG Flush just extent 67 with f:2 and g:2
Sep 22 23:15:04.287 DEBG [1] It's time to notify for 32
Sep 22 23:15:04.287 INFO Completion from [1] id:32 status:true
Sep 22 23:15:04.287 INFO [33/752] Repair commands completed
Sep 22 23:15:04.287 INFO Pop front: ReconcileIO { id: ReconciliationId(33), op: ExtentClose { repair_id: ReconciliationId(33), extent_id: 67 }, state: ClientData([New, New, New]) }
Sep 22 23:15:04.287 INFO Sent repair work, now wait for resp
Sep 22 23:15:04.287 INFO [0] received reconcile message
Sep 22 23:15:04.287 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(33), op: ExtentClose { repair_id: ReconciliationId(33), extent_id: 67 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:04.287 INFO [0] client ExtentClose { repair_id: ReconciliationId(33), extent_id: 67 }
Sep 22 23:15:04.287 INFO [1] received reconcile message
Sep 22 23:15:04.287 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(33), op: ExtentClose { repair_id: ReconciliationId(33), extent_id: 67 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:04.287 INFO [1] client ExtentClose { repair_id: ReconciliationId(33), extent_id: 67 }
Sep 22 23:15:04.287 INFO [2] received reconcile message
Sep 22 23:15:04.287 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(33), op: ExtentClose { repair_id: ReconciliationId(33), extent_id: 67 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:04.287 INFO [2] client ExtentClose { repair_id: ReconciliationId(33), extent_id: 67 }
Sep 22 23:15:04.287 DEBG 33 Close extent 67
Sep 22 23:15:04.287 DEBG 33 Close extent 67
Sep 22 23:15:04.288 DEBG 33 Close extent 67
Sep 22 23:15:04.288 DEBG [2] It's time to notify for 33
Sep 22 23:15:04.288 INFO Completion from [2] id:33 status:true
Sep 22 23:15:04.288 INFO [34/752] Repair commands completed
Sep 22 23:15:04.288 INFO Pop front: ReconcileIO { id: ReconciliationId(34), op: ExtentRepair { repair_id: ReconciliationId(34), extent_id: 67, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:04.288 INFO Sent repair work, now wait for resp
Sep 22 23:15:04.288 INFO [0] received reconcile message
Sep 22 23:15:04.288 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(34), op: ExtentRepair { repair_id: ReconciliationId(34), extent_id: 67, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:04.288 INFO [0] client ExtentRepair { repair_id: ReconciliationId(34), extent_id: 67, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:04.288 INFO [0] Sending repair request ReconciliationId(34)
Sep 22 23:15:04.288 INFO [1] received reconcile message
Sep 22 23:15:04.288 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(34), op: ExtentRepair { repair_id: ReconciliationId(34), extent_id: 67, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:04.288 INFO [1] client ExtentRepair { repair_id: ReconciliationId(34), extent_id: 67, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:04.288 INFO [1] No action required ReconciliationId(34)
Sep 22 23:15:04.288 INFO [2] received reconcile message
Sep 22 23:15:04.288 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(34), op: ExtentRepair { repair_id: ReconciliationId(34), extent_id: 67, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:04.288 INFO [2] client ExtentRepair { repair_id: ReconciliationId(34), extent_id: 67, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:04.288 INFO [2] No action required ReconciliationId(34)
Sep 22 23:15:04.289 DEBG 34 Repair extent 67 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:04.289 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/043.copy"
Sep 22 23:15:04.352 INFO accepted connection, remote_addr: 127.0.0.1:46377, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.353 TRCE incoming request, uri: /extent/67/files, method: GET, req_id: b6f10fa0-643d-4aa6-9748-bd9e27813086, remote_addr: 127.0.0.1:46377, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.353 INFO request completed, latency_us: 201, response_code: 200, uri: /extent/67/files, method: GET, req_id: b6f10fa0-643d-4aa6-9748-bd9e27813086, remote_addr: 127.0.0.1:46377, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.353 INFO eid:67 Found repair files: ["043", "043.db"]
Sep 22 23:15:04.353 TRCE incoming request, uri: /newextent/67/data, method: GET, req_id: dcfe8f0c-cb29-4033-a986-d3f6dc563319, remote_addr: 127.0.0.1:46377, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.354 INFO request completed, latency_us: 328, response_code: 200, uri: /newextent/67/data, method: GET, req_id: dcfe8f0c-cb29-4033-a986-d3f6dc563319, remote_addr: 127.0.0.1:46377, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.359 TRCE incoming request, uri: /newextent/67/db, method: GET, req_id: 17332c8d-50b9-431f-ab77-cce357c047d7, remote_addr: 127.0.0.1:46377, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.359 INFO request completed, latency_us: 292, response_code: 200, uri: /newextent/67/db, method: GET, req_id: 17332c8d-50b9-431f-ab77-cce357c047d7, remote_addr: 127.0.0.1:46377, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.360 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/043.copy" to "/tmp/downstairs-vrx8aK6L/00/000/043.replace"
Sep 22 23:15:04.360 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:04.361 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/043.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:04.361 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/043"
Sep 22 23:15:04.361 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/043.db"
Sep 22 23:15:04.361 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:04.361 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/043.replace" to "/tmp/downstairs-vrx8aK6L/00/000/043.completed"
Sep 22 23:15:04.361 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:04.362 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:04.362 DEBG [0] It's time to notify for 34
Sep 22 23:15:04.362 INFO Completion from [0] id:34 status:true
Sep 22 23:15:04.362 INFO [35/752] Repair commands completed
Sep 22 23:15:04.362 INFO Pop front: ReconcileIO { id: ReconciliationId(35), op: ExtentReopen { repair_id: ReconciliationId(35), extent_id: 67 }, state: ClientData([New, New, New]) }
Sep 22 23:15:04.362 INFO Sent repair work, now wait for resp
Sep 22 23:15:04.362 INFO [0] received reconcile message
Sep 22 23:15:04.362 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(35), op: ExtentReopen { repair_id: ReconciliationId(35), extent_id: 67 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:04.362 INFO [0] client ExtentReopen { repair_id: ReconciliationId(35), extent_id: 67 }
Sep 22 23:15:04.362 INFO [1] received reconcile message
Sep 22 23:15:04.362 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(35), op: ExtentReopen { repair_id: ReconciliationId(35), extent_id: 67 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:04.362 INFO [1] client ExtentReopen { repair_id: ReconciliationId(35), extent_id: 67 }
Sep 22 23:15:04.362 INFO [2] received reconcile message
Sep 22 23:15:04.362 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(35), op: ExtentReopen { repair_id: ReconciliationId(35), extent_id: 67 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:04.362 INFO [2] client ExtentReopen { repair_id: ReconciliationId(35), extent_id: 67 }
Sep 22 23:15:04.362 DEBG 35 Reopen extent 67
Sep 22 23:15:04.363 DEBG 35 Reopen extent 67
Sep 22 23:15:04.363 DEBG 35 Reopen extent 67
Sep 22 23:15:04.364 DEBG [2] It's time to notify for 35
Sep 22 23:15:04.364 INFO Completion from [2] id:35 status:true
Sep 22 23:15:04.364 INFO [36/752] Repair commands completed
Sep 22 23:15:04.364 INFO Pop front: ReconcileIO { id: ReconciliationId(36), op: ExtentFlush { repair_id: ReconciliationId(36), extent_id: 137, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:04.364 INFO Sent repair work, now wait for resp
Sep 22 23:15:04.364 INFO [0] received reconcile message
Sep 22 23:15:04.364 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(36), op: ExtentFlush { repair_id: ReconciliationId(36), extent_id: 137, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:04.364 INFO [0] client ExtentFlush { repair_id: ReconciliationId(36), extent_id: 137, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:04.364 INFO [1] received reconcile message
Sep 22 23:15:04.364 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(36), op: ExtentFlush { repair_id: ReconciliationId(36), extent_id: 137, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:04.364 INFO [1] client ExtentFlush { repair_id: ReconciliationId(36), extent_id: 137, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:04.364 INFO [2] received reconcile message
Sep 22 23:15:04.364 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(36), op: ExtentFlush { repair_id: ReconciliationId(36), extent_id: 137, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:04.364 INFO [2] client ExtentFlush { repair_id: ReconciliationId(36), extent_id: 137, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:04.364 DEBG 36 Flush extent 137 with f:2 g:2
Sep 22 23:15:04.364 DEBG Flush just extent 137 with f:2 and g:2
Sep 22 23:15:04.365 DEBG [1] It's time to notify for 36
Sep 22 23:15:04.365 INFO Completion from [1] id:36 status:true
Sep 22 23:15:04.365 INFO [37/752] Repair commands completed
Sep 22 23:15:04.365 INFO Pop front: ReconcileIO { id: ReconciliationId(37), op: ExtentClose { repair_id: ReconciliationId(37), extent_id: 137 }, state: ClientData([New, New, New]) }
Sep 22 23:15:04.365 INFO Sent repair work, now wait for resp
Sep 22 23:15:04.365 INFO [0] received reconcile message
Sep 22 23:15:04.365 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(37), op: ExtentClose { repair_id: ReconciliationId(37), extent_id: 137 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:04.365 INFO [0] client ExtentClose { repair_id: ReconciliationId(37), extent_id: 137 }
Sep 22 23:15:04.365 INFO [1] received reconcile message
Sep 22 23:15:04.365 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(37), op: ExtentClose { repair_id: ReconciliationId(37), extent_id: 137 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:04.365 INFO [1] client ExtentClose { repair_id: ReconciliationId(37), extent_id: 137 }
Sep 22 23:15:04.365 INFO [2] received reconcile message
Sep 22 23:15:04.365 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(37), op: ExtentClose { repair_id: ReconciliationId(37), extent_id: 137 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:04.365 INFO [2] client ExtentClose { repair_id: ReconciliationId(37), extent_id: 137 }
Sep 22 23:15:04.365 DEBG 37 Close extent 137
Sep 22 23:15:04.365 DEBG 37 Close extent 137
Sep 22 23:15:04.366 DEBG 37 Close extent 137
Sep 22 23:15:04.366 DEBG [2] It's time to notify for 37
Sep 22 23:15:04.366 INFO Completion from [2] id:37 status:true
Sep 22 23:15:04.366 INFO [38/752] Repair commands completed
Sep 22 23:15:04.366 INFO Pop front: ReconcileIO { id: ReconciliationId(38), op: ExtentRepair { repair_id: ReconciliationId(38), extent_id: 137, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:04.366 INFO Sent repair work, now wait for resp
Sep 22 23:15:04.366 INFO [0] received reconcile message
Sep 22 23:15:04.366 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(38), op: ExtentRepair { repair_id: ReconciliationId(38), extent_id: 137, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:04.366 INFO [0] client ExtentRepair { repair_id: ReconciliationId(38), extent_id: 137, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:04.366 INFO [0] Sending repair request ReconciliationId(38)
Sep 22 23:15:04.366 INFO [1] received reconcile message
Sep 22 23:15:04.366 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(38), op: ExtentRepair { repair_id: ReconciliationId(38), extent_id: 137, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:04.366 INFO [1] client ExtentRepair { repair_id: ReconciliationId(38), extent_id: 137, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:04.366 INFO [1] No action required ReconciliationId(38)
Sep 22 23:15:04.366 INFO [2] received reconcile message
Sep 22 23:15:04.366 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(38), op: ExtentRepair { repair_id: ReconciliationId(38), extent_id: 137, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:04.366 INFO [2] client ExtentRepair { repair_id: ReconciliationId(38), extent_id: 137, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:04.366 INFO [2] No action required ReconciliationId(38)
Sep 22 23:15:04.366 DEBG 38 Repair extent 137 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:04.366 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/089.copy"
Sep 22 23:15:04.426 INFO accepted connection, remote_addr: 127.0.0.1:36872, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.426 TRCE incoming request, uri: /extent/137/files, method: GET, req_id: 23b41528-ab64-4da8-a3b7-43cfbefd3844, remote_addr: 127.0.0.1:36872, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.426 INFO request completed, latency_us: 194, response_code: 200, uri: /extent/137/files, method: GET, req_id: 23b41528-ab64-4da8-a3b7-43cfbefd3844, remote_addr: 127.0.0.1:36872, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.427 INFO eid:137 Found repair files: ["089", "089.db"]
Sep 22 23:15:04.427 TRCE incoming request, uri: /newextent/137/data, method: GET, req_id: bb6bc246-6e13-4973-9bc9-5fda48a53a1c, remote_addr: 127.0.0.1:36872, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.427 INFO request completed, latency_us: 318, response_code: 200, uri: /newextent/137/data, method: GET, req_id: bb6bc246-6e13-4973-9bc9-5fda48a53a1c, remote_addr: 127.0.0.1:36872, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.428 ERRO [0] job id 1073 saw error GenericError("test error")
Sep 22 23:15:04.428 ERRO [0] job id 1073 saw error GenericError("test error")
Sep 22 23:15:04.428 ERRO [0] job id 1074 saw error GenericError("test error")
Sep 22 23:15:04.428 ERRO [0] job id 1074 saw error GenericError("test error")
Sep 22 23:15:04.429 WARN returning error on write!
Sep 22 23:15:04.429 DEBG Write     :1070 deps:[JobId(1069), JobId(1066), JobId(1063), JobId(1060), JobId(1057), JobId(1054), JobId(1051), JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:false
Sep 22 23:15:04.430 WARN 1071 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:15:04.432 TRCE incoming request, uri: /newextent/137/db, method: GET, req_id: a2684cb8-54ca-4b46-b202-2bcd71157a56, remote_addr: 127.0.0.1:36872, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.433 INFO request completed, latency_us: 291, response_code: 200, uri: /newextent/137/db, method: GET, req_id: a2684cb8-54ca-4b46-b202-2bcd71157a56, remote_addr: 127.0.0.1:36872, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.434 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/089.copy" to "/tmp/downstairs-vrx8aK6L/00/000/089.replace"
Sep 22 23:15:04.434 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:04.435 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/089.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:04.435 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/089"
Sep 22 23:15:04.435 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/089.db"
Sep 22 23:15:04.435 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:04.435 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/089.replace" to "/tmp/downstairs-vrx8aK6L/00/000/089.completed"
Sep 22 23:15:04.435 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:04.435 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:04.435 DEBG [0] It's time to notify for 38
Sep 22 23:15:04.436 INFO Completion from [0] id:38 status:true
Sep 22 23:15:04.436 INFO [39/752] Repair commands completed
Sep 22 23:15:04.436 INFO Pop front: ReconcileIO { id: ReconciliationId(39), op: ExtentReopen { repair_id: ReconciliationId(39), extent_id: 137 }, state: ClientData([New, New, New]) }
Sep 22 23:15:04.436 INFO Sent repair work, now wait for resp
Sep 22 23:15:04.436 INFO [0] received reconcile message
Sep 22 23:15:04.436 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(39), op: ExtentReopen { repair_id: ReconciliationId(39), extent_id: 137 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:04.436 INFO [0] client ExtentReopen { repair_id: ReconciliationId(39), extent_id: 137 }
Sep 22 23:15:04.436 INFO [1] received reconcile message
Sep 22 23:15:04.436 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(39), op: ExtentReopen { repair_id: ReconciliationId(39), extent_id: 137 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:04.436 INFO [1] client ExtentReopen { repair_id: ReconciliationId(39), extent_id: 137 }
Sep 22 23:15:04.436 INFO [2] received reconcile message
Sep 22 23:15:04.436 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(39), op: ExtentReopen { repair_id: ReconciliationId(39), extent_id: 137 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:04.436 INFO [2] client ExtentReopen { repair_id: ReconciliationId(39), extent_id: 137 }
Sep 22 23:15:04.436 DEBG 39 Reopen extent 137
Sep 22 23:15:04.437 DEBG 39 Reopen extent 137
Sep 22 23:15:04.437 DEBG 39 Reopen extent 137
Sep 22 23:15:04.438 DEBG [2] It's time to notify for 39
Sep 22 23:15:04.438 INFO Completion from [2] id:39 status:true
Sep 22 23:15:04.438 INFO [40/752] Repair commands completed
Sep 22 23:15:04.438 INFO Pop front: ReconcileIO { id: ReconciliationId(40), op: ExtentFlush { repair_id: ReconciliationId(40), extent_id: 146, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:04.438 INFO Sent repair work, now wait for resp
Sep 22 23:15:04.438 INFO [0] received reconcile message
Sep 22 23:15:04.438 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(40), op: ExtentFlush { repair_id: ReconciliationId(40), extent_id: 146, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:04.438 INFO [0] client ExtentFlush { repair_id: ReconciliationId(40), extent_id: 146, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:04.438 INFO [1] received reconcile message
Sep 22 23:15:04.438 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(40), op: ExtentFlush { repair_id: ReconciliationId(40), extent_id: 146, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:04.438 INFO [1] client ExtentFlush { repair_id: ReconciliationId(40), extent_id: 146, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:04.438 INFO [2] received reconcile message
Sep 22 23:15:04.438 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(40), op: ExtentFlush { repair_id: ReconciliationId(40), extent_id: 146, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:04.438 INFO [2] client ExtentFlush { repair_id: ReconciliationId(40), extent_id: 146, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:04.438 DEBG 40 Flush extent 146 with f:2 g:2
Sep 22 23:15:04.438 DEBG Flush just extent 146 with f:2 and g:2
Sep 22 23:15:04.438 DEBG [1] It's time to notify for 40
Sep 22 23:15:04.438 INFO Completion from [1] id:40 status:true
Sep 22 23:15:04.438 INFO [41/752] Repair commands completed
Sep 22 23:15:04.438 INFO Pop front: ReconcileIO { id: ReconciliationId(41), op: ExtentClose { repair_id: ReconciliationId(41), extent_id: 146 }, state: ClientData([New, New, New]) }
Sep 22 23:15:04.438 INFO Sent repair work, now wait for resp
Sep 22 23:15:04.438 INFO [0] received reconcile message
Sep 22 23:15:04.438 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(41), op: ExtentClose { repair_id: ReconciliationId(41), extent_id: 146 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:04.438 INFO [0] client ExtentClose { repair_id: ReconciliationId(41), extent_id: 146 }
Sep 22 23:15:04.438 INFO [1] received reconcile message
Sep 22 23:15:04.438 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(41), op: ExtentClose { repair_id: ReconciliationId(41), extent_id: 146 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:04.438 INFO [1] client ExtentClose { repair_id: ReconciliationId(41), extent_id: 146 }
Sep 22 23:15:04.438 INFO [2] received reconcile message
Sep 22 23:15:04.438 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(41), op: ExtentClose { repair_id: ReconciliationId(41), extent_id: 146 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:04.439 INFO [2] client ExtentClose { repair_id: ReconciliationId(41), extent_id: 146 }
Sep 22 23:15:04.439 DEBG 41 Close extent 146
Sep 22 23:15:04.439 DEBG 41 Close extent 146
Sep 22 23:15:04.439 DEBG 41 Close extent 146
Sep 22 23:15:04.440 DEBG [2] It's time to notify for 41
Sep 22 23:15:04.440 INFO Completion from [2] id:41 status:true
Sep 22 23:15:04.440 INFO [42/752] Repair commands completed
Sep 22 23:15:04.440 INFO Pop front: ReconcileIO { id: ReconciliationId(42), op: ExtentRepair { repair_id: ReconciliationId(42), extent_id: 146, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:04.440 INFO Sent repair work, now wait for resp
Sep 22 23:15:04.440 INFO [0] received reconcile message
Sep 22 23:15:04.440 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(42), op: ExtentRepair { repair_id: ReconciliationId(42), extent_id: 146, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:04.440 INFO [0] client ExtentRepair { repair_id: ReconciliationId(42), extent_id: 146, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:04.440 INFO [0] Sending repair request ReconciliationId(42)
Sep 22 23:15:04.440 INFO [1] received reconcile message
Sep 22 23:15:04.440 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(42), op: ExtentRepair { repair_id: ReconciliationId(42), extent_id: 146, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:04.440 INFO [1] client ExtentRepair { repair_id: ReconciliationId(42), extent_id: 146, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:04.440 INFO [1] No action required ReconciliationId(42)
Sep 22 23:15:04.440 INFO [2] received reconcile message
Sep 22 23:15:04.440 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(42), op: ExtentRepair { repair_id: ReconciliationId(42), extent_id: 146, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:04.440 INFO [2] client ExtentRepair { repair_id: ReconciliationId(42), extent_id: 146, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:04.440 INFO [2] No action required ReconciliationId(42)
Sep 22 23:15:04.440 DEBG 42 Repair extent 146 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:04.440 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/092.copy"
Sep 22 23:15:04.459 DEBG Write     :1070 deps:[JobId(1069), JobId(1066), JobId(1063), JobId(1060), JobId(1057), JobId(1054), JobId(1051), JobId(1048), JobId(1045), JobId(1042), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1026), JobId(1023), JobId(1020), JobId(1017), JobId(1014), JobId(1011), JobId(1008), JobId(1005), JobId(1002)] res:true
Sep 22 23:15:04.461 INFO [lossy] skipping 1071
Sep 22 23:15:04.461 INFO [lossy] skipping 1073
Sep 22 23:15:04.461 INFO [lossy] skipping 1071
Sep 22 23:15:04.461 INFO [lossy] skipping 1073
Sep 22 23:15:04.461 WARN returning error on flush!
Sep 22 23:15:04.461 DEBG Flush     :1071 extent_limit None deps:[JobId(1070), JobId(1069)] res:false f:25 g:1
Sep 22 23:15:04.461 INFO [lossy] skipping 1073
Sep 22 23:15:04.461 WARN returning error on flush!
Sep 22 23:15:04.461 DEBG Flush     :1071 extent_limit None deps:[JobId(1070), JobId(1069)] res:false f:25 g:1
Sep 22 23:15:04.461 INFO [lossy] skipping 1073
Sep 22 23:15:04.461 INFO [lossy] skipping 1071
Sep 22 23:15:04.461 INFO [lossy] skipping 1073
Sep 22 23:15:04.464 DEBG Flush     :1071 extent_limit None deps:[JobId(1070), JobId(1069)] res:true f:25 g:1
Sep 22 23:15:04.464 INFO [lossy] skipping 1073
Sep 22 23:15:04.464 WARN 1073 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:15:04.464 INFO [lossy] skipping 1072
Sep 22 23:15:04.464 INFO [lossy] skipping 1073
Sep 22 23:15:04.464 INFO [lossy] skipping 1074
Sep 22 23:15:04.470 DEBG Read      :1072 deps:[JobId(1071)] res:true
Sep 22 23:15:04.492 DEBG IO Flush 1075 has deps [JobId(1074), JobId(1073)]
Sep 22 23:15:04.492 ERRO [1] job id 1070 saw error GenericError("test error")
Sep 22 23:15:04.492 ERRO [1] job id 1071 saw error GenericError("test error")
Sep 22 23:15:04.492 ERRO [1] job id 1071 saw error GenericError("test error")
Sep 22 23:15:04.493 DEBG [rc] retire 1071 clears [JobId(1070), JobId(1071)], : downstairs
Sep 22 23:15:04.496 INFO [lossy] sleeping 1 second
Sep 22 23:15:04.502 INFO accepted connection, remote_addr: 127.0.0.1:55504, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.502 TRCE incoming request, uri: /extent/146/files, method: GET, req_id: 3b3b0721-4abd-48c3-8f81-d354f2026591, remote_addr: 127.0.0.1:55504, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.503 INFO request completed, latency_us: 217, response_code: 200, uri: /extent/146/files, method: GET, req_id: 3b3b0721-4abd-48c3-8f81-d354f2026591, remote_addr: 127.0.0.1:55504, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.503 INFO eid:146 Found repair files: ["092", "092.db"]
Sep 22 23:15:04.503 TRCE incoming request, uri: /newextent/146/data, method: GET, req_id: 8217c803-3b29-43da-ba79-94146ad04d4d, remote_addr: 127.0.0.1:55504, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.504 INFO request completed, latency_us: 338, response_code: 200, uri: /newextent/146/data, method: GET, req_id: 8217c803-3b29-43da-ba79-94146ad04d4d, remote_addr: 127.0.0.1:55504, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.509 TRCE incoming request, uri: /newextent/146/db, method: GET, req_id: 6946b944-c5c9-470c-9112-fb04347c6b00, remote_addr: 127.0.0.1:55504, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.509 INFO request completed, latency_us: 292, response_code: 200, uri: /newextent/146/db, method: GET, req_id: 6946b944-c5c9-470c-9112-fb04347c6b00, remote_addr: 127.0.0.1:55504, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.510 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/092.copy" to "/tmp/downstairs-vrx8aK6L/00/000/092.replace"
Sep 22 23:15:04.510 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:04.511 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/092.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:04.511 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/092"
Sep 22 23:15:04.511 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/092.db"
Sep 22 23:15:04.511 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:04.511 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/092.replace" to "/tmp/downstairs-vrx8aK6L/00/000/092.completed"
Sep 22 23:15:04.512 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:04.512 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:04.512 DEBG [0] It's time to notify for 42
Sep 22 23:15:04.512 INFO Completion from [0] id:42 status:true
Sep 22 23:15:04.512 INFO [43/752] Repair commands completed
Sep 22 23:15:04.512 INFO Pop front: ReconcileIO { id: ReconciliationId(43), op: ExtentReopen { repair_id: ReconciliationId(43), extent_id: 146 }, state: ClientData([New, New, New]) }
Sep 22 23:15:04.512 INFO Sent repair work, now wait for resp
Sep 22 23:15:04.512 INFO [0] received reconcile message
Sep 22 23:15:04.512 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(43), op: ExtentReopen { repair_id: ReconciliationId(43), extent_id: 146 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:04.512 INFO [0] client ExtentReopen { repair_id: ReconciliationId(43), extent_id: 146 }
Sep 22 23:15:04.512 INFO [1] received reconcile message
Sep 22 23:15:04.512 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(43), op: ExtentReopen { repair_id: ReconciliationId(43), extent_id: 146 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:04.512 INFO [1] client ExtentReopen { repair_id: ReconciliationId(43), extent_id: 146 }
Sep 22 23:15:04.512 INFO [2] received reconcile message
Sep 22 23:15:04.512 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(43), op: ExtentReopen { repair_id: ReconciliationId(43), extent_id: 146 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:04.512 INFO [2] client ExtentReopen { repair_id: ReconciliationId(43), extent_id: 146 }
Sep 22 23:15:04.512 DEBG 43 Reopen extent 146
Sep 22 23:15:04.513 DEBG 43 Reopen extent 146
Sep 22 23:15:04.514 DEBG 43 Reopen extent 146
Sep 22 23:15:04.514 DEBG [2] It's time to notify for 43
Sep 22 23:15:04.514 INFO Completion from [2] id:43 status:true
Sep 22 23:15:04.514 INFO [44/752] Repair commands completed
Sep 22 23:15:04.514 INFO Pop front: ReconcileIO { id: ReconciliationId(44), op: ExtentFlush { repair_id: ReconciliationId(44), extent_id: 105, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:04.514 INFO Sent repair work, now wait for resp
Sep 22 23:15:04.514 INFO [0] received reconcile message
Sep 22 23:15:04.514 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(44), op: ExtentFlush { repair_id: ReconciliationId(44), extent_id: 105, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:04.514 INFO [0] client ExtentFlush { repair_id: ReconciliationId(44), extent_id: 105, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:04.514 INFO [1] received reconcile message
Sep 22 23:15:04.514 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(44), op: ExtentFlush { repair_id: ReconciliationId(44), extent_id: 105, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:04.514 INFO [1] client ExtentFlush { repair_id: ReconciliationId(44), extent_id: 105, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:04.514 INFO [2] received reconcile message
Sep 22 23:15:04.514 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(44), op: ExtentFlush { repair_id: ReconciliationId(44), extent_id: 105, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:04.514 INFO [2] client ExtentFlush { repair_id: ReconciliationId(44), extent_id: 105, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:04.515 DEBG 44 Flush extent 105 with f:2 g:2
Sep 22 23:15:04.515 DEBG Flush just extent 105 with f:2 and g:2
Sep 22 23:15:04.515 DEBG [1] It's time to notify for 44
Sep 22 23:15:04.515 INFO Completion from [1] id:44 status:true
Sep 22 23:15:04.515 INFO [45/752] Repair commands completed
Sep 22 23:15:04.515 INFO Pop front: ReconcileIO { id: ReconciliationId(45), op: ExtentClose { repair_id: ReconciliationId(45), extent_id: 105 }, state: ClientData([New, New, New]) }
Sep 22 23:15:04.515 INFO Sent repair work, now wait for resp
Sep 22 23:15:04.515 INFO [0] received reconcile message
Sep 22 23:15:04.515 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(45), op: ExtentClose { repair_id: ReconciliationId(45), extent_id: 105 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:04.515 INFO [0] client ExtentClose { repair_id: ReconciliationId(45), extent_id: 105 }
Sep 22 23:15:04.515 INFO [1] received reconcile message
Sep 22 23:15:04.515 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(45), op: ExtentClose { repair_id: ReconciliationId(45), extent_id: 105 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:04.515 INFO [1] client ExtentClose { repair_id: ReconciliationId(45), extent_id: 105 }
Sep 22 23:15:04.515 INFO [2] received reconcile message
Sep 22 23:15:04.515 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(45), op: ExtentClose { repair_id: ReconciliationId(45), extent_id: 105 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:04.515 INFO [2] client ExtentClose { repair_id: ReconciliationId(45), extent_id: 105 }
Sep 22 23:15:04.515 DEBG 45 Close extent 105
Sep 22 23:15:04.515 DEBG 45 Close extent 105
Sep 22 23:15:04.516 DEBG 45 Close extent 105
Sep 22 23:15:04.516 DEBG [2] It's time to notify for 45
Sep 22 23:15:04.516 INFO Completion from [2] id:45 status:true
Sep 22 23:15:04.516 INFO [46/752] Repair commands completed
Sep 22 23:15:04.516 INFO Pop front: ReconcileIO { id: ReconciliationId(46), op: ExtentRepair { repair_id: ReconciliationId(46), extent_id: 105, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:04.516 INFO Sent repair work, now wait for resp
Sep 22 23:15:04.516 INFO [0] received reconcile message
Sep 22 23:15:04.516 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(46), op: ExtentRepair { repair_id: ReconciliationId(46), extent_id: 105, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:04.516 INFO [0] client ExtentRepair { repair_id: ReconciliationId(46), extent_id: 105, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:04.516 INFO [0] Sending repair request ReconciliationId(46)
Sep 22 23:15:04.516 INFO [1] received reconcile message
Sep 22 23:15:04.516 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(46), op: ExtentRepair { repair_id: ReconciliationId(46), extent_id: 105, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:04.516 INFO [1] client ExtentRepair { repair_id: ReconciliationId(46), extent_id: 105, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:04.516 INFO [1] No action required ReconciliationId(46)
Sep 22 23:15:04.516 INFO [2] received reconcile message
Sep 22 23:15:04.516 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(46), op: ExtentRepair { repair_id: ReconciliationId(46), extent_id: 105, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:04.516 INFO [2] client ExtentRepair { repair_id: ReconciliationId(46), extent_id: 105, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:04.516 INFO [2] No action required ReconciliationId(46)
Sep 22 23:15:04.517 DEBG 46 Repair extent 105 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:04.517 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/069.copy"
Sep 22 23:15:04.580 INFO accepted connection, remote_addr: 127.0.0.1:34460, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.580 TRCE incoming request, uri: /extent/105/files, method: GET, req_id: a51b7036-8645-4bd0-9e38-f04c561524f8, remote_addr: 127.0.0.1:34460, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.580 INFO [0] 6e5c0f70-fd56-4280-9d20-71288e488216 looper connected, looper: 0
Sep 22 23:15:04.580 INFO [0] Proc runs for 127.0.0.1:50216 in state Replaced
Sep 22 23:15:04.580 INFO accepted connection from 127.0.0.1:49037, task: main
Sep 22 23:15:04.581 INFO request completed, latency_us: 367, response_code: 200, uri: /extent/105/files, method: GET, req_id: a51b7036-8645-4bd0-9e38-f04c561524f8, remote_addr: 127.0.0.1:34460, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.581 INFO eid:105 Found repair files: ["069", "069.db"]
Sep 22 23:15:04.581 TRCE incoming request, uri: /newextent/105/data, method: GET, req_id: 63859b79-03ec-417b-a25a-1c0153228680, remote_addr: 127.0.0.1:34460, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.581 INFO request completed, latency_us: 259, response_code: 200, uri: /newextent/105/data, method: GET, req_id: 63859b79-03ec-417b-a25a-1c0153228680, remote_addr: 127.0.0.1:34460, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.587 TRCE incoming request, uri: /newextent/105/db, method: GET, req_id: 73725398-9ab7-4f67-800a-ba915c1d2207, remote_addr: 127.0.0.1:34460, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.587 INFO request completed, latency_us: 297, response_code: 200, uri: /newextent/105/db, method: GET, req_id: 73725398-9ab7-4f67-800a-ba915c1d2207, remote_addr: 127.0.0.1:34460, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.588 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/069.copy" to "/tmp/downstairs-vrx8aK6L/00/000/069.replace"
Sep 22 23:15:04.588 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:04.589 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/069.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:04.589 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/069"
Sep 22 23:15:04.589 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/069.db"
Sep 22 23:15:04.589 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:04.589 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/069.replace" to "/tmp/downstairs-vrx8aK6L/00/000/069.completed"
Sep 22 23:15:04.589 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:04.589 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:04.590 INFO Connection request from 6e5c0f70-fd56-4280-9d20-71288e488216 with version 4, task: proc
Sep 22 23:15:04.590 INFO upstairs UpstairsConnection { upstairs_id: 6e5c0f70-fd56-4280-9d20-71288e488216, session_id: f334c95e-b851-4a8a-a731-3fb69e42e934, gen: 1 } connected, version 4, task: proc
Sep 22 23:15:04.590 DEBG [0] It's time to notify for 46
Sep 22 23:15:04.590 INFO [0] 6e5c0f70-fd56-4280-9d20-71288e488216 (f334c95e-b851-4a8a-a731-3fb69e42e934) Replaced Disconnected Disconnected ds_transition to WaitActive
thread 'test::integration_test_volume_replace_downstairs_then_takeover' panicked at '[0] 6e5c0f70-fd56-4280-9d20-71288e488216 Initializing Negotiation failed, Replaced -> WaitActive', upstairs/src/lib.rs:6349:21
note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace
Sep 22 23:15:04.590 INFO Completion from [0] id:46 status:true
Sep 22 23:15:04.591 INFO [47/752] Repair commands completed
Sep 22 23:15:04.591 INFO Pop front: ReconcileIO { id: ReconciliationId(47), op: ExtentReopen { repair_id: ReconciliationId(47), extent_id: 105 }, state: ClientData([New, New, New]) }
Sep 22 23:15:04.591 INFO Sent repair work, now wait for resp
Sep 22 23:15:04.591 INFO [0] received reconcile message
Sep 22 23:15:04.591 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(47), op: ExtentReopen { repair_id: ReconciliationId(47), extent_id: 105 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:04.591 INFO [0] client ExtentReopen { repair_id: ReconciliationId(47), extent_id: 105 }
Sep 22 23:15:04.591 INFO [1] received reconcile message
Sep 22 23:15:04.591 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(47), op: ExtentReopen { repair_id: ReconciliationId(47), extent_id: 105 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:04.591 INFO [1] client ExtentReopen { repair_id: ReconciliationId(47), extent_id: 105 }
Sep 22 23:15:04.591 INFO [2] received reconcile message
Sep 22 23:15:04.591 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(47), op: ExtentReopen { repair_id: ReconciliationId(47), extent_id: 105 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:04.591 INFO [2] client ExtentReopen { repair_id: ReconciliationId(47), extent_id: 105 }
Sep 22 23:15:04.591 WARN UpstairsConnection { upstairs_id: 6e5c0f70-fd56-4280-9d20-71288e488216, session_id: f334c95e-b851-4a8a-a731-3fb69e42e934, gen: 1 } cannot grab lock, does not match UpstairsConnection { upstairs_id: 6e5c0f70-fd56-4280-9d20-71288e488216, session_id: 2cf90053-4dbd-4cc4-8468-9d35f085e47a, gen: 2 }!
Sep 22 23:15:04.591 INFO upstairs UpstairsConnection { upstairs_id: 6e5c0f70-fd56-4280-9d20-71288e488216, session_id: f334c95e-b851-4a8a-a731-3fb69e42e934, gen: 1 } disconnected, Upstairs is not active, task: proc
Sep 22 23:15:04.591 INFO connection (127.0.0.1:49037): all done
Sep 22 23:15:04.591 DEBG 47 Reopen extent 105
Sep 22 23:15:04.592 DEBG 47 Reopen extent 105
Sep 22 23:15:04.592 DEBG 47 Reopen extent 105
Sep 22 23:15:04.593 DEBG [2] It's time to notify for 47
Sep 22 23:15:04.593 INFO Completion from [2] id:47 status:true
Sep 22 23:15:04.593 INFO [48/752] Repair commands completed
Sep 22 23:15:04.593 INFO Pop front: ReconcileIO { id: ReconciliationId(48), op: ExtentFlush { repair_id: ReconciliationId(48), extent_id: 4, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:04.593 INFO Sent repair work, now wait for resp
Sep 22 23:15:04.593 INFO [0] received reconcile message
Sep 22 23:15:04.593 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(48), op: ExtentFlush { repair_id: ReconciliationId(48), extent_id: 4, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:04.593 INFO [0] client ExtentFlush { repair_id: ReconciliationId(48), extent_id: 4, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:04.593 INFO [1] received reconcile message
Sep 22 23:15:04.593 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(48), op: ExtentFlush { repair_id: ReconciliationId(48), extent_id: 4, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:04.593 INFO [1] client ExtentFlush { repair_id: ReconciliationId(48), extent_id: 4, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:04.593 INFO [2] received reconcile message
Sep 22 23:15:04.593 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(48), op: ExtentFlush { repair_id: ReconciliationId(48), extent_id: 4, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:04.593 INFO [2] client ExtentFlush { repair_id: ReconciliationId(48), extent_id: 4, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:04.593 DEBG 48 Flush extent 4 with f:2 g:2
Sep 22 23:15:04.593 DEBG Flush just extent 4 with f:2 and g:2
Sep 22 23:15:04.593 DEBG [1] It's time to notify for 48
Sep 22 23:15:04.593 INFO Completion from [1] id:48 status:true
Sep 22 23:15:04.593 INFO [49/752] Repair commands completed
Sep 22 23:15:04.593 INFO Pop front: ReconcileIO { id: ReconciliationId(49), op: ExtentClose { repair_id: ReconciliationId(49), extent_id: 4 }, state: ClientData([New, New, New]) }
Sep 22 23:15:04.593 INFO Sent repair work, now wait for resp
Sep 22 23:15:04.593 INFO [0] received reconcile message
Sep 22 23:15:04.593 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(49), op: ExtentClose { repair_id: ReconciliationId(49), extent_id: 4 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:04.593 INFO [0] client ExtentClose { repair_id: ReconciliationId(49), extent_id: 4 }
Sep 22 23:15:04.594 INFO [1] received reconcile message
Sep 22 23:15:04.594 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(49), op: ExtentClose { repair_id: ReconciliationId(49), extent_id: 4 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:04.594 INFO [1] client ExtentClose { repair_id: ReconciliationId(49), extent_id: 4 }
Sep 22 23:15:04.594 INFO [2] received reconcile message
Sep 22 23:15:04.594 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(49), op: ExtentClose { repair_id: ReconciliationId(49), extent_id: 4 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:04.594 INFO [2] client ExtentClose { repair_id: ReconciliationId(49), extent_id: 4 }
Sep 22 23:15:04.594 DEBG 49 Close extent 4
Sep 22 23:15:04.594 DEBG 49 Close extent 4
Sep 22 23:15:04.594 DEBG 49 Close extent 4
Sep 22 23:15:04.595 DEBG [2] It's time to notify for 49
Sep 22 23:15:04.595 INFO Completion from [2] id:49 status:true
Sep 22 23:15:04.595 INFO [50/752] Repair commands completed
Sep 22 23:15:04.595 INFO Pop front: ReconcileIO { id: ReconciliationId(50), op: ExtentRepair { repair_id: ReconciliationId(50), extent_id: 4, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:04.595 INFO Sent repair work, now wait for resp
Sep 22 23:15:04.595 INFO [0] received reconcile message
Sep 22 23:15:04.595 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(50), op: ExtentRepair { repair_id: ReconciliationId(50), extent_id: 4, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:04.595 INFO [0] client ExtentRepair { repair_id: ReconciliationId(50), extent_id: 4, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:04.595 INFO [0] Sending repair request ReconciliationId(50)
Sep 22 23:15:04.595 INFO [1] received reconcile message
Sep 22 23:15:04.595 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(50), op: ExtentRepair { repair_id: ReconciliationId(50), extent_id: 4, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:04.595 INFO [1] client ExtentRepair { repair_id: ReconciliationId(50), extent_id: 4, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:04.595 INFO [1] No action required ReconciliationId(50)
Sep 22 23:15:04.595 INFO [2] received reconcile message
Sep 22 23:15:04.595 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(50), op: ExtentRepair { repair_id: ReconciliationId(50), extent_id: 4, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:04.595 INFO [2] client ExtentRepair { repair_id: ReconciliationId(50), extent_id: 4, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:04.595 INFO [2] No action required ReconciliationId(50)
Sep 22 23:15:04.595 DEBG 50 Repair extent 4 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:04.595 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/004.copy"
Sep 22 23:15:04.660 INFO accepted connection, remote_addr: 127.0.0.1:36698, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.661 TRCE incoming request, uri: /extent/4/files, method: GET, req_id: ab73936a-a5ba-49a4-99d4-012d27718f57, remote_addr: 127.0.0.1:36698, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.661 INFO [1] 6e5c0f70-fd56-4280-9d20-71288e488216 looper connected, looper: 1
Sep 22 23:15:04.661 INFO [1] Proc runs for 127.0.0.1:64149 in state Disconnected
Sep 22 23:15:04.661 INFO [2] 6e5c0f70-fd56-4280-9d20-71288e488216 looper connected, looper: 2
Sep 22 23:15:04.661 INFO [2] Proc runs for 127.0.0.1:58182 in state Disconnected
Sep 22 23:15:04.661 INFO accepted connection from 127.0.0.1:34434, task: main
Sep 22 23:15:04.661 INFO accepted connection from 127.0.0.1:37115, task: main
Sep 22 23:15:04.661 INFO request completed, latency_us: 638, response_code: 200, uri: /extent/4/files, method: GET, req_id: ab73936a-a5ba-49a4-99d4-012d27718f57, remote_addr: 127.0.0.1:36698, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.662 INFO Connection request from 6e5c0f70-fd56-4280-9d20-71288e488216 with version 4, task: proc
Sep 22 23:15:04.662 INFO upstairs UpstairsConnection { upstairs_id: 6e5c0f70-fd56-4280-9d20-71288e488216, session_id: f334c95e-b851-4a8a-a731-3fb69e42e934, gen: 1 } connected, version 4, task: proc
Sep 22 23:15:04.662 INFO Connection request from 6e5c0f70-fd56-4280-9d20-71288e488216 with version 4, task: proc
Sep 22 23:15:04.662 INFO upstairs UpstairsConnection { upstairs_id: 6e5c0f70-fd56-4280-9d20-71288e488216, session_id: f334c95e-b851-4a8a-a731-3fb69e42e934, gen: 1 } connected, version 4, task: proc
Sep 22 23:15:04.662 INFO eid:4 Found repair files: ["004", "004.db"]
Sep 22 23:15:04.662 TRCE incoming request, uri: /newextent/4/data, method: GET, req_id: 6d92556d-1fa6-4d2e-aac8-16488048fcbe, remote_addr: 127.0.0.1:36698, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.662 INFO [1] 6e5c0f70-fd56-4280-9d20-71288e488216 (f334c95e-b851-4a8a-a731-3fb69e42e934) Replaced Disconnected Disconnected ds_transition to WaitActive
Sep 22 23:15:04.662 INFO [1] Transition from Disconnected to WaitActive
Sep 22 23:15:04.662 INFO [2] 6e5c0f70-fd56-4280-9d20-71288e488216 (f334c95e-b851-4a8a-a731-3fb69e42e934) Replaced WaitActive Disconnected ds_transition to WaitActive
Sep 22 23:15:04.662 INFO [2] Transition from Disconnected to WaitActive
Sep 22 23:15:04.663 INFO request completed, latency_us: 526, response_code: 200, uri: /newextent/4/data, method: GET, req_id: 6d92556d-1fa6-4d2e-aac8-16488048fcbe, remote_addr: 127.0.0.1:36698, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.668 TRCE incoming request, uri: /newextent/4/db, method: GET, req_id: 023adc09-90c4-4da1-99b2-2b793557932e, remote_addr: 127.0.0.1:36698, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.668 INFO request completed, latency_us: 291, response_code: 200, uri: /newextent/4/db, method: GET, req_id: 023adc09-90c4-4da1-99b2-2b793557932e, remote_addr: 127.0.0.1:36698, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.669 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/004.copy" to "/tmp/downstairs-vrx8aK6L/00/000/004.replace"
Sep 22 23:15:04.669 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:04.670 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/004.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:04.671 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/004"
Sep 22 23:15:04.671 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/004.db"
Sep 22 23:15:04.671 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:04.671 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/004.replace" to "/tmp/downstairs-vrx8aK6L/00/000/004.completed"
Sep 22 23:15:04.671 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:04.671 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:04.671 DEBG [0] It's time to notify for 50
Sep 22 23:15:04.671 INFO Completion from [0] id:50 status:true
Sep 22 23:15:04.671 INFO [51/752] Repair commands completed
Sep 22 23:15:04.671 INFO Pop front: ReconcileIO { id: ReconciliationId(51), op: ExtentReopen { repair_id: ReconciliationId(51), extent_id: 4 }, state: ClientData([New, New, New]) }
Sep 22 23:15:04.671 INFO Sent repair work, now wait for resp
Sep 22 23:15:04.671 INFO [0] received reconcile message
Sep 22 23:15:04.671 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(51), op: ExtentReopen { repair_id: ReconciliationId(51), extent_id: 4 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:04.671 INFO [0] client ExtentReopen { repair_id: ReconciliationId(51), extent_id: 4 }
Sep 22 23:15:04.671 INFO [1] received reconcile message
Sep 22 23:15:04.671 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(51), op: ExtentReopen { repair_id: ReconciliationId(51), extent_id: 4 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:04.671 INFO [1] client ExtentReopen { repair_id: ReconciliationId(51), extent_id: 4 }
Sep 22 23:15:04.671 INFO [2] received reconcile message
Sep 22 23:15:04.671 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(51), op: ExtentReopen { repair_id: ReconciliationId(51), extent_id: 4 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:04.671 INFO [2] client ExtentReopen { repair_id: ReconciliationId(51), extent_id: 4 }
Sep 22 23:15:04.672 DEBG 51 Reopen extent 4
Sep 22 23:15:04.672 DEBG 51 Reopen extent 4
Sep 22 23:15:04.673 DEBG 51 Reopen extent 4
Sep 22 23:15:04.674 DEBG [2] It's time to notify for 51
Sep 22 23:15:04.674 INFO Completion from [2] id:51 status:true
Sep 22 23:15:04.674 INFO [52/752] Repair commands completed
Sep 22 23:15:04.674 INFO Pop front: ReconcileIO { id: ReconciliationId(52), op: ExtentFlush { repair_id: ReconciliationId(52), extent_id: 70, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:04.674 INFO Sent repair work, now wait for resp
Sep 22 23:15:04.674 INFO [0] received reconcile message
Sep 22 23:15:04.674 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(52), op: ExtentFlush { repair_id: ReconciliationId(52), extent_id: 70, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:04.674 INFO [0] client ExtentFlush { repair_id: ReconciliationId(52), extent_id: 70, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:04.674 INFO [1] received reconcile message
Sep 22 23:15:04.674 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(52), op: ExtentFlush { repair_id: ReconciliationId(52), extent_id: 70, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:04.674 INFO [1] client ExtentFlush { repair_id: ReconciliationId(52), extent_id: 70, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:04.674 INFO [2] received reconcile message
Sep 22 23:15:04.674 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(52), op: ExtentFlush { repair_id: ReconciliationId(52), extent_id: 70, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:04.674 INFO [2] client ExtentFlush { repair_id: ReconciliationId(52), extent_id: 70, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:04.674 DEBG 52 Flush extent 70 with f:2 g:2
Sep 22 23:15:04.674 DEBG Flush just extent 70 with f:2 and g:2
Sep 22 23:15:04.674 DEBG [1] It's time to notify for 52
Sep 22 23:15:04.674 INFO Completion from [1] id:52 status:true
Sep 22 23:15:04.674 INFO [53/752] Repair commands completed
Sep 22 23:15:04.674 INFO Pop front: ReconcileIO { id: ReconciliationId(53), op: ExtentClose { repair_id: ReconciliationId(53), extent_id: 70 }, state: ClientData([New, New, New]) }
Sep 22 23:15:04.674 INFO Sent repair work, now wait for resp
Sep 22 23:15:04.674 INFO [0] received reconcile message
Sep 22 23:15:04.674 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(53), op: ExtentClose { repair_id: ReconciliationId(53), extent_id: 70 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:04.674 INFO [0] client ExtentClose { repair_id: ReconciliationId(53), extent_id: 70 }
Sep 22 23:15:04.674 INFO [1] received reconcile message
Sep 22 23:15:04.674 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(53), op: ExtentClose { repair_id: ReconciliationId(53), extent_id: 70 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:04.674 INFO [1] client ExtentClose { repair_id: ReconciliationId(53), extent_id: 70 }
Sep 22 23:15:04.674 INFO [2] received reconcile message
Sep 22 23:15:04.674 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(53), op: ExtentClose { repair_id: ReconciliationId(53), extent_id: 70 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:04.674 INFO [2] client ExtentClose { repair_id: ReconciliationId(53), extent_id: 70 }
Sep 22 23:15:04.675 DEBG 53 Close extent 70
Sep 22 23:15:04.675 DEBG 53 Close extent 70
Sep 22 23:15:04.675 DEBG 53 Close extent 70
Sep 22 23:15:04.676 DEBG [2] It's time to notify for 53
Sep 22 23:15:04.676 INFO Completion from [2] id:53 status:true
Sep 22 23:15:04.676 INFO [54/752] Repair commands completed
Sep 22 23:15:04.676 INFO Pop front: ReconcileIO { id: ReconciliationId(54), op: ExtentRepair { repair_id: ReconciliationId(54), extent_id: 70, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:04.676 INFO Sent repair work, now wait for resp
Sep 22 23:15:04.676 INFO [0] received reconcile message
Sep 22 23:15:04.676 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(54), op: ExtentRepair { repair_id: ReconciliationId(54), extent_id: 70, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:04.676 INFO [0] client ExtentRepair { repair_id: ReconciliationId(54), extent_id: 70, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:04.676 INFO [0] Sending repair request ReconciliationId(54)
Sep 22 23:15:04.676 INFO [1] received reconcile message
Sep 22 23:15:04.676 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(54), op: ExtentRepair { repair_id: ReconciliationId(54), extent_id: 70, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:04.676 INFO [1] client ExtentRepair { repair_id: ReconciliationId(54), extent_id: 70, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:04.676 INFO [1] No action required ReconciliationId(54)
Sep 22 23:15:04.676 INFO [2] received reconcile message
Sep 22 23:15:04.676 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(54), op: ExtentRepair { repair_id: ReconciliationId(54), extent_id: 70, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:04.676 INFO [2] client ExtentRepair { repair_id: ReconciliationId(54), extent_id: 70, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:04.676 INFO [2] No action required ReconciliationId(54)
Sep 22 23:15:04.676 DEBG 54 Repair extent 70 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:04.676 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/046.copy"
Sep 22 23:15:04.738 INFO accepted connection, remote_addr: 127.0.0.1:53939, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.738 TRCE incoming request, uri: /extent/70/files, method: GET, req_id: 868e61c7-efb7-4dfe-9d51-df237d187d2f, remote_addr: 127.0.0.1:53939, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.738 INFO request completed, latency_us: 236, response_code: 200, uri: /extent/70/files, method: GET, req_id: 868e61c7-efb7-4dfe-9d51-df237d187d2f, remote_addr: 127.0.0.1:53939, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.739 INFO eid:70 Found repair files: ["046", "046.db"]
Sep 22 23:15:04.739 TRCE incoming request, uri: /newextent/70/data, method: GET, req_id: 3787010d-16e3-4685-883f-ec8c8ed44c31, remote_addr: 127.0.0.1:53939, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.739 INFO request completed, latency_us: 332, response_code: 200, uri: /newextent/70/data, method: GET, req_id: 3787010d-16e3-4685-883f-ec8c8ed44c31, remote_addr: 127.0.0.1:53939, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.744 TRCE incoming request, uri: /newextent/70/db, method: GET, req_id: e5987dd1-38bc-4583-97a9-83b0d1f57100, remote_addr: 127.0.0.1:53939, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.745 INFO request completed, latency_us: 296, response_code: 200, uri: /newextent/70/db, method: GET, req_id: e5987dd1-38bc-4583-97a9-83b0d1f57100, remote_addr: 127.0.0.1:53939, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.746 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/046.copy" to "/tmp/downstairs-vrx8aK6L/00/000/046.replace"
Sep 22 23:15:04.746 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:04.747 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/046.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:04.747 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/046"
Sep 22 23:15:04.747 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/046.db"
Sep 22 23:15:04.747 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:04.747 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/046.replace" to "/tmp/downstairs-vrx8aK6L/00/000/046.completed"
Sep 22 23:15:04.747 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:04.747 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:04.747 DEBG [0] It's time to notify for 54
Sep 22 23:15:04.748 INFO Completion from [0] id:54 status:true
Sep 22 23:15:04.748 INFO [55/752] Repair commands completed
Sep 22 23:15:04.748 INFO Pop front: ReconcileIO { id: ReconciliationId(55), op: ExtentReopen { repair_id: ReconciliationId(55), extent_id: 70 }, state: ClientData([New, New, New]) }
Sep 22 23:15:04.748 INFO Sent repair work, now wait for resp
Sep 22 23:15:04.748 INFO [0] received reconcile message
Sep 22 23:15:04.748 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(55), op: ExtentReopen { repair_id: ReconciliationId(55), extent_id: 70 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:04.748 INFO [0] client ExtentReopen { repair_id: ReconciliationId(55), extent_id: 70 }
Sep 22 23:15:04.748 INFO [1] received reconcile message
Sep 22 23:15:04.748 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(55), op: ExtentReopen { repair_id: ReconciliationId(55), extent_id: 70 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:04.748 INFO [1] client ExtentReopen { repair_id: ReconciliationId(55), extent_id: 70 }
Sep 22 23:15:04.748 INFO [2] received reconcile message
Sep 22 23:15:04.748 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(55), op: ExtentReopen { repair_id: ReconciliationId(55), extent_id: 70 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:04.748 INFO [2] client ExtentReopen { repair_id: ReconciliationId(55), extent_id: 70 }
Sep 22 23:15:04.748 DEBG 55 Reopen extent 70
Sep 22 23:15:04.749 DEBG 55 Reopen extent 70
Sep 22 23:15:04.749 DEBG 55 Reopen extent 70
Sep 22 23:15:04.750 DEBG [2] It's time to notify for 55
Sep 22 23:15:04.750 INFO Completion from [2] id:55 status:true
Sep 22 23:15:04.750 INFO [56/752] Repair commands completed
Sep 22 23:15:04.750 INFO Pop front: ReconcileIO { id: ReconciliationId(56), op: ExtentFlush { repair_id: ReconciliationId(56), extent_id: 113, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:04.750 INFO Sent repair work, now wait for resp
Sep 22 23:15:04.750 INFO [0] received reconcile message
Sep 22 23:15:04.750 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(56), op: ExtentFlush { repair_id: ReconciliationId(56), extent_id: 113, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:04.750 INFO [0] client ExtentFlush { repair_id: ReconciliationId(56), extent_id: 113, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:04.750 INFO [1] received reconcile message
Sep 22 23:15:04.750 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(56), op: ExtentFlush { repair_id: ReconciliationId(56), extent_id: 113, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:04.750 INFO [1] client ExtentFlush { repair_id: ReconciliationId(56), extent_id: 113, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:04.750 INFO [2] received reconcile message
Sep 22 23:15:04.750 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(56), op: ExtentFlush { repair_id: ReconciliationId(56), extent_id: 113, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:04.750 INFO [2] client ExtentFlush { repair_id: ReconciliationId(56), extent_id: 113, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:04.750 DEBG 56 Flush extent 113 with f:2 g:2
Sep 22 23:15:04.750 DEBG Flush just extent 113 with f:2 and g:2
Sep 22 23:15:04.750 DEBG [1] It's time to notify for 56
Sep 22 23:15:04.750 INFO Completion from [1] id:56 status:true
Sep 22 23:15:04.750 INFO [57/752] Repair commands completed
Sep 22 23:15:04.750 INFO Pop front: ReconcileIO { id: ReconciliationId(57), op: ExtentClose { repair_id: ReconciliationId(57), extent_id: 113 }, state: ClientData([New, New, New]) }
Sep 22 23:15:04.750 INFO Sent repair work, now wait for resp
Sep 22 23:15:04.750 INFO [0] received reconcile message
Sep 22 23:15:04.750 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(57), op: ExtentClose { repair_id: ReconciliationId(57), extent_id: 113 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:04.751 INFO [0] client ExtentClose { repair_id: ReconciliationId(57), extent_id: 113 }
Sep 22 23:15:04.751 INFO [1] received reconcile message
Sep 22 23:15:04.751 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(57), op: ExtentClose { repair_id: ReconciliationId(57), extent_id: 113 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:04.751 INFO [1] client ExtentClose { repair_id: ReconciliationId(57), extent_id: 113 }
Sep 22 23:15:04.751 INFO [2] received reconcile message
Sep 22 23:15:04.751 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(57), op: ExtentClose { repair_id: ReconciliationId(57), extent_id: 113 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:04.751 INFO [2] client ExtentClose { repair_id: ReconciliationId(57), extent_id: 113 }
Sep 22 23:15:04.751 DEBG 57 Close extent 113
Sep 22 23:15:04.751 DEBG 57 Close extent 113
Sep 22 23:15:04.751 DEBG 57 Close extent 113
Sep 22 23:15:04.752 DEBG [2] It's time to notify for 57
Sep 22 23:15:04.752 INFO Completion from [2] id:57 status:true
Sep 22 23:15:04.752 INFO [58/752] Repair commands completed
Sep 22 23:15:04.752 INFO Pop front: ReconcileIO { id: ReconciliationId(58), op: ExtentRepair { repair_id: ReconciliationId(58), extent_id: 113, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:04.752 INFO Sent repair work, now wait for resp
Sep 22 23:15:04.752 INFO [0] received reconcile message
Sep 22 23:15:04.752 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(58), op: ExtentRepair { repair_id: ReconciliationId(58), extent_id: 113, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:04.752 INFO [0] client ExtentRepair { repair_id: ReconciliationId(58), extent_id: 113, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:04.752 INFO [0] Sending repair request ReconciliationId(58)
Sep 22 23:15:04.752 INFO [1] received reconcile message
Sep 22 23:15:04.752 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(58), op: ExtentRepair { repair_id: ReconciliationId(58), extent_id: 113, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:04.752 INFO [1] client ExtentRepair { repair_id: ReconciliationId(58), extent_id: 113, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:04.752 INFO [1] No action required ReconciliationId(58)
Sep 22 23:15:04.752 INFO [2] received reconcile message
Sep 22 23:15:04.752 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(58), op: ExtentRepair { repair_id: ReconciliationId(58), extent_id: 113, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:04.752 INFO [2] client ExtentRepair { repair_id: ReconciliationId(58), extent_id: 113, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:04.752 INFO [2] No action required ReconciliationId(58)
Sep 22 23:15:04.752 DEBG 58 Repair extent 113 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:04.752 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/071.copy"
Sep 22 23:15:04.814 INFO accepted connection, remote_addr: 127.0.0.1:38597, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.814 TRCE incoming request, uri: /extent/113/files, method: GET, req_id: ad804ade-f04e-4464-a5c3-c9ec73ce248e, remote_addr: 127.0.0.1:38597, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.815 INFO request completed, latency_us: 242, response_code: 200, uri: /extent/113/files, method: GET, req_id: ad804ade-f04e-4464-a5c3-c9ec73ce248e, remote_addr: 127.0.0.1:38597, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.815 INFO eid:113 Found repair files: ["071", "071.db"]
Sep 22 23:15:04.815 TRCE incoming request, uri: /newextent/113/data, method: GET, req_id: 96e86de9-c029-4871-bffb-506effcb2b00, remote_addr: 127.0.0.1:38597, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.816 INFO request completed, latency_us: 372, response_code: 200, uri: /newextent/113/data, method: GET, req_id: 96e86de9-c029-4871-bffb-506effcb2b00, remote_addr: 127.0.0.1:38597, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.821 TRCE incoming request, uri: /newextent/113/db, method: GET, req_id: 1614d7e7-96eb-4743-816c-71c4aa2f6045, remote_addr: 127.0.0.1:38597, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.821 INFO request completed, latency_us: 302, response_code: 200, uri: /newextent/113/db, method: GET, req_id: 1614d7e7-96eb-4743-816c-71c4aa2f6045, remote_addr: 127.0.0.1:38597, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.823 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/071.copy" to "/tmp/downstairs-vrx8aK6L/00/000/071.replace"
Sep 22 23:15:04.823 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:04.824 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/071.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:04.824 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/071"
Sep 22 23:15:04.824 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/071.db"
Sep 22 23:15:04.824 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:04.824 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/071.replace" to "/tmp/downstairs-vrx8aK6L/00/000/071.completed"
Sep 22 23:15:04.824 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:04.824 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:04.824 DEBG [0] It's time to notify for 58
Sep 22 23:15:04.825 INFO Completion from [0] id:58 status:true
Sep 22 23:15:04.825 INFO [59/752] Repair commands completed
Sep 22 23:15:04.825 INFO Pop front: ReconcileIO { id: ReconciliationId(59), op: ExtentReopen { repair_id: ReconciliationId(59), extent_id: 113 }, state: ClientData([New, New, New]) }
Sep 22 23:15:04.825 INFO Sent repair work, now wait for resp
Sep 22 23:15:04.825 INFO [0] received reconcile message
Sep 22 23:15:04.825 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(59), op: ExtentReopen { repair_id: ReconciliationId(59), extent_id: 113 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:04.825 INFO [0] client ExtentReopen { repair_id: ReconciliationId(59), extent_id: 113 }
Sep 22 23:15:04.825 INFO [1] received reconcile message
Sep 22 23:15:04.825 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(59), op: ExtentReopen { repair_id: ReconciliationId(59), extent_id: 113 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:04.825 INFO [1] client ExtentReopen { repair_id: ReconciliationId(59), extent_id: 113 }
Sep 22 23:15:04.825 INFO [2] received reconcile message
Sep 22 23:15:04.825 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(59), op: ExtentReopen { repair_id: ReconciliationId(59), extent_id: 113 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:04.825 INFO [2] client ExtentReopen { repair_id: ReconciliationId(59), extent_id: 113 }
Sep 22 23:15:04.825 DEBG 59 Reopen extent 113
Sep 22 23:15:04.826 DEBG 59 Reopen extent 113
Sep 22 23:15:04.826 DEBG 59 Reopen extent 113
Sep 22 23:15:04.827 DEBG [2] It's time to notify for 59
Sep 22 23:15:04.827 INFO Completion from [2] id:59 status:true
Sep 22 23:15:04.827 INFO [60/752] Repair commands completed
Sep 22 23:15:04.827 INFO Pop front: ReconcileIO { id: ReconciliationId(60), op: ExtentFlush { repair_id: ReconciliationId(60), extent_id: 85, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:04.827 INFO Sent repair work, now wait for resp
Sep 22 23:15:04.827 INFO [0] received reconcile message
Sep 22 23:15:04.827 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(60), op: ExtentFlush { repair_id: ReconciliationId(60), extent_id: 85, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:04.827 INFO [0] client ExtentFlush { repair_id: ReconciliationId(60), extent_id: 85, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:04.827 INFO [1] received reconcile message
Sep 22 23:15:04.827 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(60), op: ExtentFlush { repair_id: ReconciliationId(60), extent_id: 85, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:04.827 INFO [1] client ExtentFlush { repair_id: ReconciliationId(60), extent_id: 85, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:04.827 INFO [2] received reconcile message
Sep 22 23:15:04.827 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(60), op: ExtentFlush { repair_id: ReconciliationId(60), extent_id: 85, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:04.827 INFO [2] client ExtentFlush { repair_id: ReconciliationId(60), extent_id: 85, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:04.827 DEBG 60 Flush extent 85 with f:2 g:2
Sep 22 23:15:04.827 DEBG Flush just extent 85 with f:2 and g:2
Sep 22 23:15:04.827 DEBG [1] It's time to notify for 60
Sep 22 23:15:04.828 INFO Completion from [1] id:60 status:true
Sep 22 23:15:04.828 INFO [61/752] Repair commands completed
Sep 22 23:15:04.828 INFO Pop front: ReconcileIO { id: ReconciliationId(61), op: ExtentClose { repair_id: ReconciliationId(61), extent_id: 85 }, state: ClientData([New, New, New]) }
Sep 22 23:15:04.828 INFO Sent repair work, now wait for resp
Sep 22 23:15:04.828 INFO [0] received reconcile message
Sep 22 23:15:04.828 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(61), op: ExtentClose { repair_id: ReconciliationId(61), extent_id: 85 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:04.828 INFO [0] client ExtentClose { repair_id: ReconciliationId(61), extent_id: 85 }
Sep 22 23:15:04.828 INFO [1] received reconcile message
Sep 22 23:15:04.828 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(61), op: ExtentClose { repair_id: ReconciliationId(61), extent_id: 85 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:04.828 INFO [1] client ExtentClose { repair_id: ReconciliationId(61), extent_id: 85 }
Sep 22 23:15:04.828 INFO [2] received reconcile message
Sep 22 23:15:04.828 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(61), op: ExtentClose { repair_id: ReconciliationId(61), extent_id: 85 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:04.828 INFO [2] client ExtentClose { repair_id: ReconciliationId(61), extent_id: 85 }
Sep 22 23:15:04.828 DEBG 61 Close extent 85
Sep 22 23:15:04.828 DEBG 61 Close extent 85
Sep 22 23:15:04.829 DEBG 61 Close extent 85
Sep 22 23:15:04.829 DEBG [2] It's time to notify for 61
Sep 22 23:15:04.829 INFO Completion from [2] id:61 status:true
Sep 22 23:15:04.829 INFO [62/752] Repair commands completed
Sep 22 23:15:04.829 INFO Pop front: ReconcileIO { id: ReconciliationId(62), op: ExtentRepair { repair_id: ReconciliationId(62), extent_id: 85, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:04.829 INFO Sent repair work, now wait for resp
Sep 22 23:15:04.829 INFO [0] received reconcile message
Sep 22 23:15:04.829 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(62), op: ExtentRepair { repair_id: ReconciliationId(62), extent_id: 85, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:04.829 INFO [0] client ExtentRepair { repair_id: ReconciliationId(62), extent_id: 85, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:04.829 INFO [0] Sending repair request ReconciliationId(62)
Sep 22 23:15:04.829 INFO [1] received reconcile message
Sep 22 23:15:04.829 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(62), op: ExtentRepair { repair_id: ReconciliationId(62), extent_id: 85, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:04.829 INFO [1] client ExtentRepair { repair_id: ReconciliationId(62), extent_id: 85, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:04.829 INFO [1] No action required ReconciliationId(62)
Sep 22 23:15:04.829 INFO [2] received reconcile message
Sep 22 23:15:04.829 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(62), op: ExtentRepair { repair_id: ReconciliationId(62), extent_id: 85, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:04.829 INFO [2] client ExtentRepair { repair_id: ReconciliationId(62), extent_id: 85, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:04.829 INFO [2] No action required ReconciliationId(62)
Sep 22 23:15:04.829 DEBG 62 Repair extent 85 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:04.829 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/055.copy"
Sep 22 23:15:04.880 DEBG [0] Read AckReady 1074, : downstairs
Sep 22 23:15:04.881 DEBG up_ds_listen was notified
Sep 22 23:15:04.881 DEBG up_ds_listen process 1074
Sep 22 23:15:04.881 DEBG [A] ack job 1074:75, : downstairs
Sep 22 23:15:04.892 INFO accepted connection, remote_addr: 127.0.0.1:50509, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.892 TRCE incoming request, uri: /extent/85/files, method: GET, req_id: 6dbe19d5-6f02-4415-a9b8-7cf4e1d2d04e, remote_addr: 127.0.0.1:50509, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.893 INFO request completed, latency_us: 231, response_code: 200, uri: /extent/85/files, method: GET, req_id: 6dbe19d5-6f02-4415-a9b8-7cf4e1d2d04e, remote_addr: 127.0.0.1:50509, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.893 INFO eid:85 Found repair files: ["055", "055.db"]
Sep 22 23:15:04.893 TRCE incoming request, uri: /newextent/85/data, method: GET, req_id: e77f3179-2814-4f74-9c6d-500ad3eec275, remote_addr: 127.0.0.1:50509, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.894 INFO request completed, latency_us: 349, response_code: 200, uri: /newextent/85/data, method: GET, req_id: e77f3179-2814-4f74-9c6d-500ad3eec275, remote_addr: 127.0.0.1:50509, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.899 TRCE incoming request, uri: /newextent/85/db, method: GET, req_id: 0b0333f2-4c84-4343-9b24-7b581563cf33, remote_addr: 127.0.0.1:50509, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.899 INFO request completed, latency_us: 294, response_code: 200, uri: /newextent/85/db, method: GET, req_id: 0b0333f2-4c84-4343-9b24-7b581563cf33, remote_addr: 127.0.0.1:50509, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.900 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/055.copy" to "/tmp/downstairs-vrx8aK6L/00/000/055.replace"
Sep 22 23:15:04.900 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:04.901 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/055.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:04.901 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/055"
Sep 22 23:15:04.901 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/055.db"
Sep 22 23:15:04.901 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:04.902 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/055.replace" to "/tmp/downstairs-vrx8aK6L/00/000/055.completed"
Sep 22 23:15:04.902 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:04.902 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:04.902 DEBG [0] It's time to notify for 62
Sep 22 23:15:04.902 INFO Completion from [0] id:62 status:true
Sep 22 23:15:04.902 INFO [63/752] Repair commands completed
Sep 22 23:15:04.902 INFO Pop front: ReconcileIO { id: ReconciliationId(63), op: ExtentReopen { repair_id: ReconciliationId(63), extent_id: 85 }, state: ClientData([New, New, New]) }
Sep 22 23:15:04.902 INFO Sent repair work, now wait for resp
Sep 22 23:15:04.902 INFO [0] received reconcile message
Sep 22 23:15:04.902 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(63), op: ExtentReopen { repair_id: ReconciliationId(63), extent_id: 85 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:04.902 INFO [0] client ExtentReopen { repair_id: ReconciliationId(63), extent_id: 85 }
Sep 22 23:15:04.902 INFO [1] received reconcile message
Sep 22 23:15:04.902 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(63), op: ExtentReopen { repair_id: ReconciliationId(63), extent_id: 85 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:04.902 INFO [1] client ExtentReopen { repair_id: ReconciliationId(63), extent_id: 85 }
Sep 22 23:15:04.902 INFO [2] received reconcile message
Sep 22 23:15:04.902 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(63), op: ExtentReopen { repair_id: ReconciliationId(63), extent_id: 85 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:04.902 INFO [2] client ExtentReopen { repair_id: ReconciliationId(63), extent_id: 85 }
Sep 22 23:15:04.902 DEBG 63 Reopen extent 85
Sep 22 23:15:04.903 DEBG 63 Reopen extent 85
Sep 22 23:15:04.904 DEBG 63 Reopen extent 85
Sep 22 23:15:04.904 DEBG [2] It's time to notify for 63
Sep 22 23:15:04.904 INFO Completion from [2] id:63 status:true
Sep 22 23:15:04.904 INFO [64/752] Repair commands completed
Sep 22 23:15:04.904 INFO Pop front: ReconcileIO { id: ReconciliationId(64), op: ExtentFlush { repair_id: ReconciliationId(64), extent_id: 69, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:04.904 INFO Sent repair work, now wait for resp
Sep 22 23:15:04.904 INFO [0] received reconcile message
Sep 22 23:15:04.904 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(64), op: ExtentFlush { repair_id: ReconciliationId(64), extent_id: 69, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:04.904 INFO [0] client ExtentFlush { repair_id: ReconciliationId(64), extent_id: 69, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:04.904 INFO [1] received reconcile message
Sep 22 23:15:04.904 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(64), op: ExtentFlush { repair_id: ReconciliationId(64), extent_id: 69, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:04.905 INFO [1] client ExtentFlush { repair_id: ReconciliationId(64), extent_id: 69, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:04.905 INFO [2] received reconcile message
Sep 22 23:15:04.905 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(64), op: ExtentFlush { repair_id: ReconciliationId(64), extent_id: 69, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:04.905 INFO [2] client ExtentFlush { repair_id: ReconciliationId(64), extent_id: 69, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:04.905 DEBG 64 Flush extent 69 with f:2 g:2
Sep 22 23:15:04.905 DEBG Flush just extent 69 with f:2 and g:2
Sep 22 23:15:04.905 DEBG [1] It's time to notify for 64
Sep 22 23:15:04.905 INFO Completion from [1] id:64 status:true
Sep 22 23:15:04.905 INFO [65/752] Repair commands completed
Sep 22 23:15:04.905 INFO Pop front: ReconcileIO { id: ReconciliationId(65), op: ExtentClose { repair_id: ReconciliationId(65), extent_id: 69 }, state: ClientData([New, New, New]) }
Sep 22 23:15:04.905 INFO Sent repair work, now wait for resp
Sep 22 23:15:04.905 INFO [0] received reconcile message
Sep 22 23:15:04.905 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(65), op: ExtentClose { repair_id: ReconciliationId(65), extent_id: 69 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:04.905 INFO [0] client ExtentClose { repair_id: ReconciliationId(65), extent_id: 69 }
Sep 22 23:15:04.905 INFO [1] received reconcile message
Sep 22 23:15:04.905 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(65), op: ExtentClose { repair_id: ReconciliationId(65), extent_id: 69 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:04.905 INFO [1] client ExtentClose { repair_id: ReconciliationId(65), extent_id: 69 }
Sep 22 23:15:04.905 INFO [2] received reconcile message
Sep 22 23:15:04.905 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(65), op: ExtentClose { repair_id: ReconciliationId(65), extent_id: 69 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:04.905 INFO [2] client ExtentClose { repair_id: ReconciliationId(65), extent_id: 69 }
Sep 22 23:15:04.905 DEBG 65 Close extent 69
Sep 22 23:15:04.906 DEBG 65 Close extent 69
Sep 22 23:15:04.906 DEBG 65 Close extent 69
Sep 22 23:15:04.906 DEBG [2] It's time to notify for 65
Sep 22 23:15:04.906 INFO Completion from [2] id:65 status:true
Sep 22 23:15:04.906 INFO [66/752] Repair commands completed
Sep 22 23:15:04.906 INFO Pop front: ReconcileIO { id: ReconciliationId(66), op: ExtentRepair { repair_id: ReconciliationId(66), extent_id: 69, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:04.906 INFO Sent repair work, now wait for resp
Sep 22 23:15:04.906 INFO [0] received reconcile message
Sep 22 23:15:04.906 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(66), op: ExtentRepair { repair_id: ReconciliationId(66), extent_id: 69, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:04.906 INFO [0] client ExtentRepair { repair_id: ReconciliationId(66), extent_id: 69, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:04.906 INFO [0] Sending repair request ReconciliationId(66)
Sep 22 23:15:04.907 INFO [1] received reconcile message
Sep 22 23:15:04.907 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(66), op: ExtentRepair { repair_id: ReconciliationId(66), extent_id: 69, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:04.907 INFO [1] client ExtentRepair { repair_id: ReconciliationId(66), extent_id: 69, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:04.907 INFO [1] No action required ReconciliationId(66)
Sep 22 23:15:04.907 INFO [2] received reconcile message
Sep 22 23:15:04.907 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(66), op: ExtentRepair { repair_id: ReconciliationId(66), extent_id: 69, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:04.907 INFO [2] client ExtentRepair { repair_id: ReconciliationId(66), extent_id: 69, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:04.907 INFO [2] No action required ReconciliationId(66)
Sep 22 23:15:04.907 DEBG 66 Repair extent 69 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:04.907 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/045.copy"
Sep 22 23:15:04.935 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:04.937 DEBG Flush     :1073 extent_limit None deps:[JobId(1072), JobId(1071)] res:true f:26 g:1
Sep 22 23:15:04.944 DEBG Read      :1074 deps:[JobId(1073)] res:true
Sep 22 23:15:04.965 DEBG Flush     :1073 extent_limit None deps:[JobId(1072), JobId(1071)] res:true f:26 g:1
Sep 22 23:15:04.968 INFO accepted connection, remote_addr: 127.0.0.1:41468, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.969 TRCE incoming request, uri: /extent/69/files, method: GET, req_id: 0c5a8386-d5ec-43a0-b378-2f0e2615f9ed, remote_addr: 127.0.0.1:41468, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.969 INFO request completed, latency_us: 252, response_code: 200, uri: /extent/69/files, method: GET, req_id: 0c5a8386-d5ec-43a0-b378-2f0e2615f9ed, remote_addr: 127.0.0.1:41468, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.969 INFO eid:69 Found repair files: ["045", "045.db"]
Sep 22 23:15:04.970 TRCE incoming request, uri: /newextent/69/data, method: GET, req_id: 04162555-5f22-4431-929c-5701fd5f2b0e, remote_addr: 127.0.0.1:41468, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.970 INFO request completed, latency_us: 338, response_code: 200, uri: /newextent/69/data, method: GET, req_id: 04162555-5f22-4431-929c-5701fd5f2b0e, remote_addr: 127.0.0.1:41468, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.972 DEBG Read      :1074 deps:[JobId(1073)] res:true
Sep 22 23:15:04.975 TRCE incoming request, uri: /newextent/69/db, method: GET, req_id: 1b0f4554-077f-45b7-880b-d7f149841b9e, remote_addr: 127.0.0.1:41468, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.975 INFO request completed, latency_us: 294, response_code: 200, uri: /newextent/69/db, method: GET, req_id: 1b0f4554-077f-45b7-880b-d7f149841b9e, remote_addr: 127.0.0.1:41468, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:04.977 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/045.copy" to "/tmp/downstairs-vrx8aK6L/00/000/045.replace"
Sep 22 23:15:04.977 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:04.978 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/045.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:04.978 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/045"
Sep 22 23:15:04.978 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/045.db"
Sep 22 23:15:04.978 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:04.978 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/045.replace" to "/tmp/downstairs-vrx8aK6L/00/000/045.completed"
Sep 22 23:15:04.978 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:04.978 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:04.978 DEBG [0] It's time to notify for 66
Sep 22 23:15:04.978 INFO Completion from [0] id:66 status:true
Sep 22 23:15:04.979 INFO [67/752] Repair commands completed
Sep 22 23:15:04.979 INFO Pop front: ReconcileIO { id: ReconciliationId(67), op: ExtentReopen { repair_id: ReconciliationId(67), extent_id: 69 }, state: ClientData([New, New, New]) }
Sep 22 23:15:04.979 INFO Sent repair work, now wait for resp
Sep 22 23:15:04.979 INFO [0] received reconcile message
Sep 22 23:15:04.979 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(67), op: ExtentReopen { repair_id: ReconciliationId(67), extent_id: 69 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:04.979 INFO [0] client ExtentReopen { repair_id: ReconciliationId(67), extent_id: 69 }
Sep 22 23:15:04.979 INFO [1] received reconcile message
Sep 22 23:15:04.979 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(67), op: ExtentReopen { repair_id: ReconciliationId(67), extent_id: 69 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:04.979 INFO [1] client ExtentReopen { repair_id: ReconciliationId(67), extent_id: 69 }
Sep 22 23:15:04.979 INFO [2] received reconcile message
Sep 22 23:15:04.979 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(67), op: ExtentReopen { repair_id: ReconciliationId(67), extent_id: 69 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:04.979 INFO [2] client ExtentReopen { repair_id: ReconciliationId(67), extent_id: 69 }
Sep 22 23:15:04.979 DEBG 67 Reopen extent 69
Sep 22 23:15:04.980 DEBG 67 Reopen extent 69
Sep 22 23:15:04.980 DEBG 67 Reopen extent 69
Sep 22 23:15:04.981 DEBG [2] It's time to notify for 67
Sep 22 23:15:04.981 INFO Completion from [2] id:67 status:true
Sep 22 23:15:04.981 INFO [68/752] Repair commands completed
Sep 22 23:15:04.981 INFO Pop front: ReconcileIO { id: ReconciliationId(68), op: ExtentFlush { repair_id: ReconciliationId(68), extent_id: 102, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:04.981 INFO Sent repair work, now wait for resp
Sep 22 23:15:04.981 INFO [0] received reconcile message
Sep 22 23:15:04.981 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(68), op: ExtentFlush { repair_id: ReconciliationId(68), extent_id: 102, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:04.981 INFO [0] client ExtentFlush { repair_id: ReconciliationId(68), extent_id: 102, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:04.981 INFO [1] received reconcile message
Sep 22 23:15:04.981 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(68), op: ExtentFlush { repair_id: ReconciliationId(68), extent_id: 102, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:04.981 INFO [1] client ExtentFlush { repair_id: ReconciliationId(68), extent_id: 102, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:04.981 INFO [2] received reconcile message
Sep 22 23:15:04.981 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(68), op: ExtentFlush { repair_id: ReconciliationId(68), extent_id: 102, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:04.981 INFO [2] client ExtentFlush { repair_id: ReconciliationId(68), extent_id: 102, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:04.981 DEBG 68 Flush extent 102 with f:2 g:2
Sep 22 23:15:04.981 DEBG Flush just extent 102 with f:2 and g:2
Sep 22 23:15:04.981 DEBG [1] It's time to notify for 68
Sep 22 23:15:04.981 INFO Completion from [1] id:68 status:true
Sep 22 23:15:04.981 INFO [69/752] Repair commands completed
Sep 22 23:15:04.981 INFO Pop front: ReconcileIO { id: ReconciliationId(69), op: ExtentClose { repair_id: ReconciliationId(69), extent_id: 102 }, state: ClientData([New, New, New]) }
Sep 22 23:15:04.982 INFO Sent repair work, now wait for resp
Sep 22 23:15:04.982 INFO [0] received reconcile message
Sep 22 23:15:04.982 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(69), op: ExtentClose { repair_id: ReconciliationId(69), extent_id: 102 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:04.982 INFO [0] client ExtentClose { repair_id: ReconciliationId(69), extent_id: 102 }
Sep 22 23:15:04.982 INFO [1] received reconcile message
Sep 22 23:15:04.982 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(69), op: ExtentClose { repair_id: ReconciliationId(69), extent_id: 102 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:04.982 INFO [1] client ExtentClose { repair_id: ReconciliationId(69), extent_id: 102 }
Sep 22 23:15:04.982 INFO [2] received reconcile message
Sep 22 23:15:04.982 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(69), op: ExtentClose { repair_id: ReconciliationId(69), extent_id: 102 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:04.982 INFO [2] client ExtentClose { repair_id: ReconciliationId(69), extent_id: 102 }
Sep 22 23:15:04.982 DEBG 69 Close extent 102
Sep 22 23:15:04.982 DEBG 69 Close extent 102
Sep 22 23:15:04.982 DEBG 69 Close extent 102
Sep 22 23:15:04.983 DEBG [2] It's time to notify for 69
Sep 22 23:15:04.983 INFO Completion from [2] id:69 status:true
Sep 22 23:15:04.983 INFO [70/752] Repair commands completed
Sep 22 23:15:04.983 INFO Pop front: ReconcileIO { id: ReconciliationId(70), op: ExtentRepair { repair_id: ReconciliationId(70), extent_id: 102, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:04.983 INFO Sent repair work, now wait for resp
Sep 22 23:15:04.983 INFO [0] received reconcile message
Sep 22 23:15:04.983 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(70), op: ExtentRepair { repair_id: ReconciliationId(70), extent_id: 102, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:04.983 INFO [0] client ExtentRepair { repair_id: ReconciliationId(70), extent_id: 102, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:04.983 INFO [0] Sending repair request ReconciliationId(70)
Sep 22 23:15:04.983 INFO [1] received reconcile message
Sep 22 23:15:04.983 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(70), op: ExtentRepair { repair_id: ReconciliationId(70), extent_id: 102, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:04.983 INFO [1] client ExtentRepair { repair_id: ReconciliationId(70), extent_id: 102, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:04.983 INFO [1] No action required ReconciliationId(70)
Sep 22 23:15:04.983 INFO [2] received reconcile message
Sep 22 23:15:04.983 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(70), op: ExtentRepair { repair_id: ReconciliationId(70), extent_id: 102, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:04.983 INFO [2] client ExtentRepair { repair_id: ReconciliationId(70), extent_id: 102, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:04.983 INFO [2] No action required ReconciliationId(70)
Sep 22 23:15:04.983 DEBG 70 Repair extent 102 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:04.983 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/066.copy"
Sep 22 23:15:05.043 DEBG IO Read  1076 has deps [JobId(1075)]
Sep 22 23:15:05.047 INFO accepted connection, remote_addr: 127.0.0.1:33826, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.047 TRCE incoming request, uri: /extent/102/files, method: GET, req_id: 9e739f80-0aa4-4c8e-a439-bd1f1f37211f, remote_addr: 127.0.0.1:33826, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.047 INFO request completed, latency_us: 264, response_code: 200, uri: /extent/102/files, method: GET, req_id: 9e739f80-0aa4-4c8e-a439-bd1f1f37211f, remote_addr: 127.0.0.1:33826, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.048 INFO eid:102 Found repair files: ["066", "066.db"]
Sep 22 23:15:05.048 TRCE incoming request, uri: /newextent/102/data, method: GET, req_id: 0350ac0c-f7e0-4c41-ac6e-3793c41bcbd6, remote_addr: 127.0.0.1:33826, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.048 INFO request completed, latency_us: 362, response_code: 200, uri: /newextent/102/data, method: GET, req_id: 0350ac0c-f7e0-4c41-ac6e-3793c41bcbd6, remote_addr: 127.0.0.1:33826, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.053 TRCE incoming request, uri: /newextent/102/db, method: GET, req_id: b149eef7-eab5-4476-8d1d-4f4682848eaf, remote_addr: 127.0.0.1:33826, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.054 INFO request completed, latency_us: 307, response_code: 200, uri: /newextent/102/db, method: GET, req_id: b149eef7-eab5-4476-8d1d-4f4682848eaf, remote_addr: 127.0.0.1:33826, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.055 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/066.copy" to "/tmp/downstairs-vrx8aK6L/00/000/066.replace"
Sep 22 23:15:05.055 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:05.056 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/066.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:05.056 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/066"
Sep 22 23:15:05.056 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/066.db"
Sep 22 23:15:05.056 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:05.056 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/066.replace" to "/tmp/downstairs-vrx8aK6L/00/000/066.completed"
Sep 22 23:15:05.056 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:05.057 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:05.057 DEBG [0] It's time to notify for 70
Sep 22 23:15:05.057 INFO Completion from [0] id:70 status:true
Sep 22 23:15:05.057 INFO [71/752] Repair commands completed
Sep 22 23:15:05.057 INFO Pop front: ReconcileIO { id: ReconciliationId(71), op: ExtentReopen { repair_id: ReconciliationId(71), extent_id: 102 }, state: ClientData([New, New, New]) }
Sep 22 23:15:05.057 INFO Sent repair work, now wait for resp
Sep 22 23:15:05.057 INFO [0] received reconcile message
Sep 22 23:15:05.057 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(71), op: ExtentReopen { repair_id: ReconciliationId(71), extent_id: 102 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:05.057 INFO [0] client ExtentReopen { repair_id: ReconciliationId(71), extent_id: 102 }
Sep 22 23:15:05.057 INFO [1] received reconcile message
Sep 22 23:15:05.057 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(71), op: ExtentReopen { repair_id: ReconciliationId(71), extent_id: 102 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:05.057 INFO [1] client ExtentReopen { repair_id: ReconciliationId(71), extent_id: 102 }
Sep 22 23:15:05.057 INFO [2] received reconcile message
Sep 22 23:15:05.057 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(71), op: ExtentReopen { repair_id: ReconciliationId(71), extent_id: 102 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:05.057 INFO [2] client ExtentReopen { repair_id: ReconciliationId(71), extent_id: 102 }
Sep 22 23:15:05.058 DEBG 71 Reopen extent 102
Sep 22 23:15:05.059 DEBG 71 Reopen extent 102
Sep 22 23:15:05.059 DEBG 71 Reopen extent 102
Sep 22 23:15:05.060 DEBG [2] It's time to notify for 71
Sep 22 23:15:05.060 INFO Completion from [2] id:71 status:true
Sep 22 23:15:05.060 INFO [72/752] Repair commands completed
Sep 22 23:15:05.060 INFO Pop front: ReconcileIO { id: ReconciliationId(72), op: ExtentFlush { repair_id: ReconciliationId(72), extent_id: 73, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:05.060 INFO Sent repair work, now wait for resp
Sep 22 23:15:05.060 INFO [0] received reconcile message
Sep 22 23:15:05.060 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(72), op: ExtentFlush { repair_id: ReconciliationId(72), extent_id: 73, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:05.060 INFO [0] client ExtentFlush { repair_id: ReconciliationId(72), extent_id: 73, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:05.060 INFO [1] received reconcile message
Sep 22 23:15:05.060 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(72), op: ExtentFlush { repair_id: ReconciliationId(72), extent_id: 73, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:05.060 INFO [1] client ExtentFlush { repair_id: ReconciliationId(72), extent_id: 73, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:05.061 INFO [2] received reconcile message
Sep 22 23:15:05.061 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(72), op: ExtentFlush { repair_id: ReconciliationId(72), extent_id: 73, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:05.061 INFO [2] client ExtentFlush { repair_id: ReconciliationId(72), extent_id: 73, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:05.061 DEBG 72 Flush extent 73 with f:2 g:2
Sep 22 23:15:05.061 DEBG Flush just extent 73 with f:2 and g:2
Sep 22 23:15:05.061 DEBG [1] It's time to notify for 72
Sep 22 23:15:05.061 INFO Completion from [1] id:72 status:true
Sep 22 23:15:05.061 INFO [73/752] Repair commands completed
Sep 22 23:15:05.061 INFO Pop front: ReconcileIO { id: ReconciliationId(73), op: ExtentClose { repair_id: ReconciliationId(73), extent_id: 73 }, state: ClientData([New, New, New]) }
Sep 22 23:15:05.061 INFO Sent repair work, now wait for resp
Sep 22 23:15:05.061 INFO [0] received reconcile message
Sep 22 23:15:05.061 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(73), op: ExtentClose { repair_id: ReconciliationId(73), extent_id: 73 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:05.061 INFO [0] client ExtentClose { repair_id: ReconciliationId(73), extent_id: 73 }
Sep 22 23:15:05.061 INFO [1] received reconcile message
Sep 22 23:15:05.061 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(73), op: ExtentClose { repair_id: ReconciliationId(73), extent_id: 73 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:05.061 INFO [1] client ExtentClose { repair_id: ReconciliationId(73), extent_id: 73 }
Sep 22 23:15:05.061 INFO [2] received reconcile message
Sep 22 23:15:05.061 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(73), op: ExtentClose { repair_id: ReconciliationId(73), extent_id: 73 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:05.061 INFO [2] client ExtentClose { repair_id: ReconciliationId(73), extent_id: 73 }
Sep 22 23:15:05.062 DEBG 73 Close extent 73
Sep 22 23:15:05.062 DEBG 73 Close extent 73
Sep 22 23:15:05.062 DEBG 73 Close extent 73
Sep 22 23:15:05.063 DEBG [2] It's time to notify for 73
Sep 22 23:15:05.063 INFO Completion from [2] id:73 status:true
Sep 22 23:15:05.063 INFO [74/752] Repair commands completed
Sep 22 23:15:05.063 INFO Pop front: ReconcileIO { id: ReconciliationId(74), op: ExtentRepair { repair_id: ReconciliationId(74), extent_id: 73, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:05.063 INFO Sent repair work, now wait for resp
Sep 22 23:15:05.063 INFO [0] received reconcile message
Sep 22 23:15:05.063 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(74), op: ExtentRepair { repair_id: ReconciliationId(74), extent_id: 73, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:05.063 INFO [0] client ExtentRepair { repair_id: ReconciliationId(74), extent_id: 73, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:05.063 INFO [0] Sending repair request ReconciliationId(74)
Sep 22 23:15:05.063 INFO [1] received reconcile message
Sep 22 23:15:05.063 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(74), op: ExtentRepair { repair_id: ReconciliationId(74), extent_id: 73, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:05.063 INFO [1] client ExtentRepair { repair_id: ReconciliationId(74), extent_id: 73, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:05.063 INFO [1] No action required ReconciliationId(74)
Sep 22 23:15:05.063 INFO [2] received reconcile message
Sep 22 23:15:05.063 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(74), op: ExtentRepair { repair_id: ReconciliationId(74), extent_id: 73, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:05.063 INFO [2] client ExtentRepair { repair_id: ReconciliationId(74), extent_id: 73, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:05.063 INFO [2] No action required ReconciliationId(74)
Sep 22 23:15:05.063 DEBG 74 Repair extent 73 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:05.064 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/049.copy"
Sep 22 23:15:05.129 INFO accepted connection, remote_addr: 127.0.0.1:36308, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.129 TRCE incoming request, uri: /extent/73/files, method: GET, req_id: b4559a33-b140-4b58-9426-f7bc6a18adce, remote_addr: 127.0.0.1:36308, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.130 INFO request completed, latency_us: 252, response_code: 200, uri: /extent/73/files, method: GET, req_id: b4559a33-b140-4b58-9426-f7bc6a18adce, remote_addr: 127.0.0.1:36308, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.130 INFO eid:73 Found repair files: ["049", "049.db"]
Sep 22 23:15:05.130 TRCE incoming request, uri: /newextent/73/data, method: GET, req_id: ce523846-f7da-47d9-9193-ae9526ee370c, remote_addr: 127.0.0.1:36308, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.131 INFO request completed, latency_us: 360, response_code: 200, uri: /newextent/73/data, method: GET, req_id: ce523846-f7da-47d9-9193-ae9526ee370c, remote_addr: 127.0.0.1:36308, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.136 TRCE incoming request, uri: /newextent/73/db, method: GET, req_id: dada99a2-3a20-419e-8109-443c2aaffcf2, remote_addr: 127.0.0.1:36308, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.136 INFO request completed, latency_us: 298, response_code: 200, uri: /newextent/73/db, method: GET, req_id: dada99a2-3a20-419e-8109-443c2aaffcf2, remote_addr: 127.0.0.1:36308, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.137 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/049.copy" to "/tmp/downstairs-vrx8aK6L/00/000/049.replace"
Sep 22 23:15:05.137 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:05.138 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/049.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:05.138 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/049"
Sep 22 23:15:05.139 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/049.db"
Sep 22 23:15:05.139 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:05.139 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/049.replace" to "/tmp/downstairs-vrx8aK6L/00/000/049.completed"
Sep 22 23:15:05.139 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:05.139 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:05.139 DEBG [0] It's time to notify for 74
Sep 22 23:15:05.139 INFO Completion from [0] id:74 status:true
Sep 22 23:15:05.139 INFO [75/752] Repair commands completed
Sep 22 23:15:05.139 INFO Pop front: ReconcileIO { id: ReconciliationId(75), op: ExtentReopen { repair_id: ReconciliationId(75), extent_id: 73 }, state: ClientData([New, New, New]) }
Sep 22 23:15:05.139 INFO Sent repair work, now wait for resp
Sep 22 23:15:05.139 INFO [0] received reconcile message
Sep 22 23:15:05.139 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(75), op: ExtentReopen { repair_id: ReconciliationId(75), extent_id: 73 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:05.139 INFO [0] client ExtentReopen { repair_id: ReconciliationId(75), extent_id: 73 }
Sep 22 23:15:05.139 INFO [1] received reconcile message
Sep 22 23:15:05.139 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(75), op: ExtentReopen { repair_id: ReconciliationId(75), extent_id: 73 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:05.139 INFO [1] client ExtentReopen { repair_id: ReconciliationId(75), extent_id: 73 }
Sep 22 23:15:05.139 INFO [2] received reconcile message
Sep 22 23:15:05.139 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(75), op: ExtentReopen { repair_id: ReconciliationId(75), extent_id: 73 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:05.139 INFO [2] client ExtentReopen { repair_id: ReconciliationId(75), extent_id: 73 }
Sep 22 23:15:05.140 DEBG 75 Reopen extent 73
Sep 22 23:15:05.140 DEBG 75 Reopen extent 73
Sep 22 23:15:05.141 DEBG 75 Reopen extent 73
Sep 22 23:15:05.141 DEBG [2] It's time to notify for 75
Sep 22 23:15:05.141 INFO Completion from [2] id:75 status:true
Sep 22 23:15:05.141 INFO [76/752] Repair commands completed
Sep 22 23:15:05.141 INFO Pop front: ReconcileIO { id: ReconciliationId(76), op: ExtentFlush { repair_id: ReconciliationId(76), extent_id: 76, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:05.141 INFO Sent repair work, now wait for resp
Sep 22 23:15:05.141 INFO [0] received reconcile message
Sep 22 23:15:05.142 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(76), op: ExtentFlush { repair_id: ReconciliationId(76), extent_id: 76, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:05.142 INFO [0] client ExtentFlush { repair_id: ReconciliationId(76), extent_id: 76, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:05.142 INFO [1] received reconcile message
Sep 22 23:15:05.142 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(76), op: ExtentFlush { repair_id: ReconciliationId(76), extent_id: 76, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:05.142 INFO [1] client ExtentFlush { repair_id: ReconciliationId(76), extent_id: 76, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:05.142 INFO [2] received reconcile message
Sep 22 23:15:05.142 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(76), op: ExtentFlush { repair_id: ReconciliationId(76), extent_id: 76, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:05.142 INFO [2] client ExtentFlush { repair_id: ReconciliationId(76), extent_id: 76, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:05.142 DEBG 76 Flush extent 76 with f:2 g:2
Sep 22 23:15:05.142 DEBG Flush just extent 76 with f:2 and g:2
Sep 22 23:15:05.142 DEBG [1] It's time to notify for 76
Sep 22 23:15:05.142 INFO Completion from [1] id:76 status:true
Sep 22 23:15:05.142 INFO [77/752] Repair commands completed
Sep 22 23:15:05.142 INFO Pop front: ReconcileIO { id: ReconciliationId(77), op: ExtentClose { repair_id: ReconciliationId(77), extent_id: 76 }, state: ClientData([New, New, New]) }
Sep 22 23:15:05.142 INFO Sent repair work, now wait for resp
Sep 22 23:15:05.142 INFO [0] received reconcile message
Sep 22 23:15:05.142 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(77), op: ExtentClose { repair_id: ReconciliationId(77), extent_id: 76 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:05.142 INFO [0] client ExtentClose { repair_id: ReconciliationId(77), extent_id: 76 }
Sep 22 23:15:05.142 INFO [1] received reconcile message
Sep 22 23:15:05.142 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(77), op: ExtentClose { repair_id: ReconciliationId(77), extent_id: 76 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:05.142 INFO [1] client ExtentClose { repair_id: ReconciliationId(77), extent_id: 76 }
Sep 22 23:15:05.142 INFO [2] received reconcile message
Sep 22 23:15:05.142 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(77), op: ExtentClose { repair_id: ReconciliationId(77), extent_id: 76 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:05.142 INFO [2] client ExtentClose { repair_id: ReconciliationId(77), extent_id: 76 }
Sep 22 23:15:05.142 DEBG 77 Close extent 76
Sep 22 23:15:05.143 DEBG 77 Close extent 76
Sep 22 23:15:05.143 DEBG 77 Close extent 76
Sep 22 23:15:05.143 DEBG [2] It's time to notify for 77
Sep 22 23:15:05.143 INFO Completion from [2] id:77 status:true
Sep 22 23:15:05.143 INFO [78/752] Repair commands completed
Sep 22 23:15:05.143 INFO Pop front: ReconcileIO { id: ReconciliationId(78), op: ExtentRepair { repair_id: ReconciliationId(78), extent_id: 76, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:05.143 INFO Sent repair work, now wait for resp
Sep 22 23:15:05.143 INFO [0] received reconcile message
Sep 22 23:15:05.143 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(78), op: ExtentRepair { repair_id: ReconciliationId(78), extent_id: 76, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:05.143 INFO [0] client ExtentRepair { repair_id: ReconciliationId(78), extent_id: 76, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:05.144 INFO [0] Sending repair request ReconciliationId(78)
Sep 22 23:15:05.144 INFO [1] received reconcile message
Sep 22 23:15:05.144 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(78), op: ExtentRepair { repair_id: ReconciliationId(78), extent_id: 76, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:05.144 INFO [1] client ExtentRepair { repair_id: ReconciliationId(78), extent_id: 76, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:05.144 INFO [1] No action required ReconciliationId(78)
Sep 22 23:15:05.144 INFO [2] received reconcile message
Sep 22 23:15:05.144 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(78), op: ExtentRepair { repair_id: ReconciliationId(78), extent_id: 76, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:05.144 INFO [2] client ExtentRepair { repair_id: ReconciliationId(78), extent_id: 76, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:05.144 INFO [2] No action required ReconciliationId(78)
Sep 22 23:15:05.144 DEBG 78 Repair extent 76 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:05.144 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/04C.copy"
Sep 22 23:15:05.206 INFO accepted connection, remote_addr: 127.0.0.1:49240, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.206 TRCE incoming request, uri: /extent/76/files, method: GET, req_id: 6525aa45-7052-4397-8cc6-4c6d18f5d0a7, remote_addr: 127.0.0.1:49240, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.206 INFO request completed, latency_us: 222, response_code: 200, uri: /extent/76/files, method: GET, req_id: 6525aa45-7052-4397-8cc6-4c6d18f5d0a7, remote_addr: 127.0.0.1:49240, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.206 INFO eid:76 Found repair files: ["04C", "04C.db"]
Sep 22 23:15:05.207 TRCE incoming request, uri: /newextent/76/data, method: GET, req_id: 18772ffb-896a-4f08-a622-d072dbb2e496, remote_addr: 127.0.0.1:49240, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.207 INFO request completed, latency_us: 320, response_code: 200, uri: /newextent/76/data, method: GET, req_id: 18772ffb-896a-4f08-a622-d072dbb2e496, remote_addr: 127.0.0.1:49240, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.212 TRCE incoming request, uri: /newextent/76/db, method: GET, req_id: 0172b873-710f-4c6c-8454-8d48192f48a5, remote_addr: 127.0.0.1:49240, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.213 INFO request completed, latency_us: 293, response_code: 200, uri: /newextent/76/db, method: GET, req_id: 0172b873-710f-4c6c-8454-8d48192f48a5, remote_addr: 127.0.0.1:49240, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.214 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/04C.copy" to "/tmp/downstairs-vrx8aK6L/00/000/04C.replace"
Sep 22 23:15:05.214 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:05.215 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/04C.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:05.215 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/04C"
Sep 22 23:15:05.215 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/04C.db"
Sep 22 23:15:05.215 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:05.215 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/04C.replace" to "/tmp/downstairs-vrx8aK6L/00/000/04C.completed"
Sep 22 23:15:05.215 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:05.215 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:05.215 DEBG [0] It's time to notify for 78
Sep 22 23:15:05.215 INFO Completion from [0] id:78 status:true
Sep 22 23:15:05.216 INFO [79/752] Repair commands completed
Sep 22 23:15:05.216 INFO Pop front: ReconcileIO { id: ReconciliationId(79), op: ExtentReopen { repair_id: ReconciliationId(79), extent_id: 76 }, state: ClientData([New, New, New]) }
Sep 22 23:15:05.216 INFO Sent repair work, now wait for resp
Sep 22 23:15:05.216 INFO [0] received reconcile message
Sep 22 23:15:05.216 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(79), op: ExtentReopen { repair_id: ReconciliationId(79), extent_id: 76 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:05.216 INFO [0] client ExtentReopen { repair_id: ReconciliationId(79), extent_id: 76 }
Sep 22 23:15:05.216 INFO [1] received reconcile message
Sep 22 23:15:05.216 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(79), op: ExtentReopen { repair_id: ReconciliationId(79), extent_id: 76 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:05.216 INFO [1] client ExtentReopen { repair_id: ReconciliationId(79), extent_id: 76 }
Sep 22 23:15:05.216 INFO [2] received reconcile message
Sep 22 23:15:05.216 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(79), op: ExtentReopen { repair_id: ReconciliationId(79), extent_id: 76 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:05.216 INFO [2] client ExtentReopen { repair_id: ReconciliationId(79), extent_id: 76 }
Sep 22 23:15:05.216 DEBG 79 Reopen extent 76
Sep 22 23:15:05.217 DEBG 79 Reopen extent 76
Sep 22 23:15:05.217 DEBG 79 Reopen extent 76
Sep 22 23:15:05.218 DEBG [2] It's time to notify for 79
Sep 22 23:15:05.218 INFO Completion from [2] id:79 status:true
Sep 22 23:15:05.218 INFO [80/752] Repair commands completed
Sep 22 23:15:05.218 INFO Pop front: ReconcileIO { id: ReconciliationId(80), op: ExtentFlush { repair_id: ReconciliationId(80), extent_id: 98, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:05.218 INFO Sent repair work, now wait for resp
Sep 22 23:15:05.218 INFO [0] received reconcile message
Sep 22 23:15:05.218 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(80), op: ExtentFlush { repair_id: ReconciliationId(80), extent_id: 98, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:05.218 INFO [0] client ExtentFlush { repair_id: ReconciliationId(80), extent_id: 98, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:05.218 INFO [1] received reconcile message
Sep 22 23:15:05.218 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(80), op: ExtentFlush { repair_id: ReconciliationId(80), extent_id: 98, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:05.218 INFO [1] client ExtentFlush { repair_id: ReconciliationId(80), extent_id: 98, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:05.218 INFO [2] received reconcile message
Sep 22 23:15:05.218 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(80), op: ExtentFlush { repair_id: ReconciliationId(80), extent_id: 98, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:05.218 INFO [2] client ExtentFlush { repair_id: ReconciliationId(80), extent_id: 98, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:05.218 DEBG 80 Flush extent 98 with f:2 g:2
Sep 22 23:15:05.218 DEBG Flush just extent 98 with f:2 and g:2
Sep 22 23:15:05.218 DEBG [1] It's time to notify for 80
Sep 22 23:15:05.218 INFO Completion from [1] id:80 status:true
Sep 22 23:15:05.218 INFO [81/752] Repair commands completed
Sep 22 23:15:05.218 INFO Pop front: ReconcileIO { id: ReconciliationId(81), op: ExtentClose { repair_id: ReconciliationId(81), extent_id: 98 }, state: ClientData([New, New, New]) }
Sep 22 23:15:05.218 INFO Sent repair work, now wait for resp
Sep 22 23:15:05.218 INFO [0] received reconcile message
Sep 22 23:15:05.218 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(81), op: ExtentClose { repair_id: ReconciliationId(81), extent_id: 98 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:05.218 INFO [0] client ExtentClose { repair_id: ReconciliationId(81), extent_id: 98 }
Sep 22 23:15:05.218 INFO [1] received reconcile message
Sep 22 23:15:05.219 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(81), op: ExtentClose { repair_id: ReconciliationId(81), extent_id: 98 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:05.219 INFO [1] client ExtentClose { repair_id: ReconciliationId(81), extent_id: 98 }
Sep 22 23:15:05.219 INFO [2] received reconcile message
Sep 22 23:15:05.219 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(81), op: ExtentClose { repair_id: ReconciliationId(81), extent_id: 98 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:05.219 INFO [2] client ExtentClose { repair_id: ReconciliationId(81), extent_id: 98 }
Sep 22 23:15:05.219 DEBG 81 Close extent 98
Sep 22 23:15:05.219 DEBG 81 Close extent 98
Sep 22 23:15:05.219 DEBG 81 Close extent 98
Sep 22 23:15:05.220 DEBG [2] It's time to notify for 81
Sep 22 23:15:05.220 INFO Completion from [2] id:81 status:true
Sep 22 23:15:05.220 INFO [82/752] Repair commands completed
Sep 22 23:15:05.220 INFO Pop front: ReconcileIO { id: ReconciliationId(82), op: ExtentRepair { repair_id: ReconciliationId(82), extent_id: 98, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:05.220 INFO Sent repair work, now wait for resp
Sep 22 23:15:05.220 INFO [0] received reconcile message
Sep 22 23:15:05.220 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(82), op: ExtentRepair { repair_id: ReconciliationId(82), extent_id: 98, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:05.220 INFO [0] client ExtentRepair { repair_id: ReconciliationId(82), extent_id: 98, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:05.220 INFO [0] Sending repair request ReconciliationId(82)
Sep 22 23:15:05.220 INFO [1] received reconcile message
Sep 22 23:15:05.220 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(82), op: ExtentRepair { repair_id: ReconciliationId(82), extent_id: 98, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:05.220 INFO [1] client ExtentRepair { repair_id: ReconciliationId(82), extent_id: 98, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:05.220 INFO [1] No action required ReconciliationId(82)
Sep 22 23:15:05.220 INFO [2] received reconcile message
Sep 22 23:15:05.220 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(82), op: ExtentRepair { repair_id: ReconciliationId(82), extent_id: 98, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:05.220 INFO [2] client ExtentRepair { repair_id: ReconciliationId(82), extent_id: 98, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:05.220 INFO [2] No action required ReconciliationId(82)
Sep 22 23:15:05.220 DEBG 82 Repair extent 98 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:05.220 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/062.copy"
Sep 22 23:15:05.283 INFO accepted connection, remote_addr: 127.0.0.1:35298, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.283 TRCE incoming request, uri: /extent/98/files, method: GET, req_id: e709a5f2-eab0-444e-bfa2-0473c5efe0d2, remote_addr: 127.0.0.1:35298, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.283 INFO request completed, latency_us: 219, response_code: 200, uri: /extent/98/files, method: GET, req_id: e709a5f2-eab0-444e-bfa2-0473c5efe0d2, remote_addr: 127.0.0.1:35298, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.283 INFO eid:98 Found repair files: ["062", "062.db"]
Sep 22 23:15:05.284 TRCE incoming request, uri: /newextent/98/data, method: GET, req_id: 8ed1c96b-b133-4d9a-bb38-104a681dc5ef, remote_addr: 127.0.0.1:35298, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.284 INFO request completed, latency_us: 318, response_code: 200, uri: /newextent/98/data, method: GET, req_id: 8ed1c96b-b133-4d9a-bb38-104a681dc5ef, remote_addr: 127.0.0.1:35298, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.289 TRCE incoming request, uri: /newextent/98/db, method: GET, req_id: 59d19b04-69b0-4ac4-ab4c-1439fc6749c9, remote_addr: 127.0.0.1:35298, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.290 INFO request completed, latency_us: 294, response_code: 200, uri: /newextent/98/db, method: GET, req_id: 59d19b04-69b0-4ac4-ab4c-1439fc6749c9, remote_addr: 127.0.0.1:35298, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.291 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/062.copy" to "/tmp/downstairs-vrx8aK6L/00/000/062.replace"
Sep 22 23:15:05.291 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:05.292 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/062.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:05.292 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/062"
Sep 22 23:15:05.292 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/062.db"
Sep 22 23:15:05.292 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:05.292 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/062.replace" to "/tmp/downstairs-vrx8aK6L/00/000/062.completed"
Sep 22 23:15:05.292 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:05.292 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:05.292 DEBG [0] It's time to notify for 82
Sep 22 23:15:05.292 INFO Completion from [0] id:82 status:true
Sep 22 23:15:05.292 INFO [83/752] Repair commands completed
Sep 22 23:15:05.293 INFO Pop front: ReconcileIO { id: ReconciliationId(83), op: ExtentReopen { repair_id: ReconciliationId(83), extent_id: 98 }, state: ClientData([New, New, New]) }
Sep 22 23:15:05.293 INFO Sent repair work, now wait for resp
Sep 22 23:15:05.293 INFO [0] received reconcile message
Sep 22 23:15:05.293 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(83), op: ExtentReopen { repair_id: ReconciliationId(83), extent_id: 98 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:05.293 INFO [0] client ExtentReopen { repair_id: ReconciliationId(83), extent_id: 98 }
Sep 22 23:15:05.293 INFO [1] received reconcile message
Sep 22 23:15:05.293 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(83), op: ExtentReopen { repair_id: ReconciliationId(83), extent_id: 98 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:05.293 INFO [1] client ExtentReopen { repair_id: ReconciliationId(83), extent_id: 98 }
Sep 22 23:15:05.293 INFO [2] received reconcile message
Sep 22 23:15:05.293 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(83), op: ExtentReopen { repair_id: ReconciliationId(83), extent_id: 98 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:05.293 INFO [2] client ExtentReopen { repair_id: ReconciliationId(83), extent_id: 98 }
Sep 22 23:15:05.293 DEBG 83 Reopen extent 98
Sep 22 23:15:05.294 DEBG 83 Reopen extent 98
Sep 22 23:15:05.294 DEBG 83 Reopen extent 98
Sep 22 23:15:05.295 DEBG [2] It's time to notify for 83
Sep 22 23:15:05.295 INFO Completion from [2] id:83 status:true
Sep 22 23:15:05.295 INFO [84/752] Repair commands completed
Sep 22 23:15:05.295 INFO Pop front: ReconcileIO { id: ReconciliationId(84), op: ExtentFlush { repair_id: ReconciliationId(84), extent_id: 65, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:05.295 INFO Sent repair work, now wait for resp
Sep 22 23:15:05.295 INFO [0] received reconcile message
Sep 22 23:15:05.295 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(84), op: ExtentFlush { repair_id: ReconciliationId(84), extent_id: 65, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:05.295 INFO [0] client ExtentFlush { repair_id: ReconciliationId(84), extent_id: 65, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:05.295 INFO [1] received reconcile message
Sep 22 23:15:05.295 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(84), op: ExtentFlush { repair_id: ReconciliationId(84), extent_id: 65, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:05.295 INFO [1] client ExtentFlush { repair_id: ReconciliationId(84), extent_id: 65, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:05.295 INFO [2] received reconcile message
Sep 22 23:15:05.295 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(84), op: ExtentFlush { repair_id: ReconciliationId(84), extent_id: 65, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:05.295 INFO [2] client ExtentFlush { repair_id: ReconciliationId(84), extent_id: 65, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:05.295 DEBG 84 Flush extent 65 with f:2 g:2
Sep 22 23:15:05.295 DEBG Flush just extent 65 with f:2 and g:2
Sep 22 23:15:05.295 DEBG [1] It's time to notify for 84
Sep 22 23:15:05.295 INFO Completion from [1] id:84 status:true
Sep 22 23:15:05.295 INFO [85/752] Repair commands completed
Sep 22 23:15:05.295 INFO Pop front: ReconcileIO { id: ReconciliationId(85), op: ExtentClose { repair_id: ReconciliationId(85), extent_id: 65 }, state: ClientData([New, New, New]) }
Sep 22 23:15:05.295 INFO Sent repair work, now wait for resp
Sep 22 23:15:05.295 INFO [0] received reconcile message
Sep 22 23:15:05.295 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(85), op: ExtentClose { repair_id: ReconciliationId(85), extent_id: 65 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:05.295 INFO [0] client ExtentClose { repair_id: ReconciliationId(85), extent_id: 65 }
Sep 22 23:15:05.295 INFO [1] received reconcile message
Sep 22 23:15:05.295 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(85), op: ExtentClose { repair_id: ReconciliationId(85), extent_id: 65 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:05.296 INFO [1] client ExtentClose { repair_id: ReconciliationId(85), extent_id: 65 }
Sep 22 23:15:05.296 INFO [2] received reconcile message
Sep 22 23:15:05.296 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(85), op: ExtentClose { repair_id: ReconciliationId(85), extent_id: 65 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:05.296 INFO [2] client ExtentClose { repair_id: ReconciliationId(85), extent_id: 65 }
Sep 22 23:15:05.296 DEBG 85 Close extent 65
Sep 22 23:15:05.296 DEBG 85 Close extent 65
Sep 22 23:15:05.296 DEBG 85 Close extent 65
Sep 22 23:15:05.297 DEBG [2] It's time to notify for 85
Sep 22 23:15:05.297 INFO Completion from [2] id:85 status:true
Sep 22 23:15:05.297 INFO [86/752] Repair commands completed
Sep 22 23:15:05.297 INFO Pop front: ReconcileIO { id: ReconciliationId(86), op: ExtentRepair { repair_id: ReconciliationId(86), extent_id: 65, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:05.297 INFO Sent repair work, now wait for resp
Sep 22 23:15:05.297 INFO [0] received reconcile message
Sep 22 23:15:05.297 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(86), op: ExtentRepair { repair_id: ReconciliationId(86), extent_id: 65, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:05.297 INFO [0] client ExtentRepair { repair_id: ReconciliationId(86), extent_id: 65, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:05.297 INFO [0] Sending repair request ReconciliationId(86)
Sep 22 23:15:05.297 INFO [1] received reconcile message
Sep 22 23:15:05.297 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(86), op: ExtentRepair { repair_id: ReconciliationId(86), extent_id: 65, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:05.297 INFO [1] client ExtentRepair { repair_id: ReconciliationId(86), extent_id: 65, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:05.297 INFO [1] No action required ReconciliationId(86)
Sep 22 23:15:05.297 INFO [2] received reconcile message
Sep 22 23:15:05.297 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(86), op: ExtentRepair { repair_id: ReconciliationId(86), extent_id: 65, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:05.297 INFO [2] client ExtentRepair { repair_id: ReconciliationId(86), extent_id: 65, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:05.297 INFO [2] No action required ReconciliationId(86)
Sep 22 23:15:05.297 DEBG 86 Repair extent 65 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:05.297 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/041.copy"
Sep 22 23:15:05.360 INFO accepted connection, remote_addr: 127.0.0.1:60292, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.360 TRCE incoming request, uri: /extent/65/files, method: GET, req_id: 9b156b1c-7db5-41f5-9b30-015bbd1e6a61, remote_addr: 127.0.0.1:60292, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.360 INFO request completed, latency_us: 198, response_code: 200, uri: /extent/65/files, method: GET, req_id: 9b156b1c-7db5-41f5-9b30-015bbd1e6a61, remote_addr: 127.0.0.1:60292, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.360 INFO eid:65 Found repair files: ["041", "041.db"]
Sep 22 23:15:05.361 TRCE incoming request, uri: /newextent/65/data, method: GET, req_id: 40cab528-10b3-486c-9b74-258628ea0a15, remote_addr: 127.0.0.1:60292, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.361 INFO request completed, latency_us: 304, response_code: 200, uri: /newextent/65/data, method: GET, req_id: 40cab528-10b3-486c-9b74-258628ea0a15, remote_addr: 127.0.0.1:60292, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.366 TRCE incoming request, uri: /newextent/65/db, method: GET, req_id: f2b6d242-86da-4686-acb1-daadfdbe53ac, remote_addr: 127.0.0.1:60292, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.367 INFO request completed, latency_us: 293, response_code: 200, uri: /newextent/65/db, method: GET, req_id: f2b6d242-86da-4686-acb1-daadfdbe53ac, remote_addr: 127.0.0.1:60292, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.368 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/041.copy" to "/tmp/downstairs-vrx8aK6L/00/000/041.replace"
Sep 22 23:15:05.368 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:05.369 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/041.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:05.369 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/041"
Sep 22 23:15:05.369 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/041.db"
Sep 22 23:15:05.369 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:05.369 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/041.replace" to "/tmp/downstairs-vrx8aK6L/00/000/041.completed"
Sep 22 23:15:05.369 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:05.369 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:05.369 DEBG [0] It's time to notify for 86
Sep 22 23:15:05.369 INFO Completion from [0] id:86 status:true
Sep 22 23:15:05.369 INFO [87/752] Repair commands completed
Sep 22 23:15:05.369 INFO Pop front: ReconcileIO { id: ReconciliationId(87), op: ExtentReopen { repair_id: ReconciliationId(87), extent_id: 65 }, state: ClientData([New, New, New]) }
Sep 22 23:15:05.369 INFO Sent repair work, now wait for resp
Sep 22 23:15:05.369 INFO [0] received reconcile message
Sep 22 23:15:05.369 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(87), op: ExtentReopen { repair_id: ReconciliationId(87), extent_id: 65 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:05.369 INFO [0] client ExtentReopen { repair_id: ReconciliationId(87), extent_id: 65 }
Sep 22 23:15:05.369 INFO [1] received reconcile message
Sep 22 23:15:05.370 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(87), op: ExtentReopen { repair_id: ReconciliationId(87), extent_id: 65 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:05.370 INFO [1] client ExtentReopen { repair_id: ReconciliationId(87), extent_id: 65 }
Sep 22 23:15:05.370 INFO [2] received reconcile message
Sep 22 23:15:05.370 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(87), op: ExtentReopen { repair_id: ReconciliationId(87), extent_id: 65 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:05.370 INFO [2] client ExtentReopen { repair_id: ReconciliationId(87), extent_id: 65 }
Sep 22 23:15:05.370 DEBG 87 Reopen extent 65
Sep 22 23:15:05.370 DEBG 87 Reopen extent 65
Sep 22 23:15:05.371 DEBG 87 Reopen extent 65
Sep 22 23:15:05.371 DEBG [2] It's time to notify for 87
Sep 22 23:15:05.371 INFO Completion from [2] id:87 status:true
Sep 22 23:15:05.371 INFO [88/752] Repair commands completed
Sep 22 23:15:05.371 INFO Pop front: ReconcileIO { id: ReconciliationId(88), op: ExtentFlush { repair_id: ReconciliationId(88), extent_id: 45, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:05.371 INFO Sent repair work, now wait for resp
Sep 22 23:15:05.372 INFO [0] received reconcile message
Sep 22 23:15:05.372 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(88), op: ExtentFlush { repair_id: ReconciliationId(88), extent_id: 45, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:05.372 INFO [0] client ExtentFlush { repair_id: ReconciliationId(88), extent_id: 45, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:05.372 INFO [1] received reconcile message
Sep 22 23:15:05.372 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(88), op: ExtentFlush { repair_id: ReconciliationId(88), extent_id: 45, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:05.372 INFO [1] client ExtentFlush { repair_id: ReconciliationId(88), extent_id: 45, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:05.372 INFO [2] received reconcile message
Sep 22 23:15:05.372 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(88), op: ExtentFlush { repair_id: ReconciliationId(88), extent_id: 45, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:05.372 INFO [2] client ExtentFlush { repair_id: ReconciliationId(88), extent_id: 45, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:05.372 DEBG 88 Flush extent 45 with f:2 g:2
Sep 22 23:15:05.372 DEBG Flush just extent 45 with f:2 and g:2
Sep 22 23:15:05.372 DEBG [1] It's time to notify for 88
Sep 22 23:15:05.372 INFO Completion from [1] id:88 status:true
Sep 22 23:15:05.372 INFO [89/752] Repair commands completed
Sep 22 23:15:05.372 INFO Pop front: ReconcileIO { id: ReconciliationId(89), op: ExtentClose { repair_id: ReconciliationId(89), extent_id: 45 }, state: ClientData([New, New, New]) }
Sep 22 23:15:05.372 INFO Sent repair work, now wait for resp
Sep 22 23:15:05.372 INFO [0] received reconcile message
Sep 22 23:15:05.372 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(89), op: ExtentClose { repair_id: ReconciliationId(89), extent_id: 45 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:05.372 INFO [0] client ExtentClose { repair_id: ReconciliationId(89), extent_id: 45 }
Sep 22 23:15:05.372 INFO [1] received reconcile message
Sep 22 23:15:05.372 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(89), op: ExtentClose { repair_id: ReconciliationId(89), extent_id: 45 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:05.372 INFO [1] client ExtentClose { repair_id: ReconciliationId(89), extent_id: 45 }
Sep 22 23:15:05.372 INFO [2] received reconcile message
Sep 22 23:15:05.372 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(89), op: ExtentClose { repair_id: ReconciliationId(89), extent_id: 45 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:05.372 INFO [2] client ExtentClose { repair_id: ReconciliationId(89), extent_id: 45 }
Sep 22 23:15:05.372 DEBG 89 Close extent 45
Sep 22 23:15:05.373 DEBG 89 Close extent 45
Sep 22 23:15:05.373 DEBG 89 Close extent 45
Sep 22 23:15:05.373 DEBG [2] It's time to notify for 89
Sep 22 23:15:05.373 INFO Completion from [2] id:89 status:true
Sep 22 23:15:05.373 INFO [90/752] Repair commands completed
Sep 22 23:15:05.373 INFO Pop front: ReconcileIO { id: ReconciliationId(90), op: ExtentRepair { repair_id: ReconciliationId(90), extent_id: 45, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:05.373 INFO Sent repair work, now wait for resp
Sep 22 23:15:05.373 INFO [0] received reconcile message
Sep 22 23:15:05.374 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(90), op: ExtentRepair { repair_id: ReconciliationId(90), extent_id: 45, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:05.374 INFO [0] client ExtentRepair { repair_id: ReconciliationId(90), extent_id: 45, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:05.374 INFO [0] Sending repair request ReconciliationId(90)
Sep 22 23:15:05.374 INFO [1] received reconcile message
Sep 22 23:15:05.374 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(90), op: ExtentRepair { repair_id: ReconciliationId(90), extent_id: 45, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:05.374 INFO [1] client ExtentRepair { repair_id: ReconciliationId(90), extent_id: 45, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:05.374 INFO [1] No action required ReconciliationId(90)
Sep 22 23:15:05.374 INFO [2] received reconcile message
Sep 22 23:15:05.374 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(90), op: ExtentRepair { repair_id: ReconciliationId(90), extent_id: 45, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:05.374 INFO [2] client ExtentRepair { repair_id: ReconciliationId(90), extent_id: 45, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:05.374 INFO [2] No action required ReconciliationId(90)
Sep 22 23:15:05.374 DEBG 90 Repair extent 45 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:05.374 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/02D.copy"
Sep 22 23:15:05.389 DEBG up_ds_listen was notified
Sep 22 23:15:05.389 DEBG up_ds_listen process 1073
Sep 22 23:15:05.389 DEBG [A] ack job 1073:74, : downstairs
Sep 22 23:15:05.389 DEBG [rc] retire 1073 clears [JobId(1072), JobId(1073)], : downstairs
Sep 22 23:15:05.389 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:05.399 INFO [lossy] skipping 1075
Sep 22 23:15:05.399 DEBG Flush     :1075 extent_limit None deps:[JobId(1074), JobId(1073)] res:true f:27 g:1
Sep 22 23:15:05.405 DEBG Read      :1076 deps:[JobId(1075)] res:true
Sep 22 23:15:05.427 DEBG Flush     :1075 extent_limit None deps:[JobId(1074), JobId(1073)] res:true f:27 g:1
Sep 22 23:15:05.427 WARN returning error on read!
Sep 22 23:15:05.427 DEBG Read      :1076 deps:[JobId(1075)] res:false
Sep 22 23:15:05.427 INFO [lossy] skipping 1076
Sep 22 23:15:05.433 DEBG Read      :1076 deps:[JobId(1075)] res:true
Sep 22 23:15:05.437 INFO accepted connection, remote_addr: 127.0.0.1:64899, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.437 TRCE incoming request, uri: /extent/45/files, method: GET, req_id: 7665d8c0-3237-4ecd-90ae-e279e9b97fb5, remote_addr: 127.0.0.1:64899, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.437 INFO request completed, latency_us: 219, response_code: 200, uri: /extent/45/files, method: GET, req_id: 7665d8c0-3237-4ecd-90ae-e279e9b97fb5, remote_addr: 127.0.0.1:64899, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.438 INFO eid:45 Found repair files: ["02D", "02D.db"]
Sep 22 23:15:05.438 TRCE incoming request, uri: /newextent/45/data, method: GET, req_id: d0224391-7635-4a4b-bb92-77afd82e7c7f, remote_addr: 127.0.0.1:64899, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.438 INFO request completed, latency_us: 326, response_code: 200, uri: /newextent/45/data, method: GET, req_id: d0224391-7635-4a4b-bb92-77afd82e7c7f, remote_addr: 127.0.0.1:64899, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.444 TRCE incoming request, uri: /newextent/45/db, method: GET, req_id: 6efbea0d-463f-4565-a920-2c6829832ccb, remote_addr: 127.0.0.1:64899, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.444 INFO request completed, latency_us: 296, response_code: 200, uri: /newextent/45/db, method: GET, req_id: 6efbea0d-463f-4565-a920-2c6829832ccb, remote_addr: 127.0.0.1:64899, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.445 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/02D.copy" to "/tmp/downstairs-vrx8aK6L/00/000/02D.replace"
Sep 22 23:15:05.445 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:05.446 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/02D.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:05.446 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/02D"
Sep 22 23:15:05.446 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/02D.db"
Sep 22 23:15:05.446 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:05.446 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/02D.replace" to "/tmp/downstairs-vrx8aK6L/00/000/02D.completed"
Sep 22 23:15:05.446 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:05.446 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:05.447 DEBG [0] It's time to notify for 90
Sep 22 23:15:05.447 INFO Completion from [0] id:90 status:true
Sep 22 23:15:05.447 INFO [91/752] Repair commands completed
Sep 22 23:15:05.447 INFO Pop front: ReconcileIO { id: ReconciliationId(91), op: ExtentReopen { repair_id: ReconciliationId(91), extent_id: 45 }, state: ClientData([New, New, New]) }
Sep 22 23:15:05.447 INFO Sent repair work, now wait for resp
Sep 22 23:15:05.447 INFO [0] received reconcile message
Sep 22 23:15:05.447 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(91), op: ExtentReopen { repair_id: ReconciliationId(91), extent_id: 45 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:05.447 INFO [0] client ExtentReopen { repair_id: ReconciliationId(91), extent_id: 45 }
Sep 22 23:15:05.447 INFO [1] received reconcile message
Sep 22 23:15:05.447 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(91), op: ExtentReopen { repair_id: ReconciliationId(91), extent_id: 45 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:05.447 INFO [1] client ExtentReopen { repair_id: ReconciliationId(91), extent_id: 45 }
Sep 22 23:15:05.447 INFO [2] received reconcile message
Sep 22 23:15:05.447 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(91), op: ExtentReopen { repair_id: ReconciliationId(91), extent_id: 45 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:05.447 INFO [2] client ExtentReopen { repair_id: ReconciliationId(91), extent_id: 45 }
Sep 22 23:15:05.447 DEBG 91 Reopen extent 45
Sep 22 23:15:05.448 DEBG 91 Reopen extent 45
Sep 22 23:15:05.448 DEBG 91 Reopen extent 45
Sep 22 23:15:05.449 DEBG [2] It's time to notify for 91
Sep 22 23:15:05.449 INFO Completion from [2] id:91 status:true
Sep 22 23:15:05.449 INFO [92/752] Repair commands completed
Sep 22 23:15:05.449 INFO Pop front: ReconcileIO { id: ReconciliationId(92), op: ExtentFlush { repair_id: ReconciliationId(92), extent_id: 75, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:05.449 INFO Sent repair work, now wait for resp
Sep 22 23:15:05.449 INFO [0] received reconcile message
Sep 22 23:15:05.449 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(92), op: ExtentFlush { repair_id: ReconciliationId(92), extent_id: 75, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:05.449 INFO [0] client ExtentFlush { repair_id: ReconciliationId(92), extent_id: 75, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:05.449 INFO [1] received reconcile message
Sep 22 23:15:05.449 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(92), op: ExtentFlush { repair_id: ReconciliationId(92), extent_id: 75, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:05.449 INFO [1] client ExtentFlush { repair_id: ReconciliationId(92), extent_id: 75, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:05.449 INFO [2] received reconcile message
Sep 22 23:15:05.449 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(92), op: ExtentFlush { repair_id: ReconciliationId(92), extent_id: 75, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:05.449 INFO [2] client ExtentFlush { repair_id: ReconciliationId(92), extent_id: 75, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:05.449 DEBG 92 Flush extent 75 with f:2 g:2
Sep 22 23:15:05.449 DEBG Flush just extent 75 with f:2 and g:2
Sep 22 23:15:05.450 DEBG [1] It's time to notify for 92
Sep 22 23:15:05.450 INFO Completion from [1] id:92 status:true
Sep 22 23:15:05.450 INFO [93/752] Repair commands completed
Sep 22 23:15:05.450 INFO Pop front: ReconcileIO { id: ReconciliationId(93), op: ExtentClose { repair_id: ReconciliationId(93), extent_id: 75 }, state: ClientData([New, New, New]) }
Sep 22 23:15:05.450 INFO Sent repair work, now wait for resp
Sep 22 23:15:05.450 INFO [0] received reconcile message
Sep 22 23:15:05.450 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(93), op: ExtentClose { repair_id: ReconciliationId(93), extent_id: 75 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:05.450 INFO [0] client ExtentClose { repair_id: ReconciliationId(93), extent_id: 75 }
Sep 22 23:15:05.450 INFO [1] received reconcile message
Sep 22 23:15:05.450 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(93), op: ExtentClose { repair_id: ReconciliationId(93), extent_id: 75 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:05.450 INFO [1] client ExtentClose { repair_id: ReconciliationId(93), extent_id: 75 }
Sep 22 23:15:05.450 INFO [2] received reconcile message
Sep 22 23:15:05.450 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(93), op: ExtentClose { repair_id: ReconciliationId(93), extent_id: 75 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:05.450 INFO [2] client ExtentClose { repair_id: ReconciliationId(93), extent_id: 75 }
Sep 22 23:15:05.450 DEBG 93 Close extent 75
Sep 22 23:15:05.450 DEBG 93 Close extent 75
Sep 22 23:15:05.451 DEBG 93 Close extent 75
Sep 22 23:15:05.451 DEBG [2] It's time to notify for 93
Sep 22 23:15:05.451 INFO Completion from [2] id:93 status:true
Sep 22 23:15:05.451 INFO [94/752] Repair commands completed
Sep 22 23:15:05.451 INFO Pop front: ReconcileIO { id: ReconciliationId(94), op: ExtentRepair { repair_id: ReconciliationId(94), extent_id: 75, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:05.451 INFO Sent repair work, now wait for resp
Sep 22 23:15:05.451 INFO [0] received reconcile message
Sep 22 23:15:05.451 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(94), op: ExtentRepair { repair_id: ReconciliationId(94), extent_id: 75, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:05.451 INFO [0] client ExtentRepair { repair_id: ReconciliationId(94), extent_id: 75, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:05.451 INFO [0] Sending repair request ReconciliationId(94)
Sep 22 23:15:05.451 INFO [1] received reconcile message
Sep 22 23:15:05.451 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(94), op: ExtentRepair { repair_id: ReconciliationId(94), extent_id: 75, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:05.451 INFO [1] client ExtentRepair { repair_id: ReconciliationId(94), extent_id: 75, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:05.451 INFO [1] No action required ReconciliationId(94)
Sep 22 23:15:05.451 INFO [2] received reconcile message
Sep 22 23:15:05.451 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(94), op: ExtentRepair { repair_id: ReconciliationId(94), extent_id: 75, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:05.451 INFO [2] client ExtentRepair { repair_id: ReconciliationId(94), extent_id: 75, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:05.451 INFO [2] No action required ReconciliationId(94)
Sep 22 23:15:05.451 DEBG 94 Repair extent 75 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:05.451 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/04B.copy"
Sep 22 23:15:05.515 INFO accepted connection, remote_addr: 127.0.0.1:50873, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.515 TRCE incoming request, uri: /extent/75/files, method: GET, req_id: c8a73894-aa38-4869-8bcf-d5050c96838e, remote_addr: 127.0.0.1:50873, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.515 INFO request completed, latency_us: 195, response_code: 200, uri: /extent/75/files, method: GET, req_id: c8a73894-aa38-4869-8bcf-d5050c96838e, remote_addr: 127.0.0.1:50873, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.515 INFO eid:75 Found repair files: ["04B", "04B.db"]
Sep 22 23:15:05.516 TRCE incoming request, uri: /newextent/75/data, method: GET, req_id: 69ae22b0-2cb2-4ddc-ac03-774a417fdc84, remote_addr: 127.0.0.1:50873, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.516 INFO request completed, latency_us: 299, response_code: 200, uri: /newextent/75/data, method: GET, req_id: 69ae22b0-2cb2-4ddc-ac03-774a417fdc84, remote_addr: 127.0.0.1:50873, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.521 TRCE incoming request, uri: /newextent/75/db, method: GET, req_id: 017650c0-2c81-4539-b4a3-3feebbaed40c, remote_addr: 127.0.0.1:50873, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.521 INFO request completed, latency_us: 291, response_code: 200, uri: /newextent/75/db, method: GET, req_id: 017650c0-2c81-4539-b4a3-3feebbaed40c, remote_addr: 127.0.0.1:50873, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.523 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/04B.copy" to "/tmp/downstairs-vrx8aK6L/00/000/04B.replace"
Sep 22 23:15:05.523 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:05.523 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/04B.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:05.524 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/04B"
Sep 22 23:15:05.524 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/04B.db"
Sep 22 23:15:05.524 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:05.524 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/04B.replace" to "/tmp/downstairs-vrx8aK6L/00/000/04B.completed"
Sep 22 23:15:05.524 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:05.524 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:05.524 DEBG [0] It's time to notify for 94
Sep 22 23:15:05.524 INFO Completion from [0] id:94 status:true
Sep 22 23:15:05.524 INFO [95/752] Repair commands completed
Sep 22 23:15:05.524 INFO Pop front: ReconcileIO { id: ReconciliationId(95), op: ExtentReopen { repair_id: ReconciliationId(95), extent_id: 75 }, state: ClientData([New, New, New]) }
Sep 22 23:15:05.524 INFO Sent repair work, now wait for resp
Sep 22 23:15:05.524 INFO [0] received reconcile message
Sep 22 23:15:05.524 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(95), op: ExtentReopen { repair_id: ReconciliationId(95), extent_id: 75 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:05.524 INFO [0] client ExtentReopen { repair_id: ReconciliationId(95), extent_id: 75 }
Sep 22 23:15:05.524 INFO [1] received reconcile message
Sep 22 23:15:05.524 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(95), op: ExtentReopen { repair_id: ReconciliationId(95), extent_id: 75 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:05.524 INFO [1] client ExtentReopen { repair_id: ReconciliationId(95), extent_id: 75 }
Sep 22 23:15:05.524 INFO [2] received reconcile message
Sep 22 23:15:05.524 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(95), op: ExtentReopen { repair_id: ReconciliationId(95), extent_id: 75 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:05.525 INFO [2] client ExtentReopen { repair_id: ReconciliationId(95), extent_id: 75 }
Sep 22 23:15:05.525 DEBG 95 Reopen extent 75
Sep 22 23:15:05.525 DEBG 95 Reopen extent 75
Sep 22 23:15:05.526 DEBG 95 Reopen extent 75
Sep 22 23:15:05.526 DEBG [2] It's time to notify for 95
Sep 22 23:15:05.526 INFO Completion from [2] id:95 status:true
Sep 22 23:15:05.526 INFO [96/752] Repair commands completed
Sep 22 23:15:05.526 INFO Pop front: ReconcileIO { id: ReconciliationId(96), op: ExtentFlush { repair_id: ReconciliationId(96), extent_id: 68, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:05.526 INFO Sent repair work, now wait for resp
Sep 22 23:15:05.526 INFO [0] received reconcile message
Sep 22 23:15:05.526 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(96), op: ExtentFlush { repair_id: ReconciliationId(96), extent_id: 68, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:05.526 INFO [0] client ExtentFlush { repair_id: ReconciliationId(96), extent_id: 68, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:05.527 INFO [1] received reconcile message
Sep 22 23:15:05.527 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(96), op: ExtentFlush { repair_id: ReconciliationId(96), extent_id: 68, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:05.527 INFO [1] client ExtentFlush { repair_id: ReconciliationId(96), extent_id: 68, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:05.527 INFO [2] received reconcile message
Sep 22 23:15:05.527 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(96), op: ExtentFlush { repair_id: ReconciliationId(96), extent_id: 68, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:05.527 INFO [2] client ExtentFlush { repair_id: ReconciliationId(96), extent_id: 68, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:05.527 DEBG 96 Flush extent 68 with f:2 g:2
Sep 22 23:15:05.527 DEBG Flush just extent 68 with f:2 and g:2
Sep 22 23:15:05.527 DEBG [1] It's time to notify for 96
Sep 22 23:15:05.527 INFO Completion from [1] id:96 status:true
Sep 22 23:15:05.527 INFO [97/752] Repair commands completed
Sep 22 23:15:05.527 INFO Pop front: ReconcileIO { id: ReconciliationId(97), op: ExtentClose { repair_id: ReconciliationId(97), extent_id: 68 }, state: ClientData([New, New, New]) }
Sep 22 23:15:05.527 INFO Sent repair work, now wait for resp
Sep 22 23:15:05.527 INFO [0] received reconcile message
Sep 22 23:15:05.527 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(97), op: ExtentClose { repair_id: ReconciliationId(97), extent_id: 68 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:05.527 INFO [0] client ExtentClose { repair_id: ReconciliationId(97), extent_id: 68 }
Sep 22 23:15:05.527 INFO [1] received reconcile message
Sep 22 23:15:05.527 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(97), op: ExtentClose { repair_id: ReconciliationId(97), extent_id: 68 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:05.527 INFO [1] client ExtentClose { repair_id: ReconciliationId(97), extent_id: 68 }
Sep 22 23:15:05.527 INFO [2] received reconcile message
Sep 22 23:15:05.527 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(97), op: ExtentClose { repair_id: ReconciliationId(97), extent_id: 68 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:05.527 INFO [2] client ExtentClose { repair_id: ReconciliationId(97), extent_id: 68 }
Sep 22 23:15:05.527 DEBG 97 Close extent 68
Sep 22 23:15:05.528 DEBG 97 Close extent 68
Sep 22 23:15:05.528 DEBG 97 Close extent 68
Sep 22 23:15:05.528 DEBG [2] It's time to notify for 97
Sep 22 23:15:05.528 INFO Completion from [2] id:97 status:true
Sep 22 23:15:05.528 INFO [98/752] Repair commands completed
Sep 22 23:15:05.528 INFO Pop front: ReconcileIO { id: ReconciliationId(98), op: ExtentRepair { repair_id: ReconciliationId(98), extent_id: 68, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:05.528 INFO Sent repair work, now wait for resp
Sep 22 23:15:05.528 INFO [0] received reconcile message
Sep 22 23:15:05.528 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(98), op: ExtentRepair { repair_id: ReconciliationId(98), extent_id: 68, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:05.528 INFO [0] client ExtentRepair { repair_id: ReconciliationId(98), extent_id: 68, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:05.528 INFO [0] Sending repair request ReconciliationId(98)
Sep 22 23:15:05.529 INFO [1] received reconcile message
Sep 22 23:15:05.529 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(98), op: ExtentRepair { repair_id: ReconciliationId(98), extent_id: 68, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:05.529 INFO [1] client ExtentRepair { repair_id: ReconciliationId(98), extent_id: 68, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:05.529 INFO [1] No action required ReconciliationId(98)
Sep 22 23:15:05.529 INFO [2] received reconcile message
Sep 22 23:15:05.529 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(98), op: ExtentRepair { repair_id: ReconciliationId(98), extent_id: 68, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:05.529 INFO [2] client ExtentRepair { repair_id: ReconciliationId(98), extent_id: 68, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:05.529 INFO [2] No action required ReconciliationId(98)
Sep 22 23:15:05.529 DEBG 98 Repair extent 68 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:05.529 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/044.copy"
Sep 22 23:15:05.590 INFO accepted connection, remote_addr: 127.0.0.1:41162, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.591 TRCE incoming request, uri: /extent/68/files, method: GET, req_id: 41302677-ca33-48a0-9837-6aedfabdb11c, remote_addr: 127.0.0.1:41162, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.591 INFO request completed, latency_us: 212, response_code: 200, uri: /extent/68/files, method: GET, req_id: 41302677-ca33-48a0-9837-6aedfabdb11c, remote_addr: 127.0.0.1:41162, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.591 INFO eid:68 Found repair files: ["044", "044.db"]
Sep 22 23:15:05.591 TRCE incoming request, uri: /newextent/68/data, method: GET, req_id: 9b4ffae6-9ad6-4917-9440-507085965135, remote_addr: 127.0.0.1:41162, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.592 INFO request completed, latency_us: 329, response_code: 200, uri: /newextent/68/data, method: GET, req_id: 9b4ffae6-9ad6-4917-9440-507085965135, remote_addr: 127.0.0.1:41162, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.597 TRCE incoming request, uri: /newextent/68/db, method: GET, req_id: 8c9346ee-33e6-4938-b5e9-c56b76b88056, remote_addr: 127.0.0.1:41162, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.597 INFO request completed, latency_us: 291, response_code: 200, uri: /newextent/68/db, method: GET, req_id: 8c9346ee-33e6-4938-b5e9-c56b76b88056, remote_addr: 127.0.0.1:41162, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.598 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/044.copy" to "/tmp/downstairs-vrx8aK6L/00/000/044.replace"
Sep 22 23:15:05.598 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:05.599 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/044.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:05.599 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/044"
Sep 22 23:15:05.599 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/044.db"
Sep 22 23:15:05.599 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:05.599 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/044.replace" to "/tmp/downstairs-vrx8aK6L/00/000/044.completed"
Sep 22 23:15:05.600 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:05.600 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:05.600 DEBG [0] It's time to notify for 98
Sep 22 23:15:05.600 INFO Completion from [0] id:98 status:true
Sep 22 23:15:05.600 INFO [99/752] Repair commands completed
Sep 22 23:15:05.600 INFO Pop front: ReconcileIO { id: ReconciliationId(99), op: ExtentReopen { repair_id: ReconciliationId(99), extent_id: 68 }, state: ClientData([New, New, New]) }
Sep 22 23:15:05.600 INFO Sent repair work, now wait for resp
Sep 22 23:15:05.600 INFO [0] received reconcile message
Sep 22 23:15:05.600 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(99), op: ExtentReopen { repair_id: ReconciliationId(99), extent_id: 68 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:05.600 INFO [0] client ExtentReopen { repair_id: ReconciliationId(99), extent_id: 68 }
Sep 22 23:15:05.600 INFO [1] received reconcile message
Sep 22 23:15:05.600 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(99), op: ExtentReopen { repair_id: ReconciliationId(99), extent_id: 68 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:05.600 INFO [1] client ExtentReopen { repair_id: ReconciliationId(99), extent_id: 68 }
Sep 22 23:15:05.600 INFO [2] received reconcile message
Sep 22 23:15:05.600 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(99), op: ExtentReopen { repair_id: ReconciliationId(99), extent_id: 68 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:05.600 INFO [2] client ExtentReopen { repair_id: ReconciliationId(99), extent_id: 68 }
Sep 22 23:15:05.600 DEBG 99 Reopen extent 68
Sep 22 23:15:05.601 DEBG 99 Reopen extent 68
Sep 22 23:15:05.602 DEBG 99 Reopen extent 68
Sep 22 23:15:05.602 DEBG [2] It's time to notify for 99
Sep 22 23:15:05.602 INFO Completion from [2] id:99 status:true
Sep 22 23:15:05.602 INFO [100/752] Repair commands completed
Sep 22 23:15:05.602 INFO Pop front: ReconcileIO { id: ReconciliationId(100), op: ExtentFlush { repair_id: ReconciliationId(100), extent_id: 35, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:05.602 INFO Sent repair work, now wait for resp
Sep 22 23:15:05.602 INFO [0] received reconcile message
Sep 22 23:15:05.602 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(100), op: ExtentFlush { repair_id: ReconciliationId(100), extent_id: 35, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:05.602 INFO [0] client ExtentFlush { repair_id: ReconciliationId(100), extent_id: 35, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:05.602 INFO [1] received reconcile message
Sep 22 23:15:05.602 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(100), op: ExtentFlush { repair_id: ReconciliationId(100), extent_id: 35, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:05.602 INFO [1] client ExtentFlush { repair_id: ReconciliationId(100), extent_id: 35, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:05.602 INFO [2] received reconcile message
Sep 22 23:15:05.602 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(100), op: ExtentFlush { repair_id: ReconciliationId(100), extent_id: 35, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:05.602 INFO [2] client ExtentFlush { repair_id: ReconciliationId(100), extent_id: 35, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:05.603 DEBG 100 Flush extent 35 with f:2 g:2
Sep 22 23:15:05.603 DEBG Flush just extent 35 with f:2 and g:2
Sep 22 23:15:05.603 DEBG [1] It's time to notify for 100
Sep 22 23:15:05.603 INFO Completion from [1] id:100 status:true
Sep 22 23:15:05.603 INFO [101/752] Repair commands completed
Sep 22 23:15:05.603 INFO Pop front: ReconcileIO { id: ReconciliationId(101), op: ExtentClose { repair_id: ReconciliationId(101), extent_id: 35 }, state: ClientData([New, New, New]) }
Sep 22 23:15:05.603 INFO Sent repair work, now wait for resp
Sep 22 23:15:05.603 INFO [0] received reconcile message
Sep 22 23:15:05.603 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(101), op: ExtentClose { repair_id: ReconciliationId(101), extent_id: 35 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:05.603 INFO [0] client ExtentClose { repair_id: ReconciliationId(101), extent_id: 35 }
Sep 22 23:15:05.603 INFO [1] received reconcile message
Sep 22 23:15:05.603 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(101), op: ExtentClose { repair_id: ReconciliationId(101), extent_id: 35 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:05.603 INFO [1] client ExtentClose { repair_id: ReconciliationId(101), extent_id: 35 }
Sep 22 23:15:05.603 INFO [2] received reconcile message
Sep 22 23:15:05.603 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(101), op: ExtentClose { repair_id: ReconciliationId(101), extent_id: 35 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:05.603 INFO [2] client ExtentClose { repair_id: ReconciliationId(101), extent_id: 35 }
Sep 22 23:15:05.603 DEBG 101 Close extent 35
Sep 22 23:15:05.603 DEBG 101 Close extent 35
Sep 22 23:15:05.604 DEBG 101 Close extent 35
Sep 22 23:15:05.604 DEBG [2] It's time to notify for 101
Sep 22 23:15:05.604 INFO Completion from [2] id:101 status:true
Sep 22 23:15:05.604 INFO [102/752] Repair commands completed
Sep 22 23:15:05.604 INFO Pop front: ReconcileIO { id: ReconciliationId(102), op: ExtentRepair { repair_id: ReconciliationId(102), extent_id: 35, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:05.604 INFO Sent repair work, now wait for resp
Sep 22 23:15:05.604 INFO [0] received reconcile message
Sep 22 23:15:05.604 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(102), op: ExtentRepair { repair_id: ReconciliationId(102), extent_id: 35, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:05.604 INFO [0] client ExtentRepair { repair_id: ReconciliationId(102), extent_id: 35, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:05.604 INFO [0] Sending repair request ReconciliationId(102)
Sep 22 23:15:05.604 INFO [1] received reconcile message
Sep 22 23:15:05.604 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(102), op: ExtentRepair { repair_id: ReconciliationId(102), extent_id: 35, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:05.604 INFO [1] client ExtentRepair { repair_id: ReconciliationId(102), extent_id: 35, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:05.604 INFO [1] No action required ReconciliationId(102)
Sep 22 23:15:05.604 INFO [2] received reconcile message
Sep 22 23:15:05.604 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(102), op: ExtentRepair { repair_id: ReconciliationId(102), extent_id: 35, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:05.604 INFO [2] client ExtentRepair { repair_id: ReconciliationId(102), extent_id: 35, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:05.604 INFO [2] No action required ReconciliationId(102)
Sep 22 23:15:05.605 DEBG 102 Repair extent 35 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:05.605 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/023.copy"
Sep 22 23:15:05.668 INFO accepted connection, remote_addr: 127.0.0.1:48531, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.668 TRCE incoming request, uri: /extent/35/files, method: GET, req_id: a6e931b8-27fe-4d9d-a6e7-bd37f9dc7efa, remote_addr: 127.0.0.1:48531, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.668 INFO request completed, latency_us: 205, response_code: 200, uri: /extent/35/files, method: GET, req_id: a6e931b8-27fe-4d9d-a6e7-bd37f9dc7efa, remote_addr: 127.0.0.1:48531, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.668 INFO eid:35 Found repair files: ["023", "023.db"]
Sep 22 23:15:05.669 TRCE incoming request, uri: /newextent/35/data, method: GET, req_id: aa1818d1-bf02-4504-bbf9-09f20a161f3e, remote_addr: 127.0.0.1:48531, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.669 INFO request completed, latency_us: 303, response_code: 200, uri: /newextent/35/data, method: GET, req_id: aa1818d1-bf02-4504-bbf9-09f20a161f3e, remote_addr: 127.0.0.1:48531, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.674 TRCE incoming request, uri: /newextent/35/db, method: GET, req_id: 2840f5de-888b-4726-889c-afdf8551fe85, remote_addr: 127.0.0.1:48531, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.674 INFO request completed, latency_us: 292, response_code: 200, uri: /newextent/35/db, method: GET, req_id: 2840f5de-888b-4726-889c-afdf8551fe85, remote_addr: 127.0.0.1:48531, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.675 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/023.copy" to "/tmp/downstairs-vrx8aK6L/00/000/023.replace"
Sep 22 23:15:05.675 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:05.676 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/023.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:05.676 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/023"
Sep 22 23:15:05.676 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/023.db"
Sep 22 23:15:05.677 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:05.677 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/023.replace" to "/tmp/downstairs-vrx8aK6L/00/000/023.completed"
Sep 22 23:15:05.677 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:05.677 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:05.677 DEBG [0] It's time to notify for 102
Sep 22 23:15:05.677 INFO Completion from [0] id:102 status:true
Sep 22 23:15:05.677 INFO [103/752] Repair commands completed
Sep 22 23:15:05.677 INFO Pop front: ReconcileIO { id: ReconciliationId(103), op: ExtentReopen { repair_id: ReconciliationId(103), extent_id: 35 }, state: ClientData([New, New, New]) }
Sep 22 23:15:05.677 INFO Sent repair work, now wait for resp
Sep 22 23:15:05.677 INFO [0] received reconcile message
Sep 22 23:15:05.677 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(103), op: ExtentReopen { repair_id: ReconciliationId(103), extent_id: 35 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:05.677 INFO [0] client ExtentReopen { repair_id: ReconciliationId(103), extent_id: 35 }
Sep 22 23:15:05.677 INFO [1] received reconcile message
Sep 22 23:15:05.677 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(103), op: ExtentReopen { repair_id: ReconciliationId(103), extent_id: 35 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:05.677 INFO [1] client ExtentReopen { repair_id: ReconciliationId(103), extent_id: 35 }
Sep 22 23:15:05.677 INFO [2] received reconcile message
Sep 22 23:15:05.677 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(103), op: ExtentReopen { repair_id: ReconciliationId(103), extent_id: 35 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:05.677 INFO [2] client ExtentReopen { repair_id: ReconciliationId(103), extent_id: 35 }
Sep 22 23:15:05.677 DEBG 103 Reopen extent 35
Sep 22 23:15:05.678 DEBG 103 Reopen extent 35
Sep 22 23:15:05.678 DEBG 103 Reopen extent 35
Sep 22 23:15:05.679 DEBG [2] It's time to notify for 103
Sep 22 23:15:05.679 INFO Completion from [2] id:103 status:true
Sep 22 23:15:05.679 INFO [104/752] Repair commands completed
Sep 22 23:15:05.679 INFO Pop front: ReconcileIO { id: ReconciliationId(104), op: ExtentFlush { repair_id: ReconciliationId(104), extent_id: 152, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:05.679 INFO Sent repair work, now wait for resp
Sep 22 23:15:05.679 INFO [0] received reconcile message
Sep 22 23:15:05.679 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(104), op: ExtentFlush { repair_id: ReconciliationId(104), extent_id: 152, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:05.679 INFO [0] client ExtentFlush { repair_id: ReconciliationId(104), extent_id: 152, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:05.679 INFO [1] received reconcile message
Sep 22 23:15:05.679 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(104), op: ExtentFlush { repair_id: ReconciliationId(104), extent_id: 152, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:05.679 INFO [1] client ExtentFlush { repair_id: ReconciliationId(104), extent_id: 152, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:05.679 INFO [2] received reconcile message
Sep 22 23:15:05.679 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(104), op: ExtentFlush { repair_id: ReconciliationId(104), extent_id: 152, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:05.679 INFO [2] client ExtentFlush { repair_id: ReconciliationId(104), extent_id: 152, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:05.680 DEBG 104 Flush extent 152 with f:2 g:2
Sep 22 23:15:05.680 DEBG Flush just extent 152 with f:2 and g:2
Sep 22 23:15:05.680 DEBG [1] It's time to notify for 104
Sep 22 23:15:05.680 INFO Completion from [1] id:104 status:true
Sep 22 23:15:05.680 INFO [105/752] Repair commands completed
Sep 22 23:15:05.680 INFO Pop front: ReconcileIO { id: ReconciliationId(105), op: ExtentClose { repair_id: ReconciliationId(105), extent_id: 152 }, state: ClientData([New, New, New]) }
Sep 22 23:15:05.680 INFO Sent repair work, now wait for resp
Sep 22 23:15:05.680 INFO [0] received reconcile message
Sep 22 23:15:05.680 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(105), op: ExtentClose { repair_id: ReconciliationId(105), extent_id: 152 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:05.680 INFO [0] client ExtentClose { repair_id: ReconciliationId(105), extent_id: 152 }
Sep 22 23:15:05.680 INFO [1] received reconcile message
Sep 22 23:15:05.680 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(105), op: ExtentClose { repair_id: ReconciliationId(105), extent_id: 152 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:05.680 INFO [1] client ExtentClose { repair_id: ReconciliationId(105), extent_id: 152 }
Sep 22 23:15:05.680 INFO [2] received reconcile message
Sep 22 23:15:05.680 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(105), op: ExtentClose { repair_id: ReconciliationId(105), extent_id: 152 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:05.680 INFO [2] client ExtentClose { repair_id: ReconciliationId(105), extent_id: 152 }
Sep 22 23:15:05.680 DEBG 105 Close extent 152
Sep 22 23:15:05.680 DEBG 105 Close extent 152
Sep 22 23:15:05.681 DEBG 105 Close extent 152
Sep 22 23:15:05.681 DEBG [2] It's time to notify for 105
Sep 22 23:15:05.681 INFO Completion from [2] id:105 status:true
Sep 22 23:15:05.681 INFO [106/752] Repair commands completed
Sep 22 23:15:05.681 INFO Pop front: ReconcileIO { id: ReconciliationId(106), op: ExtentRepair { repair_id: ReconciliationId(106), extent_id: 152, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:05.681 INFO Sent repair work, now wait for resp
Sep 22 23:15:05.681 INFO [0] received reconcile message
Sep 22 23:15:05.681 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(106), op: ExtentRepair { repair_id: ReconciliationId(106), extent_id: 152, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:05.681 INFO [0] client ExtentRepair { repair_id: ReconciliationId(106), extent_id: 152, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:05.681 INFO [0] Sending repair request ReconciliationId(106)
Sep 22 23:15:05.681 INFO [1] received reconcile message
Sep 22 23:15:05.681 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(106), op: ExtentRepair { repair_id: ReconciliationId(106), extent_id: 152, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:05.681 INFO [1] client ExtentRepair { repair_id: ReconciliationId(106), extent_id: 152, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:05.681 INFO [1] No action required ReconciliationId(106)
Sep 22 23:15:05.681 INFO [2] received reconcile message
Sep 22 23:15:05.681 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(106), op: ExtentRepair { repair_id: ReconciliationId(106), extent_id: 152, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:05.681 INFO [2] client ExtentRepair { repair_id: ReconciliationId(106), extent_id: 152, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:05.681 INFO [2] No action required ReconciliationId(106)
Sep 22 23:15:05.682 DEBG 106 Repair extent 152 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:05.682 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/098.copy"
Sep 22 23:15:05.746 INFO accepted connection, remote_addr: 127.0.0.1:51141, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.746 TRCE incoming request, uri: /extent/152/files, method: GET, req_id: 08a45306-4a04-415d-ad14-5524b5866519, remote_addr: 127.0.0.1:51141, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.746 INFO request completed, latency_us: 189, response_code: 200, uri: /extent/152/files, method: GET, req_id: 08a45306-4a04-415d-ad14-5524b5866519, remote_addr: 127.0.0.1:51141, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.746 INFO eid:152 Found repair files: ["098", "098.db"]
Sep 22 23:15:05.747 TRCE incoming request, uri: /newextent/152/data, method: GET, req_id: 29bc57a3-1e91-40d4-882a-ce664aa53141, remote_addr: 127.0.0.1:51141, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.747 INFO request completed, latency_us: 293, response_code: 200, uri: /newextent/152/data, method: GET, req_id: 29bc57a3-1e91-40d4-882a-ce664aa53141, remote_addr: 127.0.0.1:51141, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.752 TRCE incoming request, uri: /newextent/152/db, method: GET, req_id: 77c8cb2b-a51d-4554-93e7-15e83b6c7a57, remote_addr: 127.0.0.1:51141, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.752 INFO request completed, latency_us: 291, response_code: 200, uri: /newextent/152/db, method: GET, req_id: 77c8cb2b-a51d-4554-93e7-15e83b6c7a57, remote_addr: 127.0.0.1:51141, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.753 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/098.copy" to "/tmp/downstairs-vrx8aK6L/00/000/098.replace"
Sep 22 23:15:05.753 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:05.754 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/098.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:05.754 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/098"
Sep 22 23:15:05.754 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/098.db"
Sep 22 23:15:05.754 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:05.755 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/098.replace" to "/tmp/downstairs-vrx8aK6L/00/000/098.completed"
Sep 22 23:15:05.755 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:05.755 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:05.755 DEBG [0] It's time to notify for 106
Sep 22 23:15:05.755 INFO Completion from [0] id:106 status:true
Sep 22 23:15:05.755 INFO [107/752] Repair commands completed
Sep 22 23:15:05.755 INFO Pop front: ReconcileIO { id: ReconciliationId(107), op: ExtentReopen { repair_id: ReconciliationId(107), extent_id: 152 }, state: ClientData([New, New, New]) }
Sep 22 23:15:05.755 INFO Sent repair work, now wait for resp
Sep 22 23:15:05.755 INFO [0] received reconcile message
Sep 22 23:15:05.755 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(107), op: ExtentReopen { repair_id: ReconciliationId(107), extent_id: 152 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:05.755 INFO [0] client ExtentReopen { repair_id: ReconciliationId(107), extent_id: 152 }
Sep 22 23:15:05.755 INFO [1] received reconcile message
Sep 22 23:15:05.755 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(107), op: ExtentReopen { repair_id: ReconciliationId(107), extent_id: 152 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:05.755 INFO [1] client ExtentReopen { repair_id: ReconciliationId(107), extent_id: 152 }
Sep 22 23:15:05.755 INFO [2] received reconcile message
Sep 22 23:15:05.755 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(107), op: ExtentReopen { repair_id: ReconciliationId(107), extent_id: 152 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:05.755 INFO [2] client ExtentReopen { repair_id: ReconciliationId(107), extent_id: 152 }
Sep 22 23:15:05.755 DEBG 107 Reopen extent 152
Sep 22 23:15:05.756 DEBG 107 Reopen extent 152
Sep 22 23:15:05.756 DEBG 107 Reopen extent 152
Sep 22 23:15:05.757 DEBG [2] It's time to notify for 107
Sep 22 23:15:05.757 INFO Completion from [2] id:107 status:true
Sep 22 23:15:05.757 INFO [108/752] Repair commands completed
Sep 22 23:15:05.757 INFO Pop front: ReconcileIO { id: ReconciliationId(108), op: ExtentFlush { repair_id: ReconciliationId(108), extent_id: 30, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:05.757 INFO Sent repair work, now wait for resp
Sep 22 23:15:05.757 INFO [0] received reconcile message
Sep 22 23:15:05.757 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(108), op: ExtentFlush { repair_id: ReconciliationId(108), extent_id: 30, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:05.757 INFO [0] client ExtentFlush { repair_id: ReconciliationId(108), extent_id: 30, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:05.757 INFO [1] received reconcile message
Sep 22 23:15:05.757 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(108), op: ExtentFlush { repair_id: ReconciliationId(108), extent_id: 30, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:05.757 INFO [1] client ExtentFlush { repair_id: ReconciliationId(108), extent_id: 30, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:05.757 INFO [2] received reconcile message
Sep 22 23:15:05.757 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(108), op: ExtentFlush { repair_id: ReconciliationId(108), extent_id: 30, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:05.757 INFO [2] client ExtentFlush { repair_id: ReconciliationId(108), extent_id: 30, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:05.757 DEBG 108 Flush extent 30 with f:2 g:2
Sep 22 23:15:05.757 DEBG Flush just extent 30 with f:2 and g:2
Sep 22 23:15:05.758 DEBG [1] It's time to notify for 108
Sep 22 23:15:05.758 INFO Completion from [1] id:108 status:true
Sep 22 23:15:05.758 INFO [109/752] Repair commands completed
Sep 22 23:15:05.758 INFO Pop front: ReconcileIO { id: ReconciliationId(109), op: ExtentClose { repair_id: ReconciliationId(109), extent_id: 30 }, state: ClientData([New, New, New]) }
Sep 22 23:15:05.758 INFO Sent repair work, now wait for resp
Sep 22 23:15:05.758 INFO [0] received reconcile message
Sep 22 23:15:05.758 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(109), op: ExtentClose { repair_id: ReconciliationId(109), extent_id: 30 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:05.758 INFO [0] client ExtentClose { repair_id: ReconciliationId(109), extent_id: 30 }
Sep 22 23:15:05.758 INFO [1] received reconcile message
Sep 22 23:15:05.758 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(109), op: ExtentClose { repair_id: ReconciliationId(109), extent_id: 30 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:05.758 INFO [1] client ExtentClose { repair_id: ReconciliationId(109), extent_id: 30 }
Sep 22 23:15:05.758 INFO [2] received reconcile message
Sep 22 23:15:05.758 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(109), op: ExtentClose { repair_id: ReconciliationId(109), extent_id: 30 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:05.758 INFO [2] client ExtentClose { repair_id: ReconciliationId(109), extent_id: 30 }
Sep 22 23:15:05.758 DEBG 109 Close extent 30
Sep 22 23:15:05.758 DEBG 109 Close extent 30
Sep 22 23:15:05.759 DEBG 109 Close extent 30
Sep 22 23:15:05.759 DEBG [2] It's time to notify for 109
Sep 22 23:15:05.759 INFO Completion from [2] id:109 status:true
Sep 22 23:15:05.759 INFO [110/752] Repair commands completed
Sep 22 23:15:05.759 INFO Pop front: ReconcileIO { id: ReconciliationId(110), op: ExtentRepair { repair_id: ReconciliationId(110), extent_id: 30, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:05.759 INFO Sent repair work, now wait for resp
Sep 22 23:15:05.759 INFO [0] received reconcile message
Sep 22 23:15:05.759 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(110), op: ExtentRepair { repair_id: ReconciliationId(110), extent_id: 30, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:05.759 INFO [0] client ExtentRepair { repair_id: ReconciliationId(110), extent_id: 30, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:05.759 INFO [0] Sending repair request ReconciliationId(110)
Sep 22 23:15:05.759 INFO [1] received reconcile message
Sep 22 23:15:05.759 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(110), op: ExtentRepair { repair_id: ReconciliationId(110), extent_id: 30, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:05.759 INFO [1] client ExtentRepair { repair_id: ReconciliationId(110), extent_id: 30, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:05.759 INFO [1] No action required ReconciliationId(110)
Sep 22 23:15:05.759 INFO [2] received reconcile message
Sep 22 23:15:05.759 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(110), op: ExtentRepair { repair_id: ReconciliationId(110), extent_id: 30, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:05.759 INFO [2] client ExtentRepair { repair_id: ReconciliationId(110), extent_id: 30, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:05.759 INFO [2] No action required ReconciliationId(110)
Sep 22 23:15:05.759 DEBG 110 Repair extent 30 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:05.759 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/01E.copy"
Sep 22 23:15:05.822 INFO accepted connection, remote_addr: 127.0.0.1:58609, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.822 TRCE incoming request, uri: /extent/30/files, method: GET, req_id: ab773c06-00a1-4b82-8bb5-85f3f75d287b, remote_addr: 127.0.0.1:58609, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.822 INFO request completed, latency_us: 199, response_code: 200, uri: /extent/30/files, method: GET, req_id: ab773c06-00a1-4b82-8bb5-85f3f75d287b, remote_addr: 127.0.0.1:58609, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.822 INFO eid:30 Found repair files: ["01E", "01E.db"]
Sep 22 23:15:05.823 TRCE incoming request, uri: /newextent/30/data, method: GET, req_id: b70813dd-2a37-4774-b2cc-29667788bef5, remote_addr: 127.0.0.1:58609, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.823 INFO request completed, latency_us: 303, response_code: 200, uri: /newextent/30/data, method: GET, req_id: b70813dd-2a37-4774-b2cc-29667788bef5, remote_addr: 127.0.0.1:58609, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.828 TRCE incoming request, uri: /newextent/30/db, method: GET, req_id: 89d2e979-157c-46b8-bc4e-9de7c0777eea, remote_addr: 127.0.0.1:58609, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.828 INFO request completed, latency_us: 292, response_code: 200, uri: /newextent/30/db, method: GET, req_id: 89d2e979-157c-46b8-bc4e-9de7c0777eea, remote_addr: 127.0.0.1:58609, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.829 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/01E.copy" to "/tmp/downstairs-vrx8aK6L/00/000/01E.replace"
Sep 22 23:15:05.830 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:05.830 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/01E.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:05.831 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/01E"
Sep 22 23:15:05.831 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/01E.db"
Sep 22 23:15:05.831 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:05.831 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/01E.replace" to "/tmp/downstairs-vrx8aK6L/00/000/01E.completed"
Sep 22 23:15:05.831 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:05.831 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:05.831 DEBG [0] It's time to notify for 110
Sep 22 23:15:05.831 INFO Completion from [0] id:110 status:true
Sep 22 23:15:05.831 INFO [111/752] Repair commands completed
Sep 22 23:15:05.831 INFO Pop front: ReconcileIO { id: ReconciliationId(111), op: ExtentReopen { repair_id: ReconciliationId(111), extent_id: 30 }, state: ClientData([New, New, New]) }
Sep 22 23:15:05.831 INFO Sent repair work, now wait for resp
Sep 22 23:15:05.831 INFO [0] received reconcile message
Sep 22 23:15:05.831 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(111), op: ExtentReopen { repair_id: ReconciliationId(111), extent_id: 30 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:05.831 INFO [0] client ExtentReopen { repair_id: ReconciliationId(111), extent_id: 30 }
Sep 22 23:15:05.831 INFO [1] received reconcile message
Sep 22 23:15:05.831 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(111), op: ExtentReopen { repair_id: ReconciliationId(111), extent_id: 30 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:05.831 INFO [1] client ExtentReopen { repair_id: ReconciliationId(111), extent_id: 30 }
Sep 22 23:15:05.831 INFO [2] received reconcile message
Sep 22 23:15:05.831 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(111), op: ExtentReopen { repair_id: ReconciliationId(111), extent_id: 30 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:05.831 INFO [2] client ExtentReopen { repair_id: ReconciliationId(111), extent_id: 30 }
Sep 22 23:15:05.832 DEBG 111 Reopen extent 30
Sep 22 23:15:05.832 DEBG 111 Reopen extent 30
Sep 22 23:15:05.833 DEBG 111 Reopen extent 30
Sep 22 23:15:05.833 DEBG [2] It's time to notify for 111
Sep 22 23:15:05.833 INFO Completion from [2] id:111 status:true
Sep 22 23:15:05.833 INFO [112/752] Repair commands completed
Sep 22 23:15:05.833 INFO Pop front: ReconcileIO { id: ReconciliationId(112), op: ExtentFlush { repair_id: ReconciliationId(112), extent_id: 95, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:05.833 INFO Sent repair work, now wait for resp
Sep 22 23:15:05.833 INFO [0] received reconcile message
Sep 22 23:15:05.833 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(112), op: ExtentFlush { repair_id: ReconciliationId(112), extent_id: 95, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:05.833 INFO [0] client ExtentFlush { repair_id: ReconciliationId(112), extent_id: 95, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:05.833 INFO [1] received reconcile message
Sep 22 23:15:05.833 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(112), op: ExtentFlush { repair_id: ReconciliationId(112), extent_id: 95, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:05.833 INFO [1] client ExtentFlush { repair_id: ReconciliationId(112), extent_id: 95, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:05.834 INFO [2] received reconcile message
Sep 22 23:15:05.834 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(112), op: ExtentFlush { repair_id: ReconciliationId(112), extent_id: 95, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:05.834 INFO [2] client ExtentFlush { repair_id: ReconciliationId(112), extent_id: 95, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:05.834 DEBG 112 Flush extent 95 with f:2 g:2
Sep 22 23:15:05.834 DEBG Flush just extent 95 with f:2 and g:2
Sep 22 23:15:05.834 DEBG [1] It's time to notify for 112
Sep 22 23:15:05.834 INFO Completion from [1] id:112 status:true
Sep 22 23:15:05.834 INFO [113/752] Repair commands completed
Sep 22 23:15:05.834 INFO Pop front: ReconcileIO { id: ReconciliationId(113), op: ExtentClose { repair_id: ReconciliationId(113), extent_id: 95 }, state: ClientData([New, New, New]) }
Sep 22 23:15:05.834 INFO Sent repair work, now wait for resp
Sep 22 23:15:05.834 INFO [0] received reconcile message
Sep 22 23:15:05.834 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(113), op: ExtentClose { repair_id: ReconciliationId(113), extent_id: 95 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:05.834 INFO [0] client ExtentClose { repair_id: ReconciliationId(113), extent_id: 95 }
Sep 22 23:15:05.834 INFO [1] received reconcile message
Sep 22 23:15:05.834 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(113), op: ExtentClose { repair_id: ReconciliationId(113), extent_id: 95 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:05.834 INFO [1] client ExtentClose { repair_id: ReconciliationId(113), extent_id: 95 }
Sep 22 23:15:05.834 INFO [2] received reconcile message
Sep 22 23:15:05.834 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(113), op: ExtentClose { repair_id: ReconciliationId(113), extent_id: 95 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:05.834 INFO [2] client ExtentClose { repair_id: ReconciliationId(113), extent_id: 95 }
Sep 22 23:15:05.834 DEBG 113 Close extent 95
Sep 22 23:15:05.834 DEBG 113 Close extent 95
Sep 22 23:15:05.835 DEBG 113 Close extent 95
Sep 22 23:15:05.835 DEBG [2] It's time to notify for 113
Sep 22 23:15:05.835 INFO Completion from [2] id:113 status:true
Sep 22 23:15:05.835 INFO [114/752] Repair commands completed
Sep 22 23:15:05.835 INFO Pop front: ReconcileIO { id: ReconciliationId(114), op: ExtentRepair { repair_id: ReconciliationId(114), extent_id: 95, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:05.835 INFO Sent repair work, now wait for resp
Sep 22 23:15:05.835 INFO [0] received reconcile message
Sep 22 23:15:05.835 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(114), op: ExtentRepair { repair_id: ReconciliationId(114), extent_id: 95, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:05.835 INFO [0] client ExtentRepair { repair_id: ReconciliationId(114), extent_id: 95, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:05.835 INFO [0] Sending repair request ReconciliationId(114)
Sep 22 23:15:05.835 INFO [1] received reconcile message
Sep 22 23:15:05.835 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(114), op: ExtentRepair { repair_id: ReconciliationId(114), extent_id: 95, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:05.835 INFO [1] client ExtentRepair { repair_id: ReconciliationId(114), extent_id: 95, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:05.835 INFO [1] No action required ReconciliationId(114)
Sep 22 23:15:05.836 INFO [2] received reconcile message
Sep 22 23:15:05.836 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(114), op: ExtentRepair { repair_id: ReconciliationId(114), extent_id: 95, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:05.836 INFO [2] client ExtentRepair { repair_id: ReconciliationId(114), extent_id: 95, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:05.836 INFO [2] No action required ReconciliationId(114)
Sep 22 23:15:05.836 DEBG 114 Repair extent 95 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:05.836 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/05F.copy"
Sep 22 23:15:05.882 ERRO [2] job id 1076 saw error GenericError("test error")
Sep 22 23:15:05.897 INFO accepted connection, remote_addr: 127.0.0.1:60926, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.897 TRCE incoming request, uri: /extent/95/files, method: GET, req_id: 43f0faaf-fb49-47c9-9e68-feb992573ed0, remote_addr: 127.0.0.1:60926, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.898 INFO request completed, latency_us: 191, response_code: 200, uri: /extent/95/files, method: GET, req_id: 43f0faaf-fb49-47c9-9e68-feb992573ed0, remote_addr: 127.0.0.1:60926, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.898 INFO eid:95 Found repair files: ["05F", "05F.db"]
Sep 22 23:15:05.898 TRCE incoming request, uri: /newextent/95/data, method: GET, req_id: b73a1ad7-2e44-4f57-865b-7422ba56062c, remote_addr: 127.0.0.1:60926, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.898 INFO request completed, latency_us: 250, response_code: 200, uri: /newextent/95/data, method: GET, req_id: b73a1ad7-2e44-4f57-865b-7422ba56062c, remote_addr: 127.0.0.1:60926, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.903 TRCE incoming request, uri: /newextent/95/db, method: GET, req_id: 5e8503d4-e230-4e25-a949-0586fdc821a0, remote_addr: 127.0.0.1:60926, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.904 INFO request completed, latency_us: 314, response_code: 200, uri: /newextent/95/db, method: GET, req_id: 5e8503d4-e230-4e25-a949-0586fdc821a0, remote_addr: 127.0.0.1:60926, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.905 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/05F.copy" to "/tmp/downstairs-vrx8aK6L/00/000/05F.replace"
Sep 22 23:15:05.905 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:05.906 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/05F.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:05.906 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/05F"
Sep 22 23:15:05.906 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/05F.db"
Sep 22 23:15:05.906 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:05.906 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/05F.replace" to "/tmp/downstairs-vrx8aK6L/00/000/05F.completed"
Sep 22 23:15:05.906 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:05.906 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:05.906 DEBG [0] It's time to notify for 114
Sep 22 23:15:05.906 INFO Completion from [0] id:114 status:true
Sep 22 23:15:05.906 INFO [115/752] Repair commands completed
Sep 22 23:15:05.906 INFO Pop front: ReconcileIO { id: ReconciliationId(115), op: ExtentReopen { repair_id: ReconciliationId(115), extent_id: 95 }, state: ClientData([New, New, New]) }
Sep 22 23:15:05.906 INFO Sent repair work, now wait for resp
Sep 22 23:15:05.906 INFO [0] received reconcile message
Sep 22 23:15:05.906 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(115), op: ExtentReopen { repair_id: ReconciliationId(115), extent_id: 95 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:05.906 INFO [0] client ExtentReopen { repair_id: ReconciliationId(115), extent_id: 95 }
Sep 22 23:15:05.907 INFO [1] received reconcile message
Sep 22 23:15:05.907 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(115), op: ExtentReopen { repair_id: ReconciliationId(115), extent_id: 95 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:05.907 INFO [1] client ExtentReopen { repair_id: ReconciliationId(115), extent_id: 95 }
Sep 22 23:15:05.907 INFO [2] received reconcile message
Sep 22 23:15:05.907 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(115), op: ExtentReopen { repair_id: ReconciliationId(115), extent_id: 95 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:05.907 INFO [2] client ExtentReopen { repair_id: ReconciliationId(115), extent_id: 95 }
Sep 22 23:15:05.907 DEBG 115 Reopen extent 95
Sep 22 23:15:05.907 DEBG 115 Reopen extent 95
Sep 22 23:15:05.908 DEBG 115 Reopen extent 95
Sep 22 23:15:05.908 DEBG [2] It's time to notify for 115
Sep 22 23:15:05.908 INFO Completion from [2] id:115 status:true
Sep 22 23:15:05.908 INFO [116/752] Repair commands completed
Sep 22 23:15:05.908 INFO Pop front: ReconcileIO { id: ReconciliationId(116), op: ExtentFlush { repair_id: ReconciliationId(116), extent_id: 133, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:05.909 INFO Sent repair work, now wait for resp
Sep 22 23:15:05.909 INFO [0] received reconcile message
Sep 22 23:15:05.909 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(116), op: ExtentFlush { repair_id: ReconciliationId(116), extent_id: 133, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:05.909 INFO [0] client ExtentFlush { repair_id: ReconciliationId(116), extent_id: 133, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:05.909 INFO [1] received reconcile message
Sep 22 23:15:05.909 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(116), op: ExtentFlush { repair_id: ReconciliationId(116), extent_id: 133, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:05.909 INFO [1] client ExtentFlush { repair_id: ReconciliationId(116), extent_id: 133, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:05.909 INFO [2] received reconcile message
Sep 22 23:15:05.909 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(116), op: ExtentFlush { repair_id: ReconciliationId(116), extent_id: 133, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:05.909 INFO [2] client ExtentFlush { repair_id: ReconciliationId(116), extent_id: 133, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:05.909 DEBG 116 Flush extent 133 with f:2 g:2
Sep 22 23:15:05.909 DEBG Flush just extent 133 with f:2 and g:2
Sep 22 23:15:05.909 DEBG [1] It's time to notify for 116
Sep 22 23:15:05.909 INFO Completion from [1] id:116 status:true
Sep 22 23:15:05.909 INFO [117/752] Repair commands completed
Sep 22 23:15:05.909 INFO Pop front: ReconcileIO { id: ReconciliationId(117), op: ExtentClose { repair_id: ReconciliationId(117), extent_id: 133 }, state: ClientData([New, New, New]) }
Sep 22 23:15:05.909 INFO Sent repair work, now wait for resp
Sep 22 23:15:05.909 INFO [0] received reconcile message
Sep 22 23:15:05.909 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(117), op: ExtentClose { repair_id: ReconciliationId(117), extent_id: 133 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:05.909 INFO [0] client ExtentClose { repair_id: ReconciliationId(117), extent_id: 133 }
Sep 22 23:15:05.909 INFO [1] received reconcile message
Sep 22 23:15:05.909 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(117), op: ExtentClose { repair_id: ReconciliationId(117), extent_id: 133 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:05.909 INFO [1] client ExtentClose { repair_id: ReconciliationId(117), extent_id: 133 }
Sep 22 23:15:05.909 INFO [2] received reconcile message
Sep 22 23:15:05.909 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(117), op: ExtentClose { repair_id: ReconciliationId(117), extent_id: 133 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:05.909 INFO [2] client ExtentClose { repair_id: ReconciliationId(117), extent_id: 133 }
Sep 22 23:15:05.909 DEBG 117 Close extent 133
Sep 22 23:15:05.910 DEBG 117 Close extent 133
Sep 22 23:15:05.910 DEBG 117 Close extent 133
Sep 22 23:15:05.910 DEBG [2] It's time to notify for 117
Sep 22 23:15:05.910 INFO Completion from [2] id:117 status:true
Sep 22 23:15:05.910 INFO [118/752] Repair commands completed
Sep 22 23:15:05.910 INFO Pop front: ReconcileIO { id: ReconciliationId(118), op: ExtentRepair { repair_id: ReconciliationId(118), extent_id: 133, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:05.910 INFO Sent repair work, now wait for resp
Sep 22 23:15:05.910 INFO [0] received reconcile message
Sep 22 23:15:05.911 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(118), op: ExtentRepair { repair_id: ReconciliationId(118), extent_id: 133, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:05.911 INFO [0] client ExtentRepair { repair_id: ReconciliationId(118), extent_id: 133, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:05.911 INFO [0] Sending repair request ReconciliationId(118)
Sep 22 23:15:05.911 INFO [1] received reconcile message
Sep 22 23:15:05.911 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(118), op: ExtentRepair { repair_id: ReconciliationId(118), extent_id: 133, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:05.911 INFO [1] client ExtentRepair { repair_id: ReconciliationId(118), extent_id: 133, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:05.911 INFO [1] No action required ReconciliationId(118)
Sep 22 23:15:05.911 INFO [2] received reconcile message
Sep 22 23:15:05.911 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(118), op: ExtentRepair { repair_id: ReconciliationId(118), extent_id: 133, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:05.911 INFO [2] client ExtentRepair { repair_id: ReconciliationId(118), extent_id: 133, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:05.911 INFO [2] No action required ReconciliationId(118)
Sep 22 23:15:05.911 DEBG 118 Repair extent 133 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:05.911 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/085.copy"
Sep 22 23:15:05.974 INFO accepted connection, remote_addr: 127.0.0.1:58418, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.974 TRCE incoming request, uri: /extent/133/files, method: GET, req_id: 0a5eb246-43d6-4bc7-bed1-d611e74eb8b4, remote_addr: 127.0.0.1:58418, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.974 INFO request completed, latency_us: 190, response_code: 200, uri: /extent/133/files, method: GET, req_id: 0a5eb246-43d6-4bc7-bed1-d611e74eb8b4, remote_addr: 127.0.0.1:58418, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.974 INFO eid:133 Found repair files: ["085", "085.db"]
Sep 22 23:15:05.975 TRCE incoming request, uri: /newextent/133/data, method: GET, req_id: f9397405-79ae-42b0-8746-19d45fa1caf1, remote_addr: 127.0.0.1:58418, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.975 INFO request completed, latency_us: 295, response_code: 200, uri: /newextent/133/data, method: GET, req_id: f9397405-79ae-42b0-8746-19d45fa1caf1, remote_addr: 127.0.0.1:58418, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.980 TRCE incoming request, uri: /newextent/133/db, method: GET, req_id: 1b96b039-b3b0-470b-999d-f1a5f633f865, remote_addr: 127.0.0.1:58418, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.980 INFO request completed, latency_us: 291, response_code: 200, uri: /newextent/133/db, method: GET, req_id: 1b96b039-b3b0-470b-999d-f1a5f633f865, remote_addr: 127.0.0.1:58418, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:05.981 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/085.copy" to "/tmp/downstairs-vrx8aK6L/00/000/085.replace"
Sep 22 23:15:05.981 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:05.982 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/085.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:05.982 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/085"
Sep 22 23:15:05.982 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/085.db"
Sep 22 23:15:05.982 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:05.982 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/085.replace" to "/tmp/downstairs-vrx8aK6L/00/000/085.completed"
Sep 22 23:15:05.982 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:05.983 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:05.983 DEBG [0] It's time to notify for 118
Sep 22 23:15:05.983 INFO Completion from [0] id:118 status:true
Sep 22 23:15:05.983 INFO [119/752] Repair commands completed
Sep 22 23:15:05.983 INFO Pop front: ReconcileIO { id: ReconciliationId(119), op: ExtentReopen { repair_id: ReconciliationId(119), extent_id: 133 }, state: ClientData([New, New, New]) }
Sep 22 23:15:05.983 INFO Sent repair work, now wait for resp
Sep 22 23:15:05.983 INFO [0] received reconcile message
Sep 22 23:15:05.983 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(119), op: ExtentReopen { repair_id: ReconciliationId(119), extent_id: 133 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:05.983 INFO [0] client ExtentReopen { repair_id: ReconciliationId(119), extent_id: 133 }
Sep 22 23:15:05.983 INFO [1] received reconcile message
Sep 22 23:15:05.983 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(119), op: ExtentReopen { repair_id: ReconciliationId(119), extent_id: 133 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:05.983 INFO [1] client ExtentReopen { repair_id: ReconciliationId(119), extent_id: 133 }
Sep 22 23:15:05.983 INFO [2] received reconcile message
Sep 22 23:15:05.983 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(119), op: ExtentReopen { repair_id: ReconciliationId(119), extent_id: 133 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:05.983 INFO [2] client ExtentReopen { repair_id: ReconciliationId(119), extent_id: 133 }
Sep 22 23:15:05.983 DEBG 119 Reopen extent 133
Sep 22 23:15:05.984 DEBG 119 Reopen extent 133
Sep 22 23:15:05.984 DEBG 119 Reopen extent 133
Sep 22 23:15:05.985 DEBG [2] It's time to notify for 119
Sep 22 23:15:05.985 INFO Completion from [2] id:119 status:true
Sep 22 23:15:05.985 INFO [120/752] Repair commands completed
Sep 22 23:15:05.985 INFO Pop front: ReconcileIO { id: ReconciliationId(120), op: ExtentFlush { repair_id: ReconciliationId(120), extent_id: 174, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:05.985 INFO Sent repair work, now wait for resp
Sep 22 23:15:05.985 INFO [0] received reconcile message
Sep 22 23:15:05.985 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(120), op: ExtentFlush { repair_id: ReconciliationId(120), extent_id: 174, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:05.985 INFO [0] client ExtentFlush { repair_id: ReconciliationId(120), extent_id: 174, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:05.985 INFO [1] received reconcile message
Sep 22 23:15:05.985 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(120), op: ExtentFlush { repair_id: ReconciliationId(120), extent_id: 174, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:05.985 INFO [1] client ExtentFlush { repair_id: ReconciliationId(120), extent_id: 174, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:05.985 INFO [2] received reconcile message
Sep 22 23:15:05.985 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(120), op: ExtentFlush { repair_id: ReconciliationId(120), extent_id: 174, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:05.985 INFO [2] client ExtentFlush { repair_id: ReconciliationId(120), extent_id: 174, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:05.985 DEBG 120 Flush extent 174 with f:2 g:2
Sep 22 23:15:05.985 DEBG Flush just extent 174 with f:2 and g:2
Sep 22 23:15:05.985 DEBG [1] It's time to notify for 120
Sep 22 23:15:05.985 INFO Completion from [1] id:120 status:true
Sep 22 23:15:05.985 INFO [121/752] Repair commands completed
Sep 22 23:15:05.986 INFO Pop front: ReconcileIO { id: ReconciliationId(121), op: ExtentClose { repair_id: ReconciliationId(121), extent_id: 174 }, state: ClientData([New, New, New]) }
Sep 22 23:15:05.986 INFO Sent repair work, now wait for resp
Sep 22 23:15:05.986 INFO [0] received reconcile message
Sep 22 23:15:05.986 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(121), op: ExtentClose { repair_id: ReconciliationId(121), extent_id: 174 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:05.986 INFO [0] client ExtentClose { repair_id: ReconciliationId(121), extent_id: 174 }
Sep 22 23:15:05.986 INFO [1] received reconcile message
Sep 22 23:15:05.986 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(121), op: ExtentClose { repair_id: ReconciliationId(121), extent_id: 174 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:05.986 INFO [1] client ExtentClose { repair_id: ReconciliationId(121), extent_id: 174 }
Sep 22 23:15:05.986 INFO [2] received reconcile message
Sep 22 23:15:05.986 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(121), op: ExtentClose { repair_id: ReconciliationId(121), extent_id: 174 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:05.986 INFO [2] client ExtentClose { repair_id: ReconciliationId(121), extent_id: 174 }
Sep 22 23:15:05.986 DEBG 121 Close extent 174
Sep 22 23:15:05.986 DEBG 121 Close extent 174
Sep 22 23:15:05.986 DEBG 121 Close extent 174
Sep 22 23:15:05.987 DEBG [2] It's time to notify for 121
Sep 22 23:15:05.987 INFO Completion from [2] id:121 status:true
Sep 22 23:15:05.987 INFO [122/752] Repair commands completed
Sep 22 23:15:05.987 INFO Pop front: ReconcileIO { id: ReconciliationId(122), op: ExtentRepair { repair_id: ReconciliationId(122), extent_id: 174, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:05.987 INFO Sent repair work, now wait for resp
Sep 22 23:15:05.987 INFO [0] received reconcile message
Sep 22 23:15:05.987 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(122), op: ExtentRepair { repair_id: ReconciliationId(122), extent_id: 174, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:05.987 INFO [0] client ExtentRepair { repair_id: ReconciliationId(122), extent_id: 174, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:05.987 INFO [0] Sending repair request ReconciliationId(122)
Sep 22 23:15:05.987 INFO [1] received reconcile message
Sep 22 23:15:05.987 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(122), op: ExtentRepair { repair_id: ReconciliationId(122), extent_id: 174, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:05.987 INFO [1] client ExtentRepair { repair_id: ReconciliationId(122), extent_id: 174, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:05.987 INFO [1] No action required ReconciliationId(122)
Sep 22 23:15:05.987 INFO [2] received reconcile message
Sep 22 23:15:05.987 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(122), op: ExtentRepair { repair_id: ReconciliationId(122), extent_id: 174, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:05.987 INFO [2] client ExtentRepair { repair_id: ReconciliationId(122), extent_id: 174, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:05.987 INFO [2] No action required ReconciliationId(122)
Sep 22 23:15:05.987 DEBG 122 Repair extent 174 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:05.987 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/0AE.copy"
Sep 22 23:15:06.052 INFO accepted connection, remote_addr: 127.0.0.1:54826, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.053 TRCE incoming request, uri: /extent/174/files, method: GET, req_id: 6e6a229c-fab3-40f0-91f7-1ab317ea06e7, remote_addr: 127.0.0.1:54826, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.053 INFO request completed, latency_us: 242, response_code: 200, uri: /extent/174/files, method: GET, req_id: 6e6a229c-fab3-40f0-91f7-1ab317ea06e7, remote_addr: 127.0.0.1:54826, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.053 INFO eid:174 Found repair files: ["0AE", "0AE.db"]
Sep 22 23:15:06.054 TRCE incoming request, uri: /newextent/174/data, method: GET, req_id: 580c52a5-d683-4da7-bb69-9e5ea0287288, remote_addr: 127.0.0.1:54826, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.054 INFO request completed, latency_us: 328, response_code: 200, uri: /newextent/174/data, method: GET, req_id: 580c52a5-d683-4da7-bb69-9e5ea0287288, remote_addr: 127.0.0.1:54826, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.059 TRCE incoming request, uri: /newextent/174/db, method: GET, req_id: 558bc2b1-2afa-4a95-8f69-556a9d2929c4, remote_addr: 127.0.0.1:54826, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.059 INFO request completed, latency_us: 292, response_code: 200, uri: /newextent/174/db, method: GET, req_id: 558bc2b1-2afa-4a95-8f69-556a9d2929c4, remote_addr: 127.0.0.1:54826, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.061 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/0AE.copy" to "/tmp/downstairs-vrx8aK6L/00/000/0AE.replace"
Sep 22 23:15:06.061 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:06.061 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/0AE.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:06.062 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/0AE"
Sep 22 23:15:06.062 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/0AE.db"
Sep 22 23:15:06.062 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:06.062 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/0AE.replace" to "/tmp/downstairs-vrx8aK6L/00/000/0AE.completed"
Sep 22 23:15:06.062 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:06.062 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:06.062 DEBG [0] It's time to notify for 122
Sep 22 23:15:06.062 INFO Completion from [0] id:122 status:true
Sep 22 23:15:06.062 INFO [123/752] Repair commands completed
Sep 22 23:15:06.062 INFO Pop front: ReconcileIO { id: ReconciliationId(123), op: ExtentReopen { repair_id: ReconciliationId(123), extent_id: 174 }, state: ClientData([New, New, New]) }
Sep 22 23:15:06.062 INFO Sent repair work, now wait for resp
Sep 22 23:15:06.062 INFO [0] received reconcile message
Sep 22 23:15:06.062 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(123), op: ExtentReopen { repair_id: ReconciliationId(123), extent_id: 174 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:06.062 INFO [0] client ExtentReopen { repair_id: ReconciliationId(123), extent_id: 174 }
Sep 22 23:15:06.063 INFO [1] received reconcile message
Sep 22 23:15:06.063 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(123), op: ExtentReopen { repair_id: ReconciliationId(123), extent_id: 174 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:06.063 INFO [1] client ExtentReopen { repair_id: ReconciliationId(123), extent_id: 174 }
Sep 22 23:15:06.063 INFO [2] received reconcile message
Sep 22 23:15:06.063 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(123), op: ExtentReopen { repair_id: ReconciliationId(123), extent_id: 174 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:06.063 INFO [2] client ExtentReopen { repair_id: ReconciliationId(123), extent_id: 174 }
Sep 22 23:15:06.063 DEBG 123 Reopen extent 174
Sep 22 23:15:06.064 DEBG 123 Reopen extent 174
Sep 22 23:15:06.064 DEBG 123 Reopen extent 174
Sep 22 23:15:06.065 DEBG [2] It's time to notify for 123
Sep 22 23:15:06.065 INFO Completion from [2] id:123 status:true
Sep 22 23:15:06.065 INFO [124/752] Repair commands completed
Sep 22 23:15:06.065 INFO Pop front: ReconcileIO { id: ReconciliationId(124), op: ExtentFlush { repair_id: ReconciliationId(124), extent_id: 158, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:06.065 INFO Sent repair work, now wait for resp
Sep 22 23:15:06.065 INFO [0] received reconcile message
Sep 22 23:15:06.065 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(124), op: ExtentFlush { repair_id: ReconciliationId(124), extent_id: 158, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:06.065 INFO [0] client ExtentFlush { repair_id: ReconciliationId(124), extent_id: 158, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:06.065 INFO [1] received reconcile message
Sep 22 23:15:06.065 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(124), op: ExtentFlush { repair_id: ReconciliationId(124), extent_id: 158, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:06.065 INFO [1] client ExtentFlush { repair_id: ReconciliationId(124), extent_id: 158, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:06.065 INFO [2] received reconcile message
Sep 22 23:15:06.065 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(124), op: ExtentFlush { repair_id: ReconciliationId(124), extent_id: 158, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:06.065 INFO [2] client ExtentFlush { repair_id: ReconciliationId(124), extent_id: 158, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:06.065 DEBG 124 Flush extent 158 with f:2 g:2
Sep 22 23:15:06.065 DEBG Flush just extent 158 with f:2 and g:2
Sep 22 23:15:06.065 DEBG [1] It's time to notify for 124
Sep 22 23:15:06.065 INFO Completion from [1] id:124 status:true
Sep 22 23:15:06.065 INFO [125/752] Repair commands completed
Sep 22 23:15:06.065 INFO Pop front: ReconcileIO { id: ReconciliationId(125), op: ExtentClose { repair_id: ReconciliationId(125), extent_id: 158 }, state: ClientData([New, New, New]) }
Sep 22 23:15:06.065 INFO Sent repair work, now wait for resp
Sep 22 23:15:06.065 INFO [0] received reconcile message
Sep 22 23:15:06.065 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(125), op: ExtentClose { repair_id: ReconciliationId(125), extent_id: 158 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:06.065 INFO [0] client ExtentClose { repair_id: ReconciliationId(125), extent_id: 158 }
Sep 22 23:15:06.065 INFO [1] received reconcile message
Sep 22 23:15:06.065 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(125), op: ExtentClose { repair_id: ReconciliationId(125), extent_id: 158 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:06.065 INFO [1] client ExtentClose { repair_id: ReconciliationId(125), extent_id: 158 }
Sep 22 23:15:06.065 INFO [2] received reconcile message
Sep 22 23:15:06.065 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(125), op: ExtentClose { repair_id: ReconciliationId(125), extent_id: 158 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:06.065 INFO [2] client ExtentClose { repair_id: ReconciliationId(125), extent_id: 158 }
Sep 22 23:15:06.066 DEBG 125 Close extent 158
Sep 22 23:15:06.066 DEBG 125 Close extent 158
Sep 22 23:15:06.066 DEBG 125 Close extent 158
Sep 22 23:15:06.067 DEBG [2] It's time to notify for 125
Sep 22 23:15:06.067 INFO Completion from [2] id:125 status:true
Sep 22 23:15:06.067 INFO [126/752] Repair commands completed
Sep 22 23:15:06.067 INFO Pop front: ReconcileIO { id: ReconciliationId(126), op: ExtentRepair { repair_id: ReconciliationId(126), extent_id: 158, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:06.067 INFO Sent repair work, now wait for resp
Sep 22 23:15:06.067 INFO [0] received reconcile message
Sep 22 23:15:06.067 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(126), op: ExtentRepair { repair_id: ReconciliationId(126), extent_id: 158, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:06.067 INFO [0] client ExtentRepair { repair_id: ReconciliationId(126), extent_id: 158, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:06.067 INFO [0] Sending repair request ReconciliationId(126)
Sep 22 23:15:06.067 INFO [1] received reconcile message
Sep 22 23:15:06.067 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(126), op: ExtentRepair { repair_id: ReconciliationId(126), extent_id: 158, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:06.067 INFO [1] client ExtentRepair { repair_id: ReconciliationId(126), extent_id: 158, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:06.067 INFO [1] No action required ReconciliationId(126)
Sep 22 23:15:06.067 INFO [2] received reconcile message
Sep 22 23:15:06.067 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(126), op: ExtentRepair { repair_id: ReconciliationId(126), extent_id: 158, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:06.067 INFO [2] client ExtentRepair { repair_id: ReconciliationId(126), extent_id: 158, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:06.067 INFO [2] No action required ReconciliationId(126)
Sep 22 23:15:06.067 DEBG 126 Repair extent 158 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:06.067 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/09E.copy"
Sep 22 23:15:06.130 INFO accepted connection, remote_addr: 127.0.0.1:43707, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.131 TRCE incoming request, uri: /extent/158/files, method: GET, req_id: 07473e78-377a-4306-8408-2a80e40ed040, remote_addr: 127.0.0.1:43707, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.131 INFO request completed, latency_us: 196, response_code: 200, uri: /extent/158/files, method: GET, req_id: 07473e78-377a-4306-8408-2a80e40ed040, remote_addr: 127.0.0.1:43707, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.131 INFO eid:158 Found repair files: ["09E", "09E.db"]
Sep 22 23:15:06.131 TRCE incoming request, uri: /newextent/158/data, method: GET, req_id: 8083baac-42c2-4a1a-8ac5-b5458e5d21d4, remote_addr: 127.0.0.1:43707, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.132 INFO request completed, latency_us: 318, response_code: 200, uri: /newextent/158/data, method: GET, req_id: 8083baac-42c2-4a1a-8ac5-b5458e5d21d4, remote_addr: 127.0.0.1:43707, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.137 TRCE incoming request, uri: /newextent/158/db, method: GET, req_id: 387bcfc2-7c61-4172-86b9-b2f06a5801ab, remote_addr: 127.0.0.1:43707, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.137 INFO request completed, latency_us: 292, response_code: 200, uri: /newextent/158/db, method: GET, req_id: 387bcfc2-7c61-4172-86b9-b2f06a5801ab, remote_addr: 127.0.0.1:43707, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.138 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/09E.copy" to "/tmp/downstairs-vrx8aK6L/00/000/09E.replace"
Sep 22 23:15:06.138 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:06.139 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/09E.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:06.139 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/09E"
Sep 22 23:15:06.139 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/09E.db"
Sep 22 23:15:06.139 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:06.139 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/09E.replace" to "/tmp/downstairs-vrx8aK6L/00/000/09E.completed"
Sep 22 23:15:06.139 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:06.140 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:06.140 DEBG [0] It's time to notify for 126
Sep 22 23:15:06.140 INFO Completion from [0] id:126 status:true
Sep 22 23:15:06.140 INFO [127/752] Repair commands completed
Sep 22 23:15:06.140 INFO Pop front: ReconcileIO { id: ReconciliationId(127), op: ExtentReopen { repair_id: ReconciliationId(127), extent_id: 158 }, state: ClientData([New, New, New]) }
Sep 22 23:15:06.140 INFO Sent repair work, now wait for resp
Sep 22 23:15:06.140 INFO [0] received reconcile message
Sep 22 23:15:06.140 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(127), op: ExtentReopen { repair_id: ReconciliationId(127), extent_id: 158 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:06.140 INFO [0] client ExtentReopen { repair_id: ReconciliationId(127), extent_id: 158 }
Sep 22 23:15:06.140 INFO [1] received reconcile message
Sep 22 23:15:06.140 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(127), op: ExtentReopen { repair_id: ReconciliationId(127), extent_id: 158 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:06.140 INFO [1] client ExtentReopen { repair_id: ReconciliationId(127), extent_id: 158 }
Sep 22 23:15:06.140 INFO [2] received reconcile message
Sep 22 23:15:06.140 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(127), op: ExtentReopen { repair_id: ReconciliationId(127), extent_id: 158 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:06.140 INFO [2] client ExtentReopen { repair_id: ReconciliationId(127), extent_id: 158 }
Sep 22 23:15:06.140 DEBG 127 Reopen extent 158
Sep 22 23:15:06.141 DEBG 127 Reopen extent 158
Sep 22 23:15:06.141 DEBG 127 Reopen extent 158
Sep 22 23:15:06.142 DEBG [2] It's time to notify for 127
Sep 22 23:15:06.142 INFO Completion from [2] id:127 status:true
Sep 22 23:15:06.142 INFO [128/752] Repair commands completed
Sep 22 23:15:06.142 INFO Pop front: ReconcileIO { id: ReconciliationId(128), op: ExtentFlush { repair_id: ReconciliationId(128), extent_id: 106, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:06.142 INFO Sent repair work, now wait for resp
Sep 22 23:15:06.142 INFO [0] received reconcile message
Sep 22 23:15:06.142 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(128), op: ExtentFlush { repair_id: ReconciliationId(128), extent_id: 106, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:06.142 INFO [0] client ExtentFlush { repair_id: ReconciliationId(128), extent_id: 106, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:06.142 INFO [1] received reconcile message
Sep 22 23:15:06.142 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(128), op: ExtentFlush { repair_id: ReconciliationId(128), extent_id: 106, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:06.142 INFO [1] client ExtentFlush { repair_id: ReconciliationId(128), extent_id: 106, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:06.142 INFO [2] received reconcile message
Sep 22 23:15:06.142 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(128), op: ExtentFlush { repair_id: ReconciliationId(128), extent_id: 106, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:06.142 INFO [2] client ExtentFlush { repair_id: ReconciliationId(128), extent_id: 106, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:06.142 DEBG 128 Flush extent 106 with f:2 g:2
Sep 22 23:15:06.142 DEBG Flush just extent 106 with f:2 and g:2
Sep 22 23:15:06.143 DEBG [1] It's time to notify for 128
Sep 22 23:15:06.143 INFO Completion from [1] id:128 status:true
Sep 22 23:15:06.143 INFO [129/752] Repair commands completed
Sep 22 23:15:06.143 INFO Pop front: ReconcileIO { id: ReconciliationId(129), op: ExtentClose { repair_id: ReconciliationId(129), extent_id: 106 }, state: ClientData([New, New, New]) }
Sep 22 23:15:06.143 INFO Sent repair work, now wait for resp
Sep 22 23:15:06.143 INFO [0] received reconcile message
Sep 22 23:15:06.143 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(129), op: ExtentClose { repair_id: ReconciliationId(129), extent_id: 106 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:06.143 INFO [0] client ExtentClose { repair_id: ReconciliationId(129), extent_id: 106 }
Sep 22 23:15:06.143 INFO [1] received reconcile message
Sep 22 23:15:06.143 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(129), op: ExtentClose { repair_id: ReconciliationId(129), extent_id: 106 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:06.143 INFO [1] client ExtentClose { repair_id: ReconciliationId(129), extent_id: 106 }
Sep 22 23:15:06.143 INFO [2] received reconcile message
Sep 22 23:15:06.143 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(129), op: ExtentClose { repair_id: ReconciliationId(129), extent_id: 106 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:06.143 INFO [2] client ExtentClose { repair_id: ReconciliationId(129), extent_id: 106 }
Sep 22 23:15:06.143 DEBG 129 Close extent 106
Sep 22 23:15:06.143 DEBG 129 Close extent 106
Sep 22 23:15:06.144 DEBG 129 Close extent 106
Sep 22 23:15:06.144 DEBG [2] It's time to notify for 129
Sep 22 23:15:06.144 INFO Completion from [2] id:129 status:true
Sep 22 23:15:06.144 INFO [130/752] Repair commands completed
Sep 22 23:15:06.144 INFO Pop front: ReconcileIO { id: ReconciliationId(130), op: ExtentRepair { repair_id: ReconciliationId(130), extent_id: 106, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:06.144 INFO Sent repair work, now wait for resp
Sep 22 23:15:06.144 INFO [0] received reconcile message
Sep 22 23:15:06.144 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(130), op: ExtentRepair { repair_id: ReconciliationId(130), extent_id: 106, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:06.144 INFO [0] client ExtentRepair { repair_id: ReconciliationId(130), extent_id: 106, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:06.144 INFO [0] Sending repair request ReconciliationId(130)
Sep 22 23:15:06.144 INFO [1] received reconcile message
Sep 22 23:15:06.144 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(130), op: ExtentRepair { repair_id: ReconciliationId(130), extent_id: 106, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:06.144 INFO [1] client ExtentRepair { repair_id: ReconciliationId(130), extent_id: 106, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:06.144 INFO [1] No action required ReconciliationId(130)
Sep 22 23:15:06.144 INFO [2] received reconcile message
Sep 22 23:15:06.144 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(130), op: ExtentRepair { repair_id: ReconciliationId(130), extent_id: 106, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:06.144 INFO [2] client ExtentRepair { repair_id: ReconciliationId(130), extent_id: 106, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:06.144 INFO [2] No action required ReconciliationId(130)
Sep 22 23:15:06.144 DEBG 130 Repair extent 106 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:06.144 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/06A.copy"
Sep 22 23:15:06.207 INFO accepted connection, remote_addr: 127.0.0.1:57444, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.207 TRCE incoming request, uri: /extent/106/files, method: GET, req_id: bae3674c-eeb2-425c-b529-928a3b001ebe, remote_addr: 127.0.0.1:57444, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.207 INFO request completed, latency_us: 192, response_code: 200, uri: /extent/106/files, method: GET, req_id: bae3674c-eeb2-425c-b529-928a3b001ebe, remote_addr: 127.0.0.1:57444, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.207 INFO eid:106 Found repair files: ["06A", "06A.db"]
Sep 22 23:15:06.208 TRCE incoming request, uri: /newextent/106/data, method: GET, req_id: 677b09ac-53df-4224-bb3c-e4fc52115168, remote_addr: 127.0.0.1:57444, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.208 INFO request completed, latency_us: 299, response_code: 200, uri: /newextent/106/data, method: GET, req_id: 677b09ac-53df-4224-bb3c-e4fc52115168, remote_addr: 127.0.0.1:57444, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.213 TRCE incoming request, uri: /newextent/106/db, method: GET, req_id: ed513e4f-50d4-4754-9851-690fc21c323f, remote_addr: 127.0.0.1:57444, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.213 INFO request completed, latency_us: 291, response_code: 200, uri: /newextent/106/db, method: GET, req_id: ed513e4f-50d4-4754-9851-690fc21c323f, remote_addr: 127.0.0.1:57444, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.214 DEBG up_ds_listen was notified
Sep 22 23:15:06.214 DEBG up_ds_listen process 1075
Sep 22 23:15:06.214 DEBG [A] ack job 1075:76, : downstairs
Sep 22 23:15:06.214 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:06.214 WARN returning error on flush!
Sep 22 23:15:06.214 DEBG Flush     :1075 extent_limit None deps:[JobId(1074), JobId(1073)] res:false f:27 g:1
Sep 22 23:15:06.214 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/06A.copy" to "/tmp/downstairs-vrx8aK6L/00/000/06A.replace"
Sep 22 23:15:06.214 INFO [lossy] skipping 1076
Sep 22 23:15:06.214 INFO [lossy] skipping 1075
Sep 22 23:15:06.214 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:06.214 DEBG Flush     :1075 extent_limit None deps:[JobId(1074), JobId(1073)] res:true f:27 g:1
Sep 22 23:15:06.215 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/06A.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:06.215 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/06A"
Sep 22 23:15:06.215 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/06A.db"
Sep 22 23:15:06.215 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:06.215 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/06A.replace" to "/tmp/downstairs-vrx8aK6L/00/000/06A.completed"
Sep 22 23:15:06.215 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:06.216 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:06.216 DEBG [0] It's time to notify for 130
Sep 22 23:15:06.216 INFO Completion from [0] id:130 status:true
Sep 22 23:15:06.216 INFO [131/752] Repair commands completed
Sep 22 23:15:06.216 INFO Pop front: ReconcileIO { id: ReconciliationId(131), op: ExtentReopen { repair_id: ReconciliationId(131), extent_id: 106 }, state: ClientData([New, New, New]) }
Sep 22 23:15:06.216 INFO Sent repair work, now wait for resp
Sep 22 23:15:06.216 INFO [0] received reconcile message
Sep 22 23:15:06.216 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(131), op: ExtentReopen { repair_id: ReconciliationId(131), extent_id: 106 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:06.216 INFO [0] client ExtentReopen { repair_id: ReconciliationId(131), extent_id: 106 }
Sep 22 23:15:06.216 INFO [1] received reconcile message
Sep 22 23:15:06.216 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(131), op: ExtentReopen { repair_id: ReconciliationId(131), extent_id: 106 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:06.216 INFO [1] client ExtentReopen { repair_id: ReconciliationId(131), extent_id: 106 }
Sep 22 23:15:06.216 INFO [2] received reconcile message
Sep 22 23:15:06.216 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(131), op: ExtentReopen { repair_id: ReconciliationId(131), extent_id: 106 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:06.216 INFO [2] client ExtentReopen { repair_id: ReconciliationId(131), extent_id: 106 }
Sep 22 23:15:06.216 DEBG 131 Reopen extent 106
Sep 22 23:15:06.217 DEBG 131 Reopen extent 106
Sep 22 23:15:06.217 DEBG 131 Reopen extent 106
Sep 22 23:15:06.218 DEBG [2] It's time to notify for 131
Sep 22 23:15:06.218 INFO Completion from [2] id:131 status:true
Sep 22 23:15:06.218 INFO [132/752] Repair commands completed
Sep 22 23:15:06.218 INFO Pop front: ReconcileIO { id: ReconciliationId(132), op: ExtentFlush { repair_id: ReconciliationId(132), extent_id: 103, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:06.218 INFO Sent repair work, now wait for resp
Sep 22 23:15:06.218 INFO [0] received reconcile message
Sep 22 23:15:06.218 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(132), op: ExtentFlush { repair_id: ReconciliationId(132), extent_id: 103, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:06.218 INFO [0] client ExtentFlush { repair_id: ReconciliationId(132), extent_id: 103, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:06.218 INFO [1] received reconcile message
Sep 22 23:15:06.218 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(132), op: ExtentFlush { repair_id: ReconciliationId(132), extent_id: 103, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:06.218 INFO [1] client ExtentFlush { repair_id: ReconciliationId(132), extent_id: 103, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:06.218 INFO [2] received reconcile message
Sep 22 23:15:06.218 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(132), op: ExtentFlush { repair_id: ReconciliationId(132), extent_id: 103, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:06.218 INFO [2] client ExtentFlush { repair_id: ReconciliationId(132), extent_id: 103, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:06.218 DEBG 132 Flush extent 103 with f:2 g:2
Sep 22 23:15:06.218 DEBG Flush just extent 103 with f:2 and g:2
Sep 22 23:15:06.218 DEBG [1] It's time to notify for 132
Sep 22 23:15:06.219 INFO Completion from [1] id:132 status:true
Sep 22 23:15:06.219 INFO [133/752] Repair commands completed
Sep 22 23:15:06.219 INFO Pop front: ReconcileIO { id: ReconciliationId(133), op: ExtentClose { repair_id: ReconciliationId(133), extent_id: 103 }, state: ClientData([New, New, New]) }
Sep 22 23:15:06.219 INFO Sent repair work, now wait for resp
Sep 22 23:15:06.219 INFO [0] received reconcile message
Sep 22 23:15:06.219 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(133), op: ExtentClose { repair_id: ReconciliationId(133), extent_id: 103 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:06.219 INFO [0] client ExtentClose { repair_id: ReconciliationId(133), extent_id: 103 }
Sep 22 23:15:06.219 INFO [1] received reconcile message
Sep 22 23:15:06.219 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(133), op: ExtentClose { repair_id: ReconciliationId(133), extent_id: 103 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:06.219 INFO [1] client ExtentClose { repair_id: ReconciliationId(133), extent_id: 103 }
Sep 22 23:15:06.219 INFO [2] received reconcile message
Sep 22 23:15:06.219 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(133), op: ExtentClose { repair_id: ReconciliationId(133), extent_id: 103 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:06.219 INFO [2] client ExtentClose { repair_id: ReconciliationId(133), extent_id: 103 }
Sep 22 23:15:06.219 DEBG 133 Close extent 103
Sep 22 23:15:06.219 DEBG 133 Close extent 103
Sep 22 23:15:06.219 DEBG 133 Close extent 103
Sep 22 23:15:06.220 DEBG [2] It's time to notify for 133
Sep 22 23:15:06.220 INFO Completion from [2] id:133 status:true
Sep 22 23:15:06.220 INFO [134/752] Repair commands completed
Sep 22 23:15:06.220 INFO Pop front: ReconcileIO { id: ReconciliationId(134), op: ExtentRepair { repair_id: ReconciliationId(134), extent_id: 103, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:06.220 INFO Sent repair work, now wait for resp
Sep 22 23:15:06.220 INFO [0] received reconcile message
Sep 22 23:15:06.220 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(134), op: ExtentRepair { repair_id: ReconciliationId(134), extent_id: 103, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:06.220 INFO [0] client ExtentRepair { repair_id: ReconciliationId(134), extent_id: 103, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:06.220 INFO [0] Sending repair request ReconciliationId(134)
Sep 22 23:15:06.220 INFO [1] received reconcile message
Sep 22 23:15:06.220 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(134), op: ExtentRepair { repair_id: ReconciliationId(134), extent_id: 103, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:06.220 INFO [1] client ExtentRepair { repair_id: ReconciliationId(134), extent_id: 103, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:06.220 INFO [1] No action required ReconciliationId(134)
Sep 22 23:15:06.220 INFO [2] received reconcile message
Sep 22 23:15:06.220 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(134), op: ExtentRepair { repair_id: ReconciliationId(134), extent_id: 103, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:06.220 INFO [2] client ExtentRepair { repair_id: ReconciliationId(134), extent_id: 103, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:06.220 INFO [2] No action required ReconciliationId(134)
Sep 22 23:15:06.220 DEBG Read      :1076 deps:[JobId(1075)] res:true
Sep 22 23:15:06.220 DEBG 134 Repair extent 103 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:06.220 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/067.copy"
Sep 22 23:15:06.242 DEBG IO Flush 1077 has deps [JobId(1076), JobId(1075)]
Sep 22 23:15:06.242 ERRO [0] job id 1075 saw error GenericError("test error")
Sep 22 23:15:06.242 DEBG [rc] retire 1075 clears [JobId(1074), JobId(1075)], : downstairs
Sep 22 23:15:06.245 DEBG Flush     :1077 extent_limit None deps:[JobId(1076), JobId(1075)] res:true f:28 g:1
Sep 22 23:15:06.245 INFO [lossy] sleeping 1 second
Sep 22 23:15:06.284 INFO accepted connection, remote_addr: 127.0.0.1:44956, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.285 TRCE incoming request, uri: /extent/103/files, method: GET, req_id: 53d79daa-7aa2-4f32-bf22-27f30c1d5c40, remote_addr: 127.0.0.1:44956, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.285 INFO request completed, latency_us: 217, response_code: 200, uri: /extent/103/files, method: GET, req_id: 53d79daa-7aa2-4f32-bf22-27f30c1d5c40, remote_addr: 127.0.0.1:44956, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.285 INFO eid:103 Found repair files: ["067", "067.db"]
Sep 22 23:15:06.285 TRCE incoming request, uri: /newextent/103/data, method: GET, req_id: 71fa9259-f306-49a6-b849-12df81cf107c, remote_addr: 127.0.0.1:44956, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.286 INFO request completed, latency_us: 343, response_code: 200, uri: /newextent/103/data, method: GET, req_id: 71fa9259-f306-49a6-b849-12df81cf107c, remote_addr: 127.0.0.1:44956, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.291 TRCE incoming request, uri: /newextent/103/db, method: GET, req_id: 357afc85-6eff-4b88-a925-f96c5501af67, remote_addr: 127.0.0.1:44956, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.291 INFO request completed, latency_us: 290, response_code: 200, uri: /newextent/103/db, method: GET, req_id: 357afc85-6eff-4b88-a925-f96c5501af67, remote_addr: 127.0.0.1:44956, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.292 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/067.copy" to "/tmp/downstairs-vrx8aK6L/00/000/067.replace"
Sep 22 23:15:06.292 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:06.293 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/067.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:06.293 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/067"
Sep 22 23:15:06.293 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/067.db"
Sep 22 23:15:06.293 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:06.293 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/067.replace" to "/tmp/downstairs-vrx8aK6L/00/000/067.completed"
Sep 22 23:15:06.294 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:06.294 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:06.294 DEBG [0] It's time to notify for 134
Sep 22 23:15:06.294 INFO Completion from [0] id:134 status:true
Sep 22 23:15:06.294 INFO [135/752] Repair commands completed
Sep 22 23:15:06.294 INFO Pop front: ReconcileIO { id: ReconciliationId(135), op: ExtentReopen { repair_id: ReconciliationId(135), extent_id: 103 }, state: ClientData([New, New, New]) }
Sep 22 23:15:06.294 INFO Sent repair work, now wait for resp
Sep 22 23:15:06.294 INFO [0] received reconcile message
Sep 22 23:15:06.294 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(135), op: ExtentReopen { repair_id: ReconciliationId(135), extent_id: 103 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:06.294 INFO [0] client ExtentReopen { repair_id: ReconciliationId(135), extent_id: 103 }
Sep 22 23:15:06.294 INFO [1] received reconcile message
Sep 22 23:15:06.294 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(135), op: ExtentReopen { repair_id: ReconciliationId(135), extent_id: 103 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:06.294 INFO [1] client ExtentReopen { repair_id: ReconciliationId(135), extent_id: 103 }
Sep 22 23:15:06.294 INFO [2] received reconcile message
Sep 22 23:15:06.294 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(135), op: ExtentReopen { repair_id: ReconciliationId(135), extent_id: 103 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:06.294 INFO [2] client ExtentReopen { repair_id: ReconciliationId(135), extent_id: 103 }
Sep 22 23:15:06.294 DEBG 135 Reopen extent 103
Sep 22 23:15:06.295 DEBG 135 Reopen extent 103
Sep 22 23:15:06.296 DEBG 135 Reopen extent 103
Sep 22 23:15:06.296 DEBG [2] It's time to notify for 135
Sep 22 23:15:06.296 INFO Completion from [2] id:135 status:true
Sep 22 23:15:06.296 INFO [136/752] Repair commands completed
Sep 22 23:15:06.296 INFO Pop front: ReconcileIO { id: ReconciliationId(136), op: ExtentFlush { repair_id: ReconciliationId(136), extent_id: 80, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:06.296 INFO Sent repair work, now wait for resp
Sep 22 23:15:06.296 INFO [0] received reconcile message
Sep 22 23:15:06.296 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(136), op: ExtentFlush { repair_id: ReconciliationId(136), extent_id: 80, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:06.296 INFO [0] client ExtentFlush { repair_id: ReconciliationId(136), extent_id: 80, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:06.296 INFO [1] received reconcile message
Sep 22 23:15:06.296 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(136), op: ExtentFlush { repair_id: ReconciliationId(136), extent_id: 80, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:06.296 INFO [1] client ExtentFlush { repair_id: ReconciliationId(136), extent_id: 80, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:06.296 INFO [2] received reconcile message
Sep 22 23:15:06.296 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(136), op: ExtentFlush { repair_id: ReconciliationId(136), extent_id: 80, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:06.296 INFO [2] client ExtentFlush { repair_id: ReconciliationId(136), extent_id: 80, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:06.297 DEBG 136 Flush extent 80 with f:2 g:2
Sep 22 23:15:06.297 DEBG Flush just extent 80 with f:2 and g:2
Sep 22 23:15:06.297 DEBG [1] It's time to notify for 136
Sep 22 23:15:06.297 INFO Completion from [1] id:136 status:true
Sep 22 23:15:06.297 INFO [137/752] Repair commands completed
Sep 22 23:15:06.297 INFO Pop front: ReconcileIO { id: ReconciliationId(137), op: ExtentClose { repair_id: ReconciliationId(137), extent_id: 80 }, state: ClientData([New, New, New]) }
Sep 22 23:15:06.297 INFO Sent repair work, now wait for resp
Sep 22 23:15:06.297 INFO [0] received reconcile message
Sep 22 23:15:06.297 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(137), op: ExtentClose { repair_id: ReconciliationId(137), extent_id: 80 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:06.297 INFO [0] client ExtentClose { repair_id: ReconciliationId(137), extent_id: 80 }
Sep 22 23:15:06.297 INFO [1] received reconcile message
Sep 22 23:15:06.297 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(137), op: ExtentClose { repair_id: ReconciliationId(137), extent_id: 80 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:06.297 INFO [1] client ExtentClose { repair_id: ReconciliationId(137), extent_id: 80 }
Sep 22 23:15:06.297 INFO [2] received reconcile message
Sep 22 23:15:06.297 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(137), op: ExtentClose { repair_id: ReconciliationId(137), extent_id: 80 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:06.297 INFO [2] client ExtentClose { repair_id: ReconciliationId(137), extent_id: 80 }
Sep 22 23:15:06.297 DEBG 137 Close extent 80
Sep 22 23:15:06.297 DEBG 137 Close extent 80
Sep 22 23:15:06.298 DEBG 137 Close extent 80
Sep 22 23:15:06.298 DEBG [2] It's time to notify for 137
Sep 22 23:15:06.298 INFO Completion from [2] id:137 status:true
Sep 22 23:15:06.298 INFO [138/752] Repair commands completed
Sep 22 23:15:06.298 INFO Pop front: ReconcileIO { id: ReconciliationId(138), op: ExtentRepair { repair_id: ReconciliationId(138), extent_id: 80, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:06.298 INFO Sent repair work, now wait for resp
Sep 22 23:15:06.298 INFO [0] received reconcile message
Sep 22 23:15:06.298 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(138), op: ExtentRepair { repair_id: ReconciliationId(138), extent_id: 80, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:06.298 INFO [0] client ExtentRepair { repair_id: ReconciliationId(138), extent_id: 80, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:06.298 INFO [0] Sending repair request ReconciliationId(138)
Sep 22 23:15:06.298 INFO [1] received reconcile message
Sep 22 23:15:06.298 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(138), op: ExtentRepair { repair_id: ReconciliationId(138), extent_id: 80, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:06.298 INFO [1] client ExtentRepair { repair_id: ReconciliationId(138), extent_id: 80, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:06.298 INFO [1] No action required ReconciliationId(138)
Sep 22 23:15:06.298 INFO [2] received reconcile message
Sep 22 23:15:06.298 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(138), op: ExtentRepair { repair_id: ReconciliationId(138), extent_id: 80, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:06.298 INFO [2] client ExtentRepair { repair_id: ReconciliationId(138), extent_id: 80, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:06.298 INFO [2] No action required ReconciliationId(138)
Sep 22 23:15:06.299 DEBG 138 Repair extent 80 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:06.299 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/050.copy"
Sep 22 23:15:06.362 INFO accepted connection, remote_addr: 127.0.0.1:45732, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.362 TRCE incoming request, uri: /extent/80/files, method: GET, req_id: dc886983-f442-477d-b9a0-0f17f9de4aa5, remote_addr: 127.0.0.1:45732, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.362 INFO request completed, latency_us: 220, response_code: 200, uri: /extent/80/files, method: GET, req_id: dc886983-f442-477d-b9a0-0f17f9de4aa5, remote_addr: 127.0.0.1:45732, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.363 INFO eid:80 Found repair files: ["050", "050.db"]
Sep 22 23:15:06.363 TRCE incoming request, uri: /newextent/80/data, method: GET, req_id: 2d573c60-d451-4a48-97bf-43e41496f446, remote_addr: 127.0.0.1:45732, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.363 INFO request completed, latency_us: 259, response_code: 200, uri: /newextent/80/data, method: GET, req_id: 2d573c60-d451-4a48-97bf-43e41496f446, remote_addr: 127.0.0.1:45732, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.368 TRCE incoming request, uri: /newextent/80/db, method: GET, req_id: a12e1f31-7297-458f-9bdd-6f9fb6d7b903, remote_addr: 127.0.0.1:45732, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.368 INFO request completed, latency_us: 294, response_code: 200, uri: /newextent/80/db, method: GET, req_id: a12e1f31-7297-458f-9bdd-6f9fb6d7b903, remote_addr: 127.0.0.1:45732, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.369 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/050.copy" to "/tmp/downstairs-vrx8aK6L/00/000/050.replace"
Sep 22 23:15:06.369 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:06.370 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/050.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:06.371 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/050"
Sep 22 23:15:06.371 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/050.db"
Sep 22 23:15:06.371 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:06.371 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/050.replace" to "/tmp/downstairs-vrx8aK6L/00/000/050.completed"
Sep 22 23:15:06.371 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:06.371 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:06.371 DEBG [0] It's time to notify for 138
Sep 22 23:15:06.371 INFO Completion from [0] id:138 status:true
Sep 22 23:15:06.371 INFO [139/752] Repair commands completed
Sep 22 23:15:06.371 INFO Pop front: ReconcileIO { id: ReconciliationId(139), op: ExtentReopen { repair_id: ReconciliationId(139), extent_id: 80 }, state: ClientData([New, New, New]) }
Sep 22 23:15:06.371 INFO Sent repair work, now wait for resp
Sep 22 23:15:06.371 INFO [0] received reconcile message
Sep 22 23:15:06.371 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(139), op: ExtentReopen { repair_id: ReconciliationId(139), extent_id: 80 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:06.371 INFO [0] client ExtentReopen { repair_id: ReconciliationId(139), extent_id: 80 }
Sep 22 23:15:06.371 INFO [1] received reconcile message
Sep 22 23:15:06.371 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(139), op: ExtentReopen { repair_id: ReconciliationId(139), extent_id: 80 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:06.371 INFO [1] client ExtentReopen { repair_id: ReconciliationId(139), extent_id: 80 }
Sep 22 23:15:06.371 INFO [2] received reconcile message
Sep 22 23:15:06.371 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(139), op: ExtentReopen { repair_id: ReconciliationId(139), extent_id: 80 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:06.371 INFO [2] client ExtentReopen { repair_id: ReconciliationId(139), extent_id: 80 }
Sep 22 23:15:06.372 DEBG 139 Reopen extent 80
Sep 22 23:15:06.372 DEBG 139 Reopen extent 80
Sep 22 23:15:06.373 DEBG 139 Reopen extent 80
Sep 22 23:15:06.373 DEBG [2] It's time to notify for 139
Sep 22 23:15:06.373 INFO Completion from [2] id:139 status:true
Sep 22 23:15:06.373 INFO [140/752] Repair commands completed
Sep 22 23:15:06.373 INFO Pop front: ReconcileIO { id: ReconciliationId(140), op: ExtentFlush { repair_id: ReconciliationId(140), extent_id: 177, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:06.373 INFO Sent repair work, now wait for resp
Sep 22 23:15:06.374 INFO [0] received reconcile message
Sep 22 23:15:06.374 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(140), op: ExtentFlush { repair_id: ReconciliationId(140), extent_id: 177, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:06.374 INFO [0] client ExtentFlush { repair_id: ReconciliationId(140), extent_id: 177, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:06.374 INFO [1] received reconcile message
Sep 22 23:15:06.374 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(140), op: ExtentFlush { repair_id: ReconciliationId(140), extent_id: 177, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:06.374 INFO [1] client ExtentFlush { repair_id: ReconciliationId(140), extent_id: 177, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:06.374 INFO [2] received reconcile message
Sep 22 23:15:06.374 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(140), op: ExtentFlush { repair_id: ReconciliationId(140), extent_id: 177, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:06.374 INFO [2] client ExtentFlush { repair_id: ReconciliationId(140), extent_id: 177, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:06.374 DEBG 140 Flush extent 177 with f:2 g:2
Sep 22 23:15:06.374 DEBG Flush just extent 177 with f:2 and g:2
Sep 22 23:15:06.374 DEBG [1] It's time to notify for 140
Sep 22 23:15:06.374 INFO Completion from [1] id:140 status:true
Sep 22 23:15:06.374 INFO [141/752] Repair commands completed
Sep 22 23:15:06.374 INFO Pop front: ReconcileIO { id: ReconciliationId(141), op: ExtentClose { repair_id: ReconciliationId(141), extent_id: 177 }, state: ClientData([New, New, New]) }
Sep 22 23:15:06.374 INFO Sent repair work, now wait for resp
Sep 22 23:15:06.374 INFO [0] received reconcile message
Sep 22 23:15:06.374 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(141), op: ExtentClose { repair_id: ReconciliationId(141), extent_id: 177 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:06.374 INFO [0] client ExtentClose { repair_id: ReconciliationId(141), extent_id: 177 }
Sep 22 23:15:06.374 INFO [1] received reconcile message
Sep 22 23:15:06.374 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(141), op: ExtentClose { repair_id: ReconciliationId(141), extent_id: 177 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:06.374 INFO [1] client ExtentClose { repair_id: ReconciliationId(141), extent_id: 177 }
Sep 22 23:15:06.374 INFO [2] received reconcile message
Sep 22 23:15:06.374 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(141), op: ExtentClose { repair_id: ReconciliationId(141), extent_id: 177 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:06.374 INFO [2] client ExtentClose { repair_id: ReconciliationId(141), extent_id: 177 }
Sep 22 23:15:06.374 DEBG 141 Close extent 177
Sep 22 23:15:06.375 DEBG 141 Close extent 177
Sep 22 23:15:06.375 DEBG 141 Close extent 177
Sep 22 23:15:06.375 DEBG [2] It's time to notify for 141
Sep 22 23:15:06.375 INFO Completion from [2] id:141 status:true
Sep 22 23:15:06.375 INFO [142/752] Repair commands completed
Sep 22 23:15:06.375 INFO Pop front: ReconcileIO { id: ReconciliationId(142), op: ExtentRepair { repair_id: ReconciliationId(142), extent_id: 177, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:06.375 INFO Sent repair work, now wait for resp
Sep 22 23:15:06.375 INFO [0] received reconcile message
Sep 22 23:15:06.375 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(142), op: ExtentRepair { repair_id: ReconciliationId(142), extent_id: 177, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:06.376 INFO [0] client ExtentRepair { repair_id: ReconciliationId(142), extent_id: 177, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:06.376 INFO [0] Sending repair request ReconciliationId(142)
Sep 22 23:15:06.376 INFO [1] received reconcile message
Sep 22 23:15:06.376 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(142), op: ExtentRepair { repair_id: ReconciliationId(142), extent_id: 177, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:06.376 INFO [1] client ExtentRepair { repair_id: ReconciliationId(142), extent_id: 177, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:06.376 INFO [1] No action required ReconciliationId(142)
Sep 22 23:15:06.376 INFO [2] received reconcile message
Sep 22 23:15:06.376 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(142), op: ExtentRepair { repair_id: ReconciliationId(142), extent_id: 177, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:06.376 INFO [2] client ExtentRepair { repair_id: ReconciliationId(142), extent_id: 177, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:06.376 INFO [2] No action required ReconciliationId(142)
Sep 22 23:15:06.376 DEBG 142 Repair extent 177 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:06.376 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/0B1.copy"
Sep 22 23:15:06.439 INFO accepted connection, remote_addr: 127.0.0.1:51983, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.439 TRCE incoming request, uri: /extent/177/files, method: GET, req_id: fa4f6e7b-68a9-47f3-b000-be14676a3687, remote_addr: 127.0.0.1:51983, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.440 INFO request completed, latency_us: 198, response_code: 200, uri: /extent/177/files, method: GET, req_id: fa4f6e7b-68a9-47f3-b000-be14676a3687, remote_addr: 127.0.0.1:51983, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.440 INFO eid:177 Found repair files: ["0B1", "0B1.db"]
Sep 22 23:15:06.440 TRCE incoming request, uri: /newextent/177/data, method: GET, req_id: 1563ffb4-af56-4ec4-873a-73496fb38fc6, remote_addr: 127.0.0.1:51983, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.440 INFO request completed, latency_us: 253, response_code: 200, uri: /newextent/177/data, method: GET, req_id: 1563ffb4-af56-4ec4-873a-73496fb38fc6, remote_addr: 127.0.0.1:51983, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.445 TRCE incoming request, uri: /newextent/177/db, method: GET, req_id: b7bfc40c-b8f2-4ec8-aca1-66d056995dde, remote_addr: 127.0.0.1:51983, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.445 INFO request completed, latency_us: 293, response_code: 200, uri: /newextent/177/db, method: GET, req_id: b7bfc40c-b8f2-4ec8-aca1-66d056995dde, remote_addr: 127.0.0.1:51983, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.447 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/0B1.copy" to "/tmp/downstairs-vrx8aK6L/00/000/0B1.replace"
Sep 22 23:15:06.447 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:06.447 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/0B1.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:06.448 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/0B1"
Sep 22 23:15:06.448 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/0B1.db"
Sep 22 23:15:06.448 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:06.448 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/0B1.replace" to "/tmp/downstairs-vrx8aK6L/00/000/0B1.completed"
Sep 22 23:15:06.448 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:06.448 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:06.448 DEBG [0] It's time to notify for 142
Sep 22 23:15:06.448 INFO Completion from [0] id:142 status:true
Sep 22 23:15:06.448 INFO [143/752] Repair commands completed
Sep 22 23:15:06.448 INFO Pop front: ReconcileIO { id: ReconciliationId(143), op: ExtentReopen { repair_id: ReconciliationId(143), extent_id: 177 }, state: ClientData([New, New, New]) }
Sep 22 23:15:06.448 INFO Sent repair work, now wait for resp
Sep 22 23:15:06.448 INFO [0] received reconcile message
Sep 22 23:15:06.448 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(143), op: ExtentReopen { repair_id: ReconciliationId(143), extent_id: 177 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:06.448 INFO [0] client ExtentReopen { repair_id: ReconciliationId(143), extent_id: 177 }
Sep 22 23:15:06.448 INFO [1] received reconcile message
Sep 22 23:15:06.448 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(143), op: ExtentReopen { repair_id: ReconciliationId(143), extent_id: 177 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:06.448 INFO [1] client ExtentReopen { repair_id: ReconciliationId(143), extent_id: 177 }
Sep 22 23:15:06.448 INFO [2] received reconcile message
Sep 22 23:15:06.448 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(143), op: ExtentReopen { repair_id: ReconciliationId(143), extent_id: 177 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:06.448 INFO [2] client ExtentReopen { repair_id: ReconciliationId(143), extent_id: 177 }
Sep 22 23:15:06.449 DEBG 143 Reopen extent 177
Sep 22 23:15:06.449 DEBG 143 Reopen extent 177
Sep 22 23:15:06.450 DEBG 143 Reopen extent 177
Sep 22 23:15:06.450 DEBG [2] It's time to notify for 143
Sep 22 23:15:06.450 INFO Completion from [2] id:143 status:true
Sep 22 23:15:06.450 INFO [144/752] Repair commands completed
Sep 22 23:15:06.450 INFO Pop front: ReconcileIO { id: ReconciliationId(144), op: ExtentFlush { repair_id: ReconciliationId(144), extent_id: 84, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:06.450 INFO Sent repair work, now wait for resp
Sep 22 23:15:06.450 INFO [0] received reconcile message
Sep 22 23:15:06.450 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(144), op: ExtentFlush { repair_id: ReconciliationId(144), extent_id: 84, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:06.450 INFO [0] client ExtentFlush { repair_id: ReconciliationId(144), extent_id: 84, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:06.450 INFO [1] received reconcile message
Sep 22 23:15:06.450 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(144), op: ExtentFlush { repair_id: ReconciliationId(144), extent_id: 84, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:06.450 INFO [1] client ExtentFlush { repair_id: ReconciliationId(144), extent_id: 84, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:06.451 INFO [2] received reconcile message
Sep 22 23:15:06.451 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(144), op: ExtentFlush { repair_id: ReconciliationId(144), extent_id: 84, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:06.451 INFO [2] client ExtentFlush { repair_id: ReconciliationId(144), extent_id: 84, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:06.451 DEBG 144 Flush extent 84 with f:2 g:2
Sep 22 23:15:06.451 DEBG Flush just extent 84 with f:2 and g:2
Sep 22 23:15:06.451 DEBG [1] It's time to notify for 144
Sep 22 23:15:06.451 INFO Completion from [1] id:144 status:true
Sep 22 23:15:06.451 INFO [145/752] Repair commands completed
Sep 22 23:15:06.451 INFO Pop front: ReconcileIO { id: ReconciliationId(145), op: ExtentClose { repair_id: ReconciliationId(145), extent_id: 84 }, state: ClientData([New, New, New]) }
Sep 22 23:15:06.451 INFO Sent repair work, now wait for resp
Sep 22 23:15:06.451 INFO [0] received reconcile message
Sep 22 23:15:06.451 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(145), op: ExtentClose { repair_id: ReconciliationId(145), extent_id: 84 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:06.451 INFO [0] client ExtentClose { repair_id: ReconciliationId(145), extent_id: 84 }
Sep 22 23:15:06.451 INFO [1] received reconcile message
Sep 22 23:15:06.451 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(145), op: ExtentClose { repair_id: ReconciliationId(145), extent_id: 84 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:06.451 INFO [1] client ExtentClose { repair_id: ReconciliationId(145), extent_id: 84 }
Sep 22 23:15:06.451 INFO [2] received reconcile message
Sep 22 23:15:06.451 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(145), op: ExtentClose { repair_id: ReconciliationId(145), extent_id: 84 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:06.451 INFO [2] client ExtentClose { repair_id: ReconciliationId(145), extent_id: 84 }
Sep 22 23:15:06.451 DEBG 145 Close extent 84
Sep 22 23:15:06.451 DEBG 145 Close extent 84
Sep 22 23:15:06.452 DEBG 145 Close extent 84
Sep 22 23:15:06.452 DEBG [2] It's time to notify for 145
Sep 22 23:15:06.452 INFO Completion from [2] id:145 status:true
Sep 22 23:15:06.452 INFO [146/752] Repair commands completed
Sep 22 23:15:06.452 INFO Pop front: ReconcileIO { id: ReconciliationId(146), op: ExtentRepair { repair_id: ReconciliationId(146), extent_id: 84, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:06.452 INFO Sent repair work, now wait for resp
Sep 22 23:15:06.452 INFO [0] received reconcile message
Sep 22 23:15:06.452 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(146), op: ExtentRepair { repair_id: ReconciliationId(146), extent_id: 84, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:06.452 INFO [0] client ExtentRepair { repair_id: ReconciliationId(146), extent_id: 84, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:06.452 INFO [0] Sending repair request ReconciliationId(146)
Sep 22 23:15:06.452 INFO [1] received reconcile message
Sep 22 23:15:06.452 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(146), op: ExtentRepair { repair_id: ReconciliationId(146), extent_id: 84, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:06.452 INFO [1] client ExtentRepair { repair_id: ReconciliationId(146), extent_id: 84, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:06.452 INFO [1] No action required ReconciliationId(146)
Sep 22 23:15:06.453 INFO [2] received reconcile message
Sep 22 23:15:06.453 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(146), op: ExtentRepair { repair_id: ReconciliationId(146), extent_id: 84, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:06.453 INFO [2] client ExtentRepair { repair_id: ReconciliationId(146), extent_id: 84, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:06.453 INFO [2] No action required ReconciliationId(146)
Sep 22 23:15:06.453 DEBG 146 Repair extent 84 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:06.453 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/054.copy"
Sep 22 23:15:06.516 INFO accepted connection, remote_addr: 127.0.0.1:65235, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.516 TRCE incoming request, uri: /extent/84/files, method: GET, req_id: 52bd89b2-8419-4987-a6d3-095edf180ca8, remote_addr: 127.0.0.1:65235, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.517 INFO request completed, latency_us: 189, response_code: 200, uri: /extent/84/files, method: GET, req_id: 52bd89b2-8419-4987-a6d3-095edf180ca8, remote_addr: 127.0.0.1:65235, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.517 INFO eid:84 Found repair files: ["054", "054.db"]
Sep 22 23:15:06.517 TRCE incoming request, uri: /newextent/84/data, method: GET, req_id: 11f42127-b083-4337-820e-647f55d9b7fd, remote_addr: 127.0.0.1:65235, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.517 INFO request completed, latency_us: 251, response_code: 200, uri: /newextent/84/data, method: GET, req_id: 11f42127-b083-4337-820e-647f55d9b7fd, remote_addr: 127.0.0.1:65235, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.522 TRCE incoming request, uri: /newextent/84/db, method: GET, req_id: ed78afb8-8c12-4c0f-a465-38275bf30c55, remote_addr: 127.0.0.1:65235, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.523 INFO request completed, latency_us: 299, response_code: 200, uri: /newextent/84/db, method: GET, req_id: ed78afb8-8c12-4c0f-a465-38275bf30c55, remote_addr: 127.0.0.1:65235, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.524 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/054.copy" to "/tmp/downstairs-vrx8aK6L/00/000/054.replace"
Sep 22 23:15:06.524 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:06.524 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/054.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:06.525 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/054"
Sep 22 23:15:06.525 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/054.db"
Sep 22 23:15:06.525 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:06.525 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/054.replace" to "/tmp/downstairs-vrx8aK6L/00/000/054.completed"
Sep 22 23:15:06.525 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:06.525 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:06.525 DEBG [0] It's time to notify for 146
Sep 22 23:15:06.525 INFO Completion from [0] id:146 status:true
Sep 22 23:15:06.525 INFO [147/752] Repair commands completed
Sep 22 23:15:06.525 INFO Pop front: ReconcileIO { id: ReconciliationId(147), op: ExtentReopen { repair_id: ReconciliationId(147), extent_id: 84 }, state: ClientData([New, New, New]) }
Sep 22 23:15:06.525 INFO Sent repair work, now wait for resp
Sep 22 23:15:06.525 INFO [0] received reconcile message
Sep 22 23:15:06.525 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(147), op: ExtentReopen { repair_id: ReconciliationId(147), extent_id: 84 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:06.525 INFO [0] client ExtentReopen { repair_id: ReconciliationId(147), extent_id: 84 }
Sep 22 23:15:06.525 INFO [1] received reconcile message
Sep 22 23:15:06.525 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(147), op: ExtentReopen { repair_id: ReconciliationId(147), extent_id: 84 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:06.525 INFO [1] client ExtentReopen { repair_id: ReconciliationId(147), extent_id: 84 }
Sep 22 23:15:06.525 INFO [2] received reconcile message
Sep 22 23:15:06.525 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(147), op: ExtentReopen { repair_id: ReconciliationId(147), extent_id: 84 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:06.526 INFO [2] client ExtentReopen { repair_id: ReconciliationId(147), extent_id: 84 }
Sep 22 23:15:06.526 DEBG 147 Reopen extent 84
Sep 22 23:15:06.526 DEBG 147 Reopen extent 84
Sep 22 23:15:06.527 DEBG 147 Reopen extent 84
Sep 22 23:15:06.527 DEBG [2] It's time to notify for 147
Sep 22 23:15:06.527 INFO Completion from [2] id:147 status:true
Sep 22 23:15:06.527 INFO [148/752] Repair commands completed
Sep 22 23:15:06.527 INFO Pop front: ReconcileIO { id: ReconciliationId(148), op: ExtentFlush { repair_id: ReconciliationId(148), extent_id: 91, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:06.527 INFO Sent repair work, now wait for resp
Sep 22 23:15:06.527 INFO [0] received reconcile message
Sep 22 23:15:06.527 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(148), op: ExtentFlush { repair_id: ReconciliationId(148), extent_id: 91, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:06.527 INFO [0] client ExtentFlush { repair_id: ReconciliationId(148), extent_id: 91, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:06.527 INFO [1] received reconcile message
Sep 22 23:15:06.528 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(148), op: ExtentFlush { repair_id: ReconciliationId(148), extent_id: 91, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:06.528 INFO [1] client ExtentFlush { repair_id: ReconciliationId(148), extent_id: 91, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:06.528 INFO [2] received reconcile message
Sep 22 23:15:06.528 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(148), op: ExtentFlush { repair_id: ReconciliationId(148), extent_id: 91, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:06.528 INFO [2] client ExtentFlush { repair_id: ReconciliationId(148), extent_id: 91, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:06.528 DEBG 148 Flush extent 91 with f:2 g:2
Sep 22 23:15:06.528 DEBG Flush just extent 91 with f:2 and g:2
Sep 22 23:15:06.528 DEBG [1] It's time to notify for 148
Sep 22 23:15:06.528 INFO Completion from [1] id:148 status:true
Sep 22 23:15:06.528 INFO [149/752] Repair commands completed
Sep 22 23:15:06.528 INFO Pop front: ReconcileIO { id: ReconciliationId(149), op: ExtentClose { repair_id: ReconciliationId(149), extent_id: 91 }, state: ClientData([New, New, New]) }
Sep 22 23:15:06.528 INFO Sent repair work, now wait for resp
Sep 22 23:15:06.528 INFO [0] received reconcile message
Sep 22 23:15:06.528 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(149), op: ExtentClose { repair_id: ReconciliationId(149), extent_id: 91 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:06.528 INFO [0] client ExtentClose { repair_id: ReconciliationId(149), extent_id: 91 }
Sep 22 23:15:06.528 INFO [1] received reconcile message
Sep 22 23:15:06.528 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(149), op: ExtentClose { repair_id: ReconciliationId(149), extent_id: 91 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:06.528 INFO [1] client ExtentClose { repair_id: ReconciliationId(149), extent_id: 91 }
Sep 22 23:15:06.528 INFO [2] received reconcile message
Sep 22 23:15:06.528 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(149), op: ExtentClose { repair_id: ReconciliationId(149), extent_id: 91 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:06.528 INFO [2] client ExtentClose { repair_id: ReconciliationId(149), extent_id: 91 }
Sep 22 23:15:06.528 DEBG 149 Close extent 91
Sep 22 23:15:06.529 DEBG 149 Close extent 91
Sep 22 23:15:06.529 DEBG 149 Close extent 91
Sep 22 23:15:06.529 DEBG [2] It's time to notify for 149
Sep 22 23:15:06.529 INFO Completion from [2] id:149 status:true
Sep 22 23:15:06.529 INFO [150/752] Repair commands completed
Sep 22 23:15:06.529 INFO Pop front: ReconcileIO { id: ReconciliationId(150), op: ExtentRepair { repair_id: ReconciliationId(150), extent_id: 91, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:06.529 INFO Sent repair work, now wait for resp
Sep 22 23:15:06.529 INFO [0] received reconcile message
Sep 22 23:15:06.529 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(150), op: ExtentRepair { repair_id: ReconciliationId(150), extent_id: 91, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:06.529 INFO [0] client ExtentRepair { repair_id: ReconciliationId(150), extent_id: 91, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:06.529 INFO [0] Sending repair request ReconciliationId(150)
Sep 22 23:15:06.530 INFO [1] received reconcile message
Sep 22 23:15:06.530 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(150), op: ExtentRepair { repair_id: ReconciliationId(150), extent_id: 91, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:06.530 INFO [1] client ExtentRepair { repair_id: ReconciliationId(150), extent_id: 91, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:06.530 INFO [1] No action required ReconciliationId(150)
Sep 22 23:15:06.530 INFO [2] received reconcile message
Sep 22 23:15:06.530 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(150), op: ExtentRepair { repair_id: ReconciliationId(150), extent_id: 91, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:06.530 INFO [2] client ExtentRepair { repair_id: ReconciliationId(150), extent_id: 91, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:06.530 INFO [2] No action required ReconciliationId(150)
Sep 22 23:15:06.530 DEBG 150 Repair extent 91 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:06.530 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/05B.copy"
Sep 22 23:15:06.594 INFO accepted connection, remote_addr: 127.0.0.1:56542, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.594 TRCE incoming request, uri: /extent/91/files, method: GET, req_id: 186f8ffd-0269-47e2-b743-690c971979c6, remote_addr: 127.0.0.1:56542, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.595 INFO request completed, latency_us: 191, response_code: 200, uri: /extent/91/files, method: GET, req_id: 186f8ffd-0269-47e2-b743-690c971979c6, remote_addr: 127.0.0.1:56542, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.595 INFO eid:91 Found repair files: ["05B", "05B.db"]
Sep 22 23:15:06.595 TRCE incoming request, uri: /newextent/91/data, method: GET, req_id: 967527ba-384d-4767-9af9-84252d59611f, remote_addr: 127.0.0.1:56542, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.595 INFO request completed, latency_us: 251, response_code: 200, uri: /newextent/91/data, method: GET, req_id: 967527ba-384d-4767-9af9-84252d59611f, remote_addr: 127.0.0.1:56542, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.600 TRCE incoming request, uri: /newextent/91/db, method: GET, req_id: 63c3aaa1-23d4-4073-9c80-965816ea0c63, remote_addr: 127.0.0.1:56542, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.600 INFO request completed, latency_us: 293, response_code: 200, uri: /newextent/91/db, method: GET, req_id: 63c3aaa1-23d4-4073-9c80-965816ea0c63, remote_addr: 127.0.0.1:56542, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.602 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/05B.copy" to "/tmp/downstairs-vrx8aK6L/00/000/05B.replace"
Sep 22 23:15:06.602 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:06.602 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/05B.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:06.603 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/05B"
Sep 22 23:15:06.603 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/05B.db"
Sep 22 23:15:06.603 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:06.603 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/05B.replace" to "/tmp/downstairs-vrx8aK6L/00/000/05B.completed"
Sep 22 23:15:06.603 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:06.603 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:06.603 DEBG [0] It's time to notify for 150
Sep 22 23:15:06.603 INFO Completion from [0] id:150 status:true
Sep 22 23:15:06.603 INFO [151/752] Repair commands completed
Sep 22 23:15:06.603 INFO Pop front: ReconcileIO { id: ReconciliationId(151), op: ExtentReopen { repair_id: ReconciliationId(151), extent_id: 91 }, state: ClientData([New, New, New]) }
Sep 22 23:15:06.603 INFO Sent repair work, now wait for resp
Sep 22 23:15:06.603 INFO [0] received reconcile message
Sep 22 23:15:06.603 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(151), op: ExtentReopen { repair_id: ReconciliationId(151), extent_id: 91 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:06.603 INFO [0] client ExtentReopen { repair_id: ReconciliationId(151), extent_id: 91 }
Sep 22 23:15:06.603 INFO [1] received reconcile message
Sep 22 23:15:06.603 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(151), op: ExtentReopen { repair_id: ReconciliationId(151), extent_id: 91 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:06.603 INFO [1] client ExtentReopen { repair_id: ReconciliationId(151), extent_id: 91 }
Sep 22 23:15:06.603 INFO [2] received reconcile message
Sep 22 23:15:06.603 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(151), op: ExtentReopen { repair_id: ReconciliationId(151), extent_id: 91 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:06.603 INFO [2] client ExtentReopen { repair_id: ReconciliationId(151), extent_id: 91 }
Sep 22 23:15:06.604 DEBG 151 Reopen extent 91
Sep 22 23:15:06.604 DEBG 151 Reopen extent 91
Sep 22 23:15:06.605 DEBG 151 Reopen extent 91
Sep 22 23:15:06.605 DEBG [2] It's time to notify for 151
Sep 22 23:15:06.605 INFO Completion from [2] id:151 status:true
Sep 22 23:15:06.605 INFO [152/752] Repair commands completed
Sep 22 23:15:06.605 INFO Pop front: ReconcileIO { id: ReconciliationId(152), op: ExtentFlush { repair_id: ReconciliationId(152), extent_id: 2, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:06.605 INFO Sent repair work, now wait for resp
Sep 22 23:15:06.605 INFO [0] received reconcile message
Sep 22 23:15:06.605 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(152), op: ExtentFlush { repair_id: ReconciliationId(152), extent_id: 2, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:06.605 INFO [0] client ExtentFlush { repair_id: ReconciliationId(152), extent_id: 2, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:06.605 INFO [1] received reconcile message
Sep 22 23:15:06.605 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(152), op: ExtentFlush { repair_id: ReconciliationId(152), extent_id: 2, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:06.605 INFO [1] client ExtentFlush { repair_id: ReconciliationId(152), extent_id: 2, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:06.605 INFO [2] received reconcile message
Sep 22 23:15:06.606 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(152), op: ExtentFlush { repair_id: ReconciliationId(152), extent_id: 2, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:06.606 INFO [2] client ExtentFlush { repair_id: ReconciliationId(152), extent_id: 2, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:06.606 DEBG 152 Flush extent 2 with f:2 g:2
Sep 22 23:15:06.606 DEBG Flush just extent 2 with f:2 and g:2
Sep 22 23:15:06.606 DEBG [1] It's time to notify for 152
Sep 22 23:15:06.606 INFO Completion from [1] id:152 status:true
Sep 22 23:15:06.606 INFO [153/752] Repair commands completed
Sep 22 23:15:06.606 INFO Pop front: ReconcileIO { id: ReconciliationId(153), op: ExtentClose { repair_id: ReconciliationId(153), extent_id: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:06.606 INFO Sent repair work, now wait for resp
Sep 22 23:15:06.606 INFO [0] received reconcile message
Sep 22 23:15:06.606 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(153), op: ExtentClose { repair_id: ReconciliationId(153), extent_id: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:06.606 INFO [0] client ExtentClose { repair_id: ReconciliationId(153), extent_id: 2 }
Sep 22 23:15:06.606 INFO [1] received reconcile message
Sep 22 23:15:06.606 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(153), op: ExtentClose { repair_id: ReconciliationId(153), extent_id: 2 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:06.606 INFO [1] client ExtentClose { repair_id: ReconciliationId(153), extent_id: 2 }
Sep 22 23:15:06.606 INFO [2] received reconcile message
Sep 22 23:15:06.606 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(153), op: ExtentClose { repair_id: ReconciliationId(153), extent_id: 2 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:06.606 INFO [2] client ExtentClose { repair_id: ReconciliationId(153), extent_id: 2 }
Sep 22 23:15:06.606 DEBG 153 Close extent 2
Sep 22 23:15:06.606 DEBG 153 Close extent 2
Sep 22 23:15:06.607 DEBG 153 Close extent 2
Sep 22 23:15:06.607 DEBG [2] It's time to notify for 153
Sep 22 23:15:06.607 INFO Completion from [2] id:153 status:true
Sep 22 23:15:06.607 INFO [154/752] Repair commands completed
Sep 22 23:15:06.607 INFO Pop front: ReconcileIO { id: ReconciliationId(154), op: ExtentRepair { repair_id: ReconciliationId(154), extent_id: 2, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:06.607 INFO Sent repair work, now wait for resp
Sep 22 23:15:06.607 INFO [0] received reconcile message
Sep 22 23:15:06.607 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(154), op: ExtentRepair { repair_id: ReconciliationId(154), extent_id: 2, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:06.607 INFO [0] client ExtentRepair { repair_id: ReconciliationId(154), extent_id: 2, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:06.607 INFO [0] Sending repair request ReconciliationId(154)
Sep 22 23:15:06.607 INFO [1] received reconcile message
Sep 22 23:15:06.607 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(154), op: ExtentRepair { repair_id: ReconciliationId(154), extent_id: 2, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:06.607 INFO [1] client ExtentRepair { repair_id: ReconciliationId(154), extent_id: 2, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:06.607 INFO [1] No action required ReconciliationId(154)
Sep 22 23:15:06.607 INFO [2] received reconcile message
Sep 22 23:15:06.607 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(154), op: ExtentRepair { repair_id: ReconciliationId(154), extent_id: 2, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:06.608 INFO [2] client ExtentRepair { repair_id: ReconciliationId(154), extent_id: 2, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:06.608 INFO [2] No action required ReconciliationId(154)
Sep 22 23:15:06.608 DEBG 154 Repair extent 2 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:06.608 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/002.copy"
Sep 22 23:15:06.622 DEBG [2] Read AckReady 1076, : downstairs
Sep 22 23:15:06.623 DEBG up_ds_listen was notified
Sep 22 23:15:06.623 DEBG up_ds_listen process 1076
Sep 22 23:15:06.623 DEBG [A] ack job 1076:77, : downstairs
Sep 22 23:15:06.672 INFO accepted connection, remote_addr: 127.0.0.1:56428, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.672 TRCE incoming request, uri: /extent/2/files, method: GET, req_id: 2d4d3eb2-542d-4858-a034-8ce67eb076e4, remote_addr: 127.0.0.1:56428, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.672 INFO request completed, latency_us: 195, response_code: 200, uri: /extent/2/files, method: GET, req_id: 2d4d3eb2-542d-4858-a034-8ce67eb076e4, remote_addr: 127.0.0.1:56428, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.672 INFO eid:2 Found repair files: ["002", "002.db"]
Sep 22 23:15:06.673 TRCE incoming request, uri: /newextent/2/data, method: GET, req_id: 85a83a53-d586-477c-ac09-dcbcc062a82a, remote_addr: 127.0.0.1:56428, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.673 INFO request completed, latency_us: 297, response_code: 200, uri: /newextent/2/data, method: GET, req_id: 85a83a53-d586-477c-ac09-dcbcc062a82a, remote_addr: 127.0.0.1:56428, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.676 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:06.678 TRCE incoming request, uri: /newextent/2/db, method: GET, req_id: d3ff304f-acbe-4218-bcc7-3d6b04cf2f68, remote_addr: 127.0.0.1:56428, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.678 DEBG Flush     :1077 extent_limit None deps:[JobId(1076), JobId(1075)] res:true f:28 g:1
Sep 22 23:15:06.678 INFO request completed, latency_us: 296, response_code: 200, uri: /newextent/2/db, method: GET, req_id: d3ff304f-acbe-4218-bcc7-3d6b04cf2f68, remote_addr: 127.0.0.1:56428, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.679 INFO [lossy] sleeping 1 second
Sep 22 23:15:06.680 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/002.copy" to "/tmp/downstairs-vrx8aK6L/00/000/002.replace"
Sep 22 23:15:06.680 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:06.680 DEBG Flush     :1077 extent_limit None deps:[JobId(1076), JobId(1075)] res:true f:28 g:1
Sep 22 23:15:06.680 INFO [lossy] sleeping 1 second
Sep 22 23:15:06.680 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/002.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:06.681 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/002"
Sep 22 23:15:06.681 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/002.db"
Sep 22 23:15:06.681 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:06.681 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/002.replace" to "/tmp/downstairs-vrx8aK6L/00/000/002.completed"
Sep 22 23:15:06.681 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:06.681 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:06.681 DEBG [0] It's time to notify for 154
Sep 22 23:15:06.681 INFO Completion from [0] id:154 status:true
Sep 22 23:15:06.681 INFO [155/752] Repair commands completed
Sep 22 23:15:06.681 INFO Pop front: ReconcileIO { id: ReconciliationId(155), op: ExtentReopen { repair_id: ReconciliationId(155), extent_id: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:06.681 INFO Sent repair work, now wait for resp
Sep 22 23:15:06.681 INFO [0] received reconcile message
Sep 22 23:15:06.681 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(155), op: ExtentReopen { repair_id: ReconciliationId(155), extent_id: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:06.681 INFO [0] client ExtentReopen { repair_id: ReconciliationId(155), extent_id: 2 }
Sep 22 23:15:06.681 INFO [1] received reconcile message
Sep 22 23:15:06.681 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(155), op: ExtentReopen { repair_id: ReconciliationId(155), extent_id: 2 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:06.681 INFO [1] client ExtentReopen { repair_id: ReconciliationId(155), extent_id: 2 }
Sep 22 23:15:06.681 INFO [2] received reconcile message
Sep 22 23:15:06.681 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(155), op: ExtentReopen { repair_id: ReconciliationId(155), extent_id: 2 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:06.681 INFO [2] client ExtentReopen { repair_id: ReconciliationId(155), extent_id: 2 }
Sep 22 23:15:06.682 DEBG 155 Reopen extent 2
Sep 22 23:15:06.682 DEBG 155 Reopen extent 2
Sep 22 23:15:06.683 DEBG 155 Reopen extent 2
Sep 22 23:15:06.683 DEBG [2] It's time to notify for 155
Sep 22 23:15:06.683 INFO Completion from [2] id:155 status:true
Sep 22 23:15:06.683 INFO [156/752] Repair commands completed
Sep 22 23:15:06.683 INFO Pop front: ReconcileIO { id: ReconciliationId(156), op: ExtentFlush { repair_id: ReconciliationId(156), extent_id: 99, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:06.683 INFO Sent repair work, now wait for resp
Sep 22 23:15:06.683 INFO [0] received reconcile message
Sep 22 23:15:06.683 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(156), op: ExtentFlush { repair_id: ReconciliationId(156), extent_id: 99, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:06.683 INFO [0] client ExtentFlush { repair_id: ReconciliationId(156), extent_id: 99, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:06.683 INFO [1] received reconcile message
Sep 22 23:15:06.683 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(156), op: ExtentFlush { repair_id: ReconciliationId(156), extent_id: 99, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:06.683 INFO [1] client ExtentFlush { repair_id: ReconciliationId(156), extent_id: 99, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:06.684 INFO [2] received reconcile message
Sep 22 23:15:06.684 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(156), op: ExtentFlush { repair_id: ReconciliationId(156), extent_id: 99, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:06.684 INFO [2] client ExtentFlush { repair_id: ReconciliationId(156), extent_id: 99, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:06.684 DEBG 156 Flush extent 99 with f:2 g:2
Sep 22 23:15:06.684 DEBG Flush just extent 99 with f:2 and g:2
Sep 22 23:15:06.684 DEBG [1] It's time to notify for 156
Sep 22 23:15:06.684 INFO Completion from [1] id:156 status:true
Sep 22 23:15:06.684 INFO [157/752] Repair commands completed
Sep 22 23:15:06.684 INFO Pop front: ReconcileIO { id: ReconciliationId(157), op: ExtentClose { repair_id: ReconciliationId(157), extent_id: 99 }, state: ClientData([New, New, New]) }
Sep 22 23:15:06.684 INFO Sent repair work, now wait for resp
Sep 22 23:15:06.684 INFO [0] received reconcile message
Sep 22 23:15:06.684 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(157), op: ExtentClose { repair_id: ReconciliationId(157), extent_id: 99 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:06.684 INFO [0] client ExtentClose { repair_id: ReconciliationId(157), extent_id: 99 }
Sep 22 23:15:06.684 INFO [1] received reconcile message
Sep 22 23:15:06.684 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(157), op: ExtentClose { repair_id: ReconciliationId(157), extent_id: 99 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:06.684 INFO [1] client ExtentClose { repair_id: ReconciliationId(157), extent_id: 99 }
Sep 22 23:15:06.684 INFO [2] received reconcile message
Sep 22 23:15:06.684 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(157), op: ExtentClose { repair_id: ReconciliationId(157), extent_id: 99 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:06.684 INFO [2] client ExtentClose { repair_id: ReconciliationId(157), extent_id: 99 }
Sep 22 23:15:06.684 DEBG 157 Close extent 99
Sep 22 23:15:06.684 DEBG 157 Close extent 99
Sep 22 23:15:06.685 DEBG 157 Close extent 99
Sep 22 23:15:06.685 DEBG [2] It's time to notify for 157
Sep 22 23:15:06.685 INFO Completion from [2] id:157 status:true
Sep 22 23:15:06.685 INFO [158/752] Repair commands completed
Sep 22 23:15:06.685 INFO Pop front: ReconcileIO { id: ReconciliationId(158), op: ExtentRepair { repair_id: ReconciliationId(158), extent_id: 99, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:06.685 INFO Sent repair work, now wait for resp
Sep 22 23:15:06.685 INFO [0] received reconcile message
Sep 22 23:15:06.685 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(158), op: ExtentRepair { repair_id: ReconciliationId(158), extent_id: 99, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:06.685 INFO [0] client ExtentRepair { repair_id: ReconciliationId(158), extent_id: 99, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:06.685 INFO [0] Sending repair request ReconciliationId(158)
Sep 22 23:15:06.685 INFO [1] received reconcile message
Sep 22 23:15:06.685 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(158), op: ExtentRepair { repair_id: ReconciliationId(158), extent_id: 99, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:06.685 INFO [1] client ExtentRepair { repair_id: ReconciliationId(158), extent_id: 99, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:06.685 INFO [1] No action required ReconciliationId(158)
Sep 22 23:15:06.686 INFO [2] received reconcile message
Sep 22 23:15:06.686 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(158), op: ExtentRepair { repair_id: ReconciliationId(158), extent_id: 99, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:06.686 INFO [2] client ExtentRepair { repair_id: ReconciliationId(158), extent_id: 99, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:06.686 INFO [2] No action required ReconciliationId(158)
Sep 22 23:15:06.686 DEBG 158 Repair extent 99 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:06.686 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/063.copy"
Sep 22 23:15:06.749 INFO accepted connection, remote_addr: 127.0.0.1:52616, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.750 TRCE incoming request, uri: /extent/99/files, method: GET, req_id: 9a61a459-8283-4c21-a850-25ef26bfd0d4, remote_addr: 127.0.0.1:52616, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.750 INFO request completed, latency_us: 191, response_code: 200, uri: /extent/99/files, method: GET, req_id: 9a61a459-8283-4c21-a850-25ef26bfd0d4, remote_addr: 127.0.0.1:52616, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.750 INFO eid:99 Found repair files: ["063", "063.db"]
Sep 22 23:15:06.750 TRCE incoming request, uri: /newextent/99/data, method: GET, req_id: 199b8185-6db8-4c26-b4ee-72c0a4add789, remote_addr: 127.0.0.1:52616, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.751 INFO request completed, latency_us: 315, response_code: 200, uri: /newextent/99/data, method: GET, req_id: 199b8185-6db8-4c26-b4ee-72c0a4add789, remote_addr: 127.0.0.1:52616, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.756 TRCE incoming request, uri: /newextent/99/db, method: GET, req_id: 1c8bb198-56ef-4157-8d00-545923156001, remote_addr: 127.0.0.1:52616, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.756 INFO request completed, latency_us: 288, response_code: 200, uri: /newextent/99/db, method: GET, req_id: 1c8bb198-56ef-4157-8d00-545923156001, remote_addr: 127.0.0.1:52616, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.757 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/063.copy" to "/tmp/downstairs-vrx8aK6L/00/000/063.replace"
Sep 22 23:15:06.757 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:06.758 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/063.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:06.758 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/063"
Sep 22 23:15:06.758 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/063.db"
Sep 22 23:15:06.758 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:06.758 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/063.replace" to "/tmp/downstairs-vrx8aK6L/00/000/063.completed"
Sep 22 23:15:06.758 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:06.758 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:06.758 DEBG [0] It's time to notify for 158
Sep 22 23:15:06.759 INFO Completion from [0] id:158 status:true
Sep 22 23:15:06.759 INFO [159/752] Repair commands completed
Sep 22 23:15:06.759 INFO Pop front: ReconcileIO { id: ReconciliationId(159), op: ExtentReopen { repair_id: ReconciliationId(159), extent_id: 99 }, state: ClientData([New, New, New]) }
Sep 22 23:15:06.759 INFO Sent repair work, now wait for resp
Sep 22 23:15:06.759 INFO [0] received reconcile message
Sep 22 23:15:06.759 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(159), op: ExtentReopen { repair_id: ReconciliationId(159), extent_id: 99 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:06.759 INFO [0] client ExtentReopen { repair_id: ReconciliationId(159), extent_id: 99 }
Sep 22 23:15:06.759 INFO [1] received reconcile message
Sep 22 23:15:06.759 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(159), op: ExtentReopen { repair_id: ReconciliationId(159), extent_id: 99 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:06.759 INFO [1] client ExtentReopen { repair_id: ReconciliationId(159), extent_id: 99 }
Sep 22 23:15:06.759 INFO [2] received reconcile message
Sep 22 23:15:06.759 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(159), op: ExtentReopen { repair_id: ReconciliationId(159), extent_id: 99 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:06.759 INFO [2] client ExtentReopen { repair_id: ReconciliationId(159), extent_id: 99 }
Sep 22 23:15:06.759 DEBG 159 Reopen extent 99
Sep 22 23:15:06.760 DEBG 159 Reopen extent 99
Sep 22 23:15:06.760 DEBG 159 Reopen extent 99
Sep 22 23:15:06.761 DEBG [2] It's time to notify for 159
Sep 22 23:15:06.761 INFO Completion from [2] id:159 status:true
Sep 22 23:15:06.761 INFO [160/752] Repair commands completed
Sep 22 23:15:06.761 INFO Pop front: ReconcileIO { id: ReconciliationId(160), op: ExtentFlush { repair_id: ReconciliationId(160), extent_id: 110, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:06.761 INFO Sent repair work, now wait for resp
Sep 22 23:15:06.761 INFO [0] received reconcile message
Sep 22 23:15:06.761 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(160), op: ExtentFlush { repair_id: ReconciliationId(160), extent_id: 110, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:06.761 INFO [0] client ExtentFlush { repair_id: ReconciliationId(160), extent_id: 110, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:06.761 INFO [1] received reconcile message
Sep 22 23:15:06.761 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(160), op: ExtentFlush { repair_id: ReconciliationId(160), extent_id: 110, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:06.761 INFO [1] client ExtentFlush { repair_id: ReconciliationId(160), extent_id: 110, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:06.761 INFO [2] received reconcile message
Sep 22 23:15:06.761 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(160), op: ExtentFlush { repair_id: ReconciliationId(160), extent_id: 110, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:06.761 INFO [2] client ExtentFlush { repair_id: ReconciliationId(160), extent_id: 110, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:06.761 DEBG 160 Flush extent 110 with f:2 g:2
Sep 22 23:15:06.761 DEBG Flush just extent 110 with f:2 and g:2
Sep 22 23:15:06.761 DEBG [1] It's time to notify for 160
Sep 22 23:15:06.761 INFO Completion from [1] id:160 status:true
Sep 22 23:15:06.761 INFO [161/752] Repair commands completed
Sep 22 23:15:06.761 INFO Pop front: ReconcileIO { id: ReconciliationId(161), op: ExtentClose { repair_id: ReconciliationId(161), extent_id: 110 }, state: ClientData([New, New, New]) }
Sep 22 23:15:06.761 INFO Sent repair work, now wait for resp
Sep 22 23:15:06.761 INFO [0] received reconcile message
Sep 22 23:15:06.761 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(161), op: ExtentClose { repair_id: ReconciliationId(161), extent_id: 110 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:06.761 INFO [0] client ExtentClose { repair_id: ReconciliationId(161), extent_id: 110 }
Sep 22 23:15:06.761 INFO [1] received reconcile message
Sep 22 23:15:06.761 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(161), op: ExtentClose { repair_id: ReconciliationId(161), extent_id: 110 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:06.761 INFO [1] client ExtentClose { repair_id: ReconciliationId(161), extent_id: 110 }
Sep 22 23:15:06.762 INFO [2] received reconcile message
Sep 22 23:15:06.762 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(161), op: ExtentClose { repair_id: ReconciliationId(161), extent_id: 110 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:06.762 INFO [2] client ExtentClose { repair_id: ReconciliationId(161), extent_id: 110 }
Sep 22 23:15:06.762 DEBG 161 Close extent 110
Sep 22 23:15:06.762 DEBG 161 Close extent 110
Sep 22 23:15:06.762 DEBG 161 Close extent 110
Sep 22 23:15:06.763 DEBG [2] It's time to notify for 161
Sep 22 23:15:06.763 INFO Completion from [2] id:161 status:true
Sep 22 23:15:06.763 INFO [162/752] Repair commands completed
Sep 22 23:15:06.763 INFO Pop front: ReconcileIO { id: ReconciliationId(162), op: ExtentRepair { repair_id: ReconciliationId(162), extent_id: 110, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:06.763 INFO Sent repair work, now wait for resp
Sep 22 23:15:06.763 INFO [0] received reconcile message
Sep 22 23:15:06.763 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(162), op: ExtentRepair { repair_id: ReconciliationId(162), extent_id: 110, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:06.763 INFO [0] client ExtentRepair { repair_id: ReconciliationId(162), extent_id: 110, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:06.763 INFO [0] Sending repair request ReconciliationId(162)
Sep 22 23:15:06.763 INFO [1] received reconcile message
Sep 22 23:15:06.763 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(162), op: ExtentRepair { repair_id: ReconciliationId(162), extent_id: 110, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:06.763 INFO [1] client ExtentRepair { repair_id: ReconciliationId(162), extent_id: 110, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:06.763 INFO [1] No action required ReconciliationId(162)
Sep 22 23:15:06.763 INFO [2] received reconcile message
Sep 22 23:15:06.763 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(162), op: ExtentRepair { repair_id: ReconciliationId(162), extent_id: 110, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:06.763 INFO [2] client ExtentRepair { repair_id: ReconciliationId(162), extent_id: 110, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:06.763 INFO [2] No action required ReconciliationId(162)
Sep 22 23:15:06.763 DEBG 162 Repair extent 110 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:06.763 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/06E.copy"
Sep 22 23:15:06.776 DEBG IO Read  1078 has deps [JobId(1077)]
Sep 22 23:15:06.827 INFO accepted connection, remote_addr: 127.0.0.1:59853, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.827 TRCE incoming request, uri: /extent/110/files, method: GET, req_id: 385f0e90-4f86-4603-9219-f316826e61e6, remote_addr: 127.0.0.1:59853, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.827 INFO request completed, latency_us: 198, response_code: 200, uri: /extent/110/files, method: GET, req_id: 385f0e90-4f86-4603-9219-f316826e61e6, remote_addr: 127.0.0.1:59853, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.827 INFO eid:110 Found repair files: ["06E", "06E.db"]
Sep 22 23:15:06.828 TRCE incoming request, uri: /newextent/110/data, method: GET, req_id: a4bf169f-0f71-4487-a3c0-14bcca017142, remote_addr: 127.0.0.1:59853, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.828 INFO request completed, latency_us: 303, response_code: 200, uri: /newextent/110/data, method: GET, req_id: a4bf169f-0f71-4487-a3c0-14bcca017142, remote_addr: 127.0.0.1:59853, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.833 TRCE incoming request, uri: /newextent/110/db, method: GET, req_id: ea6d063a-69b1-4e35-a9c9-59e06a3c63ea, remote_addr: 127.0.0.1:59853, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.833 INFO request completed, latency_us: 291, response_code: 200, uri: /newextent/110/db, method: GET, req_id: ea6d063a-69b1-4e35-a9c9-59e06a3c63ea, remote_addr: 127.0.0.1:59853, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.834 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/06E.copy" to "/tmp/downstairs-vrx8aK6L/00/000/06E.replace"
Sep 22 23:15:06.834 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:06.835 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/06E.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:06.835 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/06E"
Sep 22 23:15:06.835 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/06E.db"
Sep 22 23:15:06.835 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:06.835 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/06E.replace" to "/tmp/downstairs-vrx8aK6L/00/000/06E.completed"
Sep 22 23:15:06.835 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:06.836 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:06.836 DEBG [0] It's time to notify for 162
Sep 22 23:15:06.836 INFO Completion from [0] id:162 status:true
Sep 22 23:15:06.836 INFO [163/752] Repair commands completed
Sep 22 23:15:06.836 INFO Pop front: ReconcileIO { id: ReconciliationId(163), op: ExtentReopen { repair_id: ReconciliationId(163), extent_id: 110 }, state: ClientData([New, New, New]) }
Sep 22 23:15:06.836 INFO Sent repair work, now wait for resp
Sep 22 23:15:06.836 INFO [0] received reconcile message
Sep 22 23:15:06.836 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(163), op: ExtentReopen { repair_id: ReconciliationId(163), extent_id: 110 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:06.836 INFO [0] client ExtentReopen { repair_id: ReconciliationId(163), extent_id: 110 }
Sep 22 23:15:06.836 INFO [1] received reconcile message
Sep 22 23:15:06.836 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(163), op: ExtentReopen { repair_id: ReconciliationId(163), extent_id: 110 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:06.836 INFO [1] client ExtentReopen { repair_id: ReconciliationId(163), extent_id: 110 }
Sep 22 23:15:06.836 INFO [2] received reconcile message
Sep 22 23:15:06.836 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(163), op: ExtentReopen { repair_id: ReconciliationId(163), extent_id: 110 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:06.836 INFO [2] client ExtentReopen { repair_id: ReconciliationId(163), extent_id: 110 }
Sep 22 23:15:06.836 DEBG 163 Reopen extent 110
Sep 22 23:15:06.837 DEBG 163 Reopen extent 110
Sep 22 23:15:06.837 DEBG 163 Reopen extent 110
Sep 22 23:15:06.838 DEBG [2] It's time to notify for 163
Sep 22 23:15:06.838 INFO Completion from [2] id:163 status:true
Sep 22 23:15:06.838 INFO [164/752] Repair commands completed
Sep 22 23:15:06.838 INFO Pop front: ReconcileIO { id: ReconciliationId(164), op: ExtentFlush { repair_id: ReconciliationId(164), extent_id: 121, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:06.838 INFO Sent repair work, now wait for resp
Sep 22 23:15:06.838 INFO [0] received reconcile message
Sep 22 23:15:06.838 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(164), op: ExtentFlush { repair_id: ReconciliationId(164), extent_id: 121, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:06.838 INFO [0] client ExtentFlush { repair_id: ReconciliationId(164), extent_id: 121, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:06.838 INFO [1] received reconcile message
Sep 22 23:15:06.838 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(164), op: ExtentFlush { repair_id: ReconciliationId(164), extent_id: 121, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:06.838 INFO [1] client ExtentFlush { repair_id: ReconciliationId(164), extent_id: 121, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:06.838 INFO [2] received reconcile message
Sep 22 23:15:06.838 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(164), op: ExtentFlush { repair_id: ReconciliationId(164), extent_id: 121, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:06.838 INFO [2] client ExtentFlush { repair_id: ReconciliationId(164), extent_id: 121, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:06.838 DEBG 164 Flush extent 121 with f:2 g:2
Sep 22 23:15:06.838 DEBG Flush just extent 121 with f:2 and g:2
Sep 22 23:15:06.839 DEBG [1] It's time to notify for 164
Sep 22 23:15:06.839 INFO Completion from [1] id:164 status:true
Sep 22 23:15:06.839 INFO [165/752] Repair commands completed
Sep 22 23:15:06.839 INFO Pop front: ReconcileIO { id: ReconciliationId(165), op: ExtentClose { repair_id: ReconciliationId(165), extent_id: 121 }, state: ClientData([New, New, New]) }
Sep 22 23:15:06.839 INFO Sent repair work, now wait for resp
Sep 22 23:15:06.839 INFO [0] received reconcile message
Sep 22 23:15:06.839 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(165), op: ExtentClose { repair_id: ReconciliationId(165), extent_id: 121 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:06.839 INFO [0] client ExtentClose { repair_id: ReconciliationId(165), extent_id: 121 }
Sep 22 23:15:06.839 INFO [1] received reconcile message
Sep 22 23:15:06.839 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(165), op: ExtentClose { repair_id: ReconciliationId(165), extent_id: 121 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:06.839 INFO [1] client ExtentClose { repair_id: ReconciliationId(165), extent_id: 121 }
Sep 22 23:15:06.839 INFO [2] received reconcile message
Sep 22 23:15:06.839 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(165), op: ExtentClose { repair_id: ReconciliationId(165), extent_id: 121 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:06.839 INFO [2] client ExtentClose { repair_id: ReconciliationId(165), extent_id: 121 }
Sep 22 23:15:06.839 DEBG 165 Close extent 121
Sep 22 23:15:06.839 DEBG 165 Close extent 121
Sep 22 23:15:06.840 DEBG 165 Close extent 121
Sep 22 23:15:06.840 DEBG [2] It's time to notify for 165
Sep 22 23:15:06.840 INFO Completion from [2] id:165 status:true
Sep 22 23:15:06.840 INFO [166/752] Repair commands completed
Sep 22 23:15:06.840 INFO Pop front: ReconcileIO { id: ReconciliationId(166), op: ExtentRepair { repair_id: ReconciliationId(166), extent_id: 121, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:06.840 INFO Sent repair work, now wait for resp
Sep 22 23:15:06.840 INFO [0] received reconcile message
Sep 22 23:15:06.840 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(166), op: ExtentRepair { repair_id: ReconciliationId(166), extent_id: 121, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:06.840 INFO [0] client ExtentRepair { repair_id: ReconciliationId(166), extent_id: 121, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:06.840 INFO [0] Sending repair request ReconciliationId(166)
Sep 22 23:15:06.840 INFO [1] received reconcile message
Sep 22 23:15:06.840 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(166), op: ExtentRepair { repair_id: ReconciliationId(166), extent_id: 121, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:06.840 INFO [1] client ExtentRepair { repair_id: ReconciliationId(166), extent_id: 121, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:06.840 INFO [1] No action required ReconciliationId(166)
Sep 22 23:15:06.840 INFO [2] received reconcile message
Sep 22 23:15:06.840 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(166), op: ExtentRepair { repair_id: ReconciliationId(166), extent_id: 121, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:06.840 INFO [2] client ExtentRepair { repair_id: ReconciliationId(166), extent_id: 121, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:06.840 INFO [2] No action required ReconciliationId(166)
Sep 22 23:15:06.840 DEBG 166 Repair extent 121 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:06.841 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/079.copy"
Sep 22 23:15:06.905 INFO accepted connection, remote_addr: 127.0.0.1:41663, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.905 TRCE incoming request, uri: /extent/121/files, method: GET, req_id: 08c5172a-08f8-4385-a78b-23ad62c90619, remote_addr: 127.0.0.1:41663, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.905 INFO request completed, latency_us: 213, response_code: 200, uri: /extent/121/files, method: GET, req_id: 08c5172a-08f8-4385-a78b-23ad62c90619, remote_addr: 127.0.0.1:41663, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.906 INFO eid:121 Found repair files: ["079", "079.db"]
Sep 22 23:15:06.906 TRCE incoming request, uri: /newextent/121/data, method: GET, req_id: 61c8684b-cf99-40a4-9c70-50e247ff89b3, remote_addr: 127.0.0.1:41663, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.906 INFO request completed, latency_us: 263, response_code: 200, uri: /newextent/121/data, method: GET, req_id: 61c8684b-cf99-40a4-9c70-50e247ff89b3, remote_addr: 127.0.0.1:41663, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.912 TRCE incoming request, uri: /newextent/121/db, method: GET, req_id: 425c9b04-a2a5-495e-8d62-5c097744efc0, remote_addr: 127.0.0.1:41663, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.912 INFO request completed, latency_us: 294, response_code: 200, uri: /newextent/121/db, method: GET, req_id: 425c9b04-a2a5-495e-8d62-5c097744efc0, remote_addr: 127.0.0.1:41663, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.913 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/079.copy" to "/tmp/downstairs-vrx8aK6L/00/000/079.replace"
Sep 22 23:15:06.913 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:06.914 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/079.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:06.914 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/079"
Sep 22 23:15:06.914 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/079.db"
Sep 22 23:15:06.914 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:06.914 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/079.replace" to "/tmp/downstairs-vrx8aK6L/00/000/079.completed"
Sep 22 23:15:06.914 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:06.914 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:06.914 DEBG [0] It's time to notify for 166
Sep 22 23:15:06.915 INFO Completion from [0] id:166 status:true
Sep 22 23:15:06.915 INFO [167/752] Repair commands completed
Sep 22 23:15:06.915 INFO Pop front: ReconcileIO { id: ReconciliationId(167), op: ExtentReopen { repair_id: ReconciliationId(167), extent_id: 121 }, state: ClientData([New, New, New]) }
Sep 22 23:15:06.915 INFO Sent repair work, now wait for resp
Sep 22 23:15:06.915 INFO [0] received reconcile message
Sep 22 23:15:06.915 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(167), op: ExtentReopen { repair_id: ReconciliationId(167), extent_id: 121 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:06.915 INFO [0] client ExtentReopen { repair_id: ReconciliationId(167), extent_id: 121 }
Sep 22 23:15:06.915 INFO [1] received reconcile message
Sep 22 23:15:06.915 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(167), op: ExtentReopen { repair_id: ReconciliationId(167), extent_id: 121 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:06.915 INFO [1] client ExtentReopen { repair_id: ReconciliationId(167), extent_id: 121 }
Sep 22 23:15:06.915 INFO [2] received reconcile message
Sep 22 23:15:06.915 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(167), op: ExtentReopen { repair_id: ReconciliationId(167), extent_id: 121 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:06.915 INFO [2] client ExtentReopen { repair_id: ReconciliationId(167), extent_id: 121 }
Sep 22 23:15:06.915 DEBG 167 Reopen extent 121
Sep 22 23:15:06.916 DEBG 167 Reopen extent 121
Sep 22 23:15:06.916 DEBG 167 Reopen extent 121
Sep 22 23:15:06.917 DEBG [2] It's time to notify for 167
Sep 22 23:15:06.917 INFO Completion from [2] id:167 status:true
Sep 22 23:15:06.917 INFO [168/752] Repair commands completed
Sep 22 23:15:06.917 INFO Pop front: ReconcileIO { id: ReconciliationId(168), op: ExtentFlush { repair_id: ReconciliationId(168), extent_id: 157, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:06.917 INFO Sent repair work, now wait for resp
Sep 22 23:15:06.917 INFO [0] received reconcile message
Sep 22 23:15:06.917 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(168), op: ExtentFlush { repair_id: ReconciliationId(168), extent_id: 157, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:06.917 INFO [0] client ExtentFlush { repair_id: ReconciliationId(168), extent_id: 157, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:06.917 INFO [1] received reconcile message
Sep 22 23:15:06.917 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(168), op: ExtentFlush { repair_id: ReconciliationId(168), extent_id: 157, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:06.917 INFO [1] client ExtentFlush { repair_id: ReconciliationId(168), extent_id: 157, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:06.917 INFO [2] received reconcile message
Sep 22 23:15:06.917 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(168), op: ExtentFlush { repair_id: ReconciliationId(168), extent_id: 157, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:06.917 INFO [2] client ExtentFlush { repair_id: ReconciliationId(168), extent_id: 157, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:06.917 DEBG 168 Flush extent 157 with f:2 g:2
Sep 22 23:15:06.917 DEBG Flush just extent 157 with f:2 and g:2
Sep 22 23:15:06.917 DEBG [1] It's time to notify for 168
Sep 22 23:15:06.917 INFO Completion from [1] id:168 status:true
Sep 22 23:15:06.917 INFO [169/752] Repair commands completed
Sep 22 23:15:06.917 INFO Pop front: ReconcileIO { id: ReconciliationId(169), op: ExtentClose { repair_id: ReconciliationId(169), extent_id: 157 }, state: ClientData([New, New, New]) }
Sep 22 23:15:06.917 INFO Sent repair work, now wait for resp
Sep 22 23:15:06.917 INFO [0] received reconcile message
Sep 22 23:15:06.917 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(169), op: ExtentClose { repair_id: ReconciliationId(169), extent_id: 157 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:06.917 INFO [0] client ExtentClose { repair_id: ReconciliationId(169), extent_id: 157 }
Sep 22 23:15:06.918 INFO [1] received reconcile message
Sep 22 23:15:06.918 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(169), op: ExtentClose { repair_id: ReconciliationId(169), extent_id: 157 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:06.918 INFO [1] client ExtentClose { repair_id: ReconciliationId(169), extent_id: 157 }
Sep 22 23:15:06.918 INFO [2] received reconcile message
Sep 22 23:15:06.918 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(169), op: ExtentClose { repair_id: ReconciliationId(169), extent_id: 157 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:06.918 INFO [2] client ExtentClose { repair_id: ReconciliationId(169), extent_id: 157 }
Sep 22 23:15:06.918 DEBG 169 Close extent 157
Sep 22 23:15:06.918 DEBG 169 Close extent 157
Sep 22 23:15:06.918 DEBG 169 Close extent 157
Sep 22 23:15:06.919 DEBG [2] It's time to notify for 169
Sep 22 23:15:06.919 INFO Completion from [2] id:169 status:true
Sep 22 23:15:06.919 INFO [170/752] Repair commands completed
Sep 22 23:15:06.919 INFO Pop front: ReconcileIO { id: ReconciliationId(170), op: ExtentRepair { repair_id: ReconciliationId(170), extent_id: 157, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:06.919 INFO Sent repair work, now wait for resp
Sep 22 23:15:06.919 INFO [0] received reconcile message
Sep 22 23:15:06.919 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(170), op: ExtentRepair { repair_id: ReconciliationId(170), extent_id: 157, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:06.919 INFO [0] client ExtentRepair { repair_id: ReconciliationId(170), extent_id: 157, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:06.919 INFO [0] Sending repair request ReconciliationId(170)
Sep 22 23:15:06.919 INFO [1] received reconcile message
Sep 22 23:15:06.919 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(170), op: ExtentRepair { repair_id: ReconciliationId(170), extent_id: 157, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:06.919 INFO [1] client ExtentRepair { repair_id: ReconciliationId(170), extent_id: 157, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:06.919 INFO [1] No action required ReconciliationId(170)
Sep 22 23:15:06.919 INFO [2] received reconcile message
Sep 22 23:15:06.919 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(170), op: ExtentRepair { repair_id: ReconciliationId(170), extent_id: 157, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:06.919 INFO [2] client ExtentRepair { repair_id: ReconciliationId(170), extent_id: 157, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:06.919 INFO [2] No action required ReconciliationId(170)
Sep 22 23:15:06.919 DEBG 170 Repair extent 157 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:06.919 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/09D.copy"
Sep 22 23:15:06.983 INFO accepted connection, remote_addr: 127.0.0.1:51237, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.983 TRCE incoming request, uri: /extent/157/files, method: GET, req_id: 57449d40-5f30-430a-a517-823cfe9c5743, remote_addr: 127.0.0.1:51237, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.983 INFO request completed, latency_us: 210, response_code: 200, uri: /extent/157/files, method: GET, req_id: 57449d40-5f30-430a-a517-823cfe9c5743, remote_addr: 127.0.0.1:51237, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.984 INFO eid:157 Found repair files: ["09D", "09D.db"]
Sep 22 23:15:06.984 TRCE incoming request, uri: /newextent/157/data, method: GET, req_id: 33aa0d1a-c9ec-4060-a461-6399cd950aa7, remote_addr: 127.0.0.1:51237, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.984 INFO request completed, latency_us: 257, response_code: 200, uri: /newextent/157/data, method: GET, req_id: 33aa0d1a-c9ec-4060-a461-6399cd950aa7, remote_addr: 127.0.0.1:51237, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.989 TRCE incoming request, uri: /newextent/157/db, method: GET, req_id: c20aeb78-ed74-4e94-b3c7-b784fd4fceda, remote_addr: 127.0.0.1:51237, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.989 INFO request completed, latency_us: 302, response_code: 200, uri: /newextent/157/db, method: GET, req_id: c20aeb78-ed74-4e94-b3c7-b784fd4fceda, remote_addr: 127.0.0.1:51237, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:06.991 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/09D.copy" to "/tmp/downstairs-vrx8aK6L/00/000/09D.replace"
Sep 22 23:15:06.991 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:06.992 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/09D.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:06.992 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/09D"
Sep 22 23:15:06.992 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/09D.db"
Sep 22 23:15:06.992 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:06.992 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/09D.replace" to "/tmp/downstairs-vrx8aK6L/00/000/09D.completed"
Sep 22 23:15:06.992 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:06.992 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:06.992 DEBG [0] It's time to notify for 170
Sep 22 23:15:06.992 INFO Completion from [0] id:170 status:true
Sep 22 23:15:06.992 INFO [171/752] Repair commands completed
Sep 22 23:15:06.992 INFO Pop front: ReconcileIO { id: ReconciliationId(171), op: ExtentReopen { repair_id: ReconciliationId(171), extent_id: 157 }, state: ClientData([New, New, New]) }
Sep 22 23:15:06.992 INFO Sent repair work, now wait for resp
Sep 22 23:15:06.992 INFO [0] received reconcile message
Sep 22 23:15:06.992 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(171), op: ExtentReopen { repair_id: ReconciliationId(171), extent_id: 157 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:06.992 INFO [0] client ExtentReopen { repair_id: ReconciliationId(171), extent_id: 157 }
Sep 22 23:15:06.993 INFO [1] received reconcile message
Sep 22 23:15:06.993 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(171), op: ExtentReopen { repair_id: ReconciliationId(171), extent_id: 157 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:06.993 INFO [1] client ExtentReopen { repair_id: ReconciliationId(171), extent_id: 157 }
Sep 22 23:15:06.993 INFO [2] received reconcile message
Sep 22 23:15:06.993 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(171), op: ExtentReopen { repair_id: ReconciliationId(171), extent_id: 157 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:06.993 INFO [2] client ExtentReopen { repair_id: ReconciliationId(171), extent_id: 157 }
Sep 22 23:15:06.993 DEBG 171 Reopen extent 157
Sep 22 23:15:06.993 DEBG 171 Reopen extent 157
Sep 22 23:15:06.994 DEBG 171 Reopen extent 157
Sep 22 23:15:06.995 DEBG [2] It's time to notify for 171
Sep 22 23:15:06.995 INFO Completion from [2] id:171 status:true
Sep 22 23:15:06.995 INFO [172/752] Repair commands completed
Sep 22 23:15:06.995 INFO Pop front: ReconcileIO { id: ReconciliationId(172), op: ExtentFlush { repair_id: ReconciliationId(172), extent_id: 57, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:06.995 INFO Sent repair work, now wait for resp
Sep 22 23:15:06.995 INFO [0] received reconcile message
Sep 22 23:15:06.995 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(172), op: ExtentFlush { repair_id: ReconciliationId(172), extent_id: 57, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:06.995 INFO [0] client ExtentFlush { repair_id: ReconciliationId(172), extent_id: 57, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:06.995 INFO [1] received reconcile message
Sep 22 23:15:06.995 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(172), op: ExtentFlush { repair_id: ReconciliationId(172), extent_id: 57, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:06.995 INFO [1] client ExtentFlush { repair_id: ReconciliationId(172), extent_id: 57, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:06.995 INFO [2] received reconcile message
Sep 22 23:15:06.995 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(172), op: ExtentFlush { repair_id: ReconciliationId(172), extent_id: 57, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:06.995 INFO [2] client ExtentFlush { repair_id: ReconciliationId(172), extent_id: 57, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:06.995 DEBG 172 Flush extent 57 with f:2 g:2
Sep 22 23:15:06.995 DEBG Flush just extent 57 with f:2 and g:2
Sep 22 23:15:06.995 DEBG [1] It's time to notify for 172
Sep 22 23:15:06.995 INFO Completion from [1] id:172 status:true
Sep 22 23:15:06.995 INFO [173/752] Repair commands completed
Sep 22 23:15:06.995 INFO Pop front: ReconcileIO { id: ReconciliationId(173), op: ExtentClose { repair_id: ReconciliationId(173), extent_id: 57 }, state: ClientData([New, New, New]) }
Sep 22 23:15:06.995 INFO Sent repair work, now wait for resp
Sep 22 23:15:06.995 INFO [0] received reconcile message
Sep 22 23:15:06.995 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(173), op: ExtentClose { repair_id: ReconciliationId(173), extent_id: 57 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:06.995 INFO [0] client ExtentClose { repair_id: ReconciliationId(173), extent_id: 57 }
Sep 22 23:15:06.995 INFO [1] received reconcile message
Sep 22 23:15:06.995 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(173), op: ExtentClose { repair_id: ReconciliationId(173), extent_id: 57 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:06.995 INFO [1] client ExtentClose { repair_id: ReconciliationId(173), extent_id: 57 }
Sep 22 23:15:06.995 INFO [2] received reconcile message
Sep 22 23:15:06.995 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(173), op: ExtentClose { repair_id: ReconciliationId(173), extent_id: 57 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:06.995 INFO [2] client ExtentClose { repair_id: ReconciliationId(173), extent_id: 57 }
Sep 22 23:15:06.996 DEBG 173 Close extent 57
Sep 22 23:15:06.996 DEBG 173 Close extent 57
Sep 22 23:15:06.996 DEBG 173 Close extent 57
Sep 22 23:15:06.996 DEBG [2] It's time to notify for 173
Sep 22 23:15:06.997 INFO Completion from [2] id:173 status:true
Sep 22 23:15:06.997 INFO [174/752] Repair commands completed
Sep 22 23:15:06.997 INFO Pop front: ReconcileIO { id: ReconciliationId(174), op: ExtentRepair { repair_id: ReconciliationId(174), extent_id: 57, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:06.997 INFO Sent repair work, now wait for resp
Sep 22 23:15:06.997 INFO [0] received reconcile message
Sep 22 23:15:06.997 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(174), op: ExtentRepair { repair_id: ReconciliationId(174), extent_id: 57, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:06.997 INFO [0] client ExtentRepair { repair_id: ReconciliationId(174), extent_id: 57, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:06.997 INFO [0] Sending repair request ReconciliationId(174)
Sep 22 23:15:06.997 INFO [1] received reconcile message
Sep 22 23:15:06.997 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(174), op: ExtentRepair { repair_id: ReconciliationId(174), extent_id: 57, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:06.997 INFO [1] client ExtentRepair { repair_id: ReconciliationId(174), extent_id: 57, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:06.997 INFO [1] No action required ReconciliationId(174)
Sep 22 23:15:06.997 INFO [2] received reconcile message
Sep 22 23:15:06.997 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(174), op: ExtentRepair { repair_id: ReconciliationId(174), extent_id: 57, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:06.997 INFO [2] client ExtentRepair { repair_id: ReconciliationId(174), extent_id: 57, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:06.997 INFO [2] No action required ReconciliationId(174)
Sep 22 23:15:06.997 DEBG 174 Repair extent 57 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:06.997 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/039.copy"
Sep 22 23:15:07.061 INFO accepted connection, remote_addr: 127.0.0.1:51549, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.062 TRCE incoming request, uri: /extent/57/files, method: GET, req_id: 1bbf7f2f-ca70-4b4a-9ad8-cd82197f11cc, remote_addr: 127.0.0.1:51549, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.062 INFO request completed, latency_us: 253, response_code: 200, uri: /extent/57/files, method: GET, req_id: 1bbf7f2f-ca70-4b4a-9ad8-cd82197f11cc, remote_addr: 127.0.0.1:51549, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.062 INFO eid:57 Found repair files: ["039", "039.db"]
Sep 22 23:15:07.063 TRCE incoming request, uri: /newextent/57/data, method: GET, req_id: 145e4961-4452-4bea-b77c-b94a12da60cf, remote_addr: 127.0.0.1:51549, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.063 INFO request completed, latency_us: 336, response_code: 200, uri: /newextent/57/data, method: GET, req_id: 145e4961-4452-4bea-b77c-b94a12da60cf, remote_addr: 127.0.0.1:51549, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.068 TRCE incoming request, uri: /newextent/57/db, method: GET, req_id: 6285ffce-be82-4c28-ae47-f064788a27cd, remote_addr: 127.0.0.1:51549, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.068 INFO request completed, latency_us: 292, response_code: 200, uri: /newextent/57/db, method: GET, req_id: 6285ffce-be82-4c28-ae47-f064788a27cd, remote_addr: 127.0.0.1:51549, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.069 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/039.copy" to "/tmp/downstairs-vrx8aK6L/00/000/039.replace"
Sep 22 23:15:07.069 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:07.070 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/039.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:07.070 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/039"
Sep 22 23:15:07.070 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/039.db"
Sep 22 23:15:07.070 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:07.070 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/039.replace" to "/tmp/downstairs-vrx8aK6L/00/000/039.completed"
Sep 22 23:15:07.070 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:07.070 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:07.071 DEBG [0] It's time to notify for 174
Sep 22 23:15:07.071 INFO Completion from [0] id:174 status:true
Sep 22 23:15:07.071 INFO [175/752] Repair commands completed
Sep 22 23:15:07.071 INFO Pop front: ReconcileIO { id: ReconciliationId(175), op: ExtentReopen { repair_id: ReconciliationId(175), extent_id: 57 }, state: ClientData([New, New, New]) }
Sep 22 23:15:07.071 INFO Sent repair work, now wait for resp
Sep 22 23:15:07.071 INFO [0] received reconcile message
Sep 22 23:15:07.071 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(175), op: ExtentReopen { repair_id: ReconciliationId(175), extent_id: 57 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:07.071 INFO [0] client ExtentReopen { repair_id: ReconciliationId(175), extent_id: 57 }
Sep 22 23:15:07.071 INFO [1] received reconcile message
Sep 22 23:15:07.071 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(175), op: ExtentReopen { repair_id: ReconciliationId(175), extent_id: 57 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:07.071 INFO [1] client ExtentReopen { repair_id: ReconciliationId(175), extent_id: 57 }
Sep 22 23:15:07.071 INFO [2] received reconcile message
Sep 22 23:15:07.071 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(175), op: ExtentReopen { repair_id: ReconciliationId(175), extent_id: 57 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:07.071 INFO [2] client ExtentReopen { repair_id: ReconciliationId(175), extent_id: 57 }
Sep 22 23:15:07.071 DEBG 175 Reopen extent 57
Sep 22 23:15:07.072 DEBG 175 Reopen extent 57
Sep 22 23:15:07.072 DEBG 175 Reopen extent 57
Sep 22 23:15:07.073 DEBG [2] It's time to notify for 175
Sep 22 23:15:07.073 INFO Completion from [2] id:175 status:true
Sep 22 23:15:07.073 INFO [176/752] Repair commands completed
Sep 22 23:15:07.073 INFO Pop front: ReconcileIO { id: ReconciliationId(176), op: ExtentFlush { repair_id: ReconciliationId(176), extent_id: 32, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:07.073 INFO Sent repair work, now wait for resp
Sep 22 23:15:07.073 INFO [0] received reconcile message
Sep 22 23:15:07.073 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(176), op: ExtentFlush { repair_id: ReconciliationId(176), extent_id: 32, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:07.073 INFO [0] client ExtentFlush { repair_id: ReconciliationId(176), extent_id: 32, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:07.073 INFO [1] received reconcile message
Sep 22 23:15:07.073 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(176), op: ExtentFlush { repair_id: ReconciliationId(176), extent_id: 32, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:07.073 INFO [1] client ExtentFlush { repair_id: ReconciliationId(176), extent_id: 32, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:07.073 INFO [2] received reconcile message
Sep 22 23:15:07.073 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(176), op: ExtentFlush { repair_id: ReconciliationId(176), extent_id: 32, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:07.073 INFO [2] client ExtentFlush { repair_id: ReconciliationId(176), extent_id: 32, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:07.073 DEBG 176 Flush extent 32 with f:2 g:2
Sep 22 23:15:07.073 DEBG Flush just extent 32 with f:2 and g:2
Sep 22 23:15:07.074 DEBG [1] It's time to notify for 176
Sep 22 23:15:07.074 INFO Completion from [1] id:176 status:true
Sep 22 23:15:07.074 INFO [177/752] Repair commands completed
Sep 22 23:15:07.074 INFO Pop front: ReconcileIO { id: ReconciliationId(177), op: ExtentClose { repair_id: ReconciliationId(177), extent_id: 32 }, state: ClientData([New, New, New]) }
Sep 22 23:15:07.074 INFO Sent repair work, now wait for resp
Sep 22 23:15:07.074 INFO [0] received reconcile message
Sep 22 23:15:07.074 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(177), op: ExtentClose { repair_id: ReconciliationId(177), extent_id: 32 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:07.074 INFO [0] client ExtentClose { repair_id: ReconciliationId(177), extent_id: 32 }
Sep 22 23:15:07.074 INFO [1] received reconcile message
Sep 22 23:15:07.074 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(177), op: ExtentClose { repair_id: ReconciliationId(177), extent_id: 32 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:07.074 INFO [1] client ExtentClose { repair_id: ReconciliationId(177), extent_id: 32 }
Sep 22 23:15:07.074 INFO [2] received reconcile message
Sep 22 23:15:07.074 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(177), op: ExtentClose { repair_id: ReconciliationId(177), extent_id: 32 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:07.074 INFO [2] client ExtentClose { repair_id: ReconciliationId(177), extent_id: 32 }
Sep 22 23:15:07.074 DEBG 177 Close extent 32
Sep 22 23:15:07.074 DEBG 177 Close extent 32
Sep 22 23:15:07.075 DEBG 177 Close extent 32
Sep 22 23:15:07.075 DEBG [2] It's time to notify for 177
Sep 22 23:15:07.075 INFO Completion from [2] id:177 status:true
Sep 22 23:15:07.075 INFO [178/752] Repair commands completed
Sep 22 23:15:07.075 INFO Pop front: ReconcileIO { id: ReconciliationId(178), op: ExtentRepair { repair_id: ReconciliationId(178), extent_id: 32, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:07.075 INFO Sent repair work, now wait for resp
Sep 22 23:15:07.075 INFO [0] received reconcile message
Sep 22 23:15:07.075 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(178), op: ExtentRepair { repair_id: ReconciliationId(178), extent_id: 32, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:07.075 INFO [0] client ExtentRepair { repair_id: ReconciliationId(178), extent_id: 32, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:07.075 INFO [0] Sending repair request ReconciliationId(178)
Sep 22 23:15:07.075 INFO [1] received reconcile message
Sep 22 23:15:07.075 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(178), op: ExtentRepair { repair_id: ReconciliationId(178), extent_id: 32, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:07.075 INFO [1] client ExtentRepair { repair_id: ReconciliationId(178), extent_id: 32, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:07.075 INFO [1] No action required ReconciliationId(178)
Sep 22 23:15:07.075 INFO [2] received reconcile message
Sep 22 23:15:07.075 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(178), op: ExtentRepair { repair_id: ReconciliationId(178), extent_id: 32, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:07.075 INFO [2] client ExtentRepair { repair_id: ReconciliationId(178), extent_id: 32, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:07.075 INFO [2] No action required ReconciliationId(178)
Sep 22 23:15:07.075 DEBG 178 Repair extent 32 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:07.076 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/020.copy"
Sep 22 23:15:07.137 INFO accepted connection, remote_addr: 127.0.0.1:62704, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.137 TRCE incoming request, uri: /extent/32/files, method: GET, req_id: c3686191-e1cd-42af-91be-659bfa4e7518, remote_addr: 127.0.0.1:62704, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.138 INFO request completed, latency_us: 243, response_code: 200, uri: /extent/32/files, method: GET, req_id: c3686191-e1cd-42af-91be-659bfa4e7518, remote_addr: 127.0.0.1:62704, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.138 INFO eid:32 Found repair files: ["020", "020.db"]
Sep 22 23:15:07.138 TRCE incoming request, uri: /newextent/32/data, method: GET, req_id: 37b947fc-bc57-4f99-98b4-7730b6dfc953, remote_addr: 127.0.0.1:62704, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.139 INFO request completed, latency_us: 345, response_code: 200, uri: /newextent/32/data, method: GET, req_id: 37b947fc-bc57-4f99-98b4-7730b6dfc953, remote_addr: 127.0.0.1:62704, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.144 TRCE incoming request, uri: /newextent/32/db, method: GET, req_id: 62eac362-1dae-441b-84bd-df708f6f9424, remote_addr: 127.0.0.1:62704, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.144 INFO request completed, latency_us: 290, response_code: 200, uri: /newextent/32/db, method: GET, req_id: 62eac362-1dae-441b-84bd-df708f6f9424, remote_addr: 127.0.0.1:62704, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.145 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/020.copy" to "/tmp/downstairs-vrx8aK6L/00/000/020.replace"
Sep 22 23:15:07.145 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:07.146 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/020.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:07.146 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/020"
Sep 22 23:15:07.147 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/020.db"
Sep 22 23:15:07.147 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:07.147 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/020.replace" to "/tmp/downstairs-vrx8aK6L/00/000/020.completed"
Sep 22 23:15:07.147 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:07.147 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:07.147 DEBG [0] It's time to notify for 178
Sep 22 23:15:07.147 INFO Completion from [0] id:178 status:true
Sep 22 23:15:07.147 INFO [179/752] Repair commands completed
Sep 22 23:15:07.147 INFO Pop front: ReconcileIO { id: ReconciliationId(179), op: ExtentReopen { repair_id: ReconciliationId(179), extent_id: 32 }, state: ClientData([New, New, New]) }
Sep 22 23:15:07.147 INFO Sent repair work, now wait for resp
Sep 22 23:15:07.147 INFO [0] received reconcile message
Sep 22 23:15:07.147 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(179), op: ExtentReopen { repair_id: ReconciliationId(179), extent_id: 32 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:07.147 INFO [0] client ExtentReopen { repair_id: ReconciliationId(179), extent_id: 32 }
Sep 22 23:15:07.147 INFO [1] received reconcile message
Sep 22 23:15:07.147 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(179), op: ExtentReopen { repair_id: ReconciliationId(179), extent_id: 32 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:07.147 INFO [1] client ExtentReopen { repair_id: ReconciliationId(179), extent_id: 32 }
Sep 22 23:15:07.147 INFO [2] received reconcile message
Sep 22 23:15:07.147 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(179), op: ExtentReopen { repair_id: ReconciliationId(179), extent_id: 32 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:07.147 INFO [2] client ExtentReopen { repair_id: ReconciliationId(179), extent_id: 32 }
Sep 22 23:15:07.147 DEBG 179 Reopen extent 32
Sep 22 23:15:07.148 DEBG 179 Reopen extent 32
Sep 22 23:15:07.149 DEBG 179 Reopen extent 32
Sep 22 23:15:07.149 DEBG [2] It's time to notify for 179
Sep 22 23:15:07.149 INFO Completion from [2] id:179 status:true
Sep 22 23:15:07.149 INFO [180/752] Repair commands completed
Sep 22 23:15:07.149 INFO Pop front: ReconcileIO { id: ReconciliationId(180), op: ExtentFlush { repair_id: ReconciliationId(180), extent_id: 60, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:07.149 INFO Sent repair work, now wait for resp
Sep 22 23:15:07.149 INFO [0] received reconcile message
Sep 22 23:15:07.149 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(180), op: ExtentFlush { repair_id: ReconciliationId(180), extent_id: 60, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:07.150 INFO [0] client ExtentFlush { repair_id: ReconciliationId(180), extent_id: 60, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:07.150 INFO [1] received reconcile message
Sep 22 23:15:07.150 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(180), op: ExtentFlush { repair_id: ReconciliationId(180), extent_id: 60, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:07.150 INFO [1] client ExtentFlush { repair_id: ReconciliationId(180), extent_id: 60, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:07.150 INFO [2] received reconcile message
Sep 22 23:15:07.150 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(180), op: ExtentFlush { repair_id: ReconciliationId(180), extent_id: 60, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:07.150 INFO [2] client ExtentFlush { repair_id: ReconciliationId(180), extent_id: 60, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:07.150 DEBG 180 Flush extent 60 with f:2 g:2
Sep 22 23:15:07.150 DEBG Flush just extent 60 with f:2 and g:2
Sep 22 23:15:07.150 DEBG [1] It's time to notify for 180
Sep 22 23:15:07.150 INFO Completion from [1] id:180 status:true
Sep 22 23:15:07.150 INFO [181/752] Repair commands completed
Sep 22 23:15:07.150 INFO Pop front: ReconcileIO { id: ReconciliationId(181), op: ExtentClose { repair_id: ReconciliationId(181), extent_id: 60 }, state: ClientData([New, New, New]) }
Sep 22 23:15:07.150 INFO Sent repair work, now wait for resp
Sep 22 23:15:07.150 INFO [0] received reconcile message
Sep 22 23:15:07.150 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(181), op: ExtentClose { repair_id: ReconciliationId(181), extent_id: 60 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:07.150 INFO [0] client ExtentClose { repair_id: ReconciliationId(181), extent_id: 60 }
Sep 22 23:15:07.150 INFO [1] received reconcile message
Sep 22 23:15:07.150 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(181), op: ExtentClose { repair_id: ReconciliationId(181), extent_id: 60 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:07.150 INFO [1] client ExtentClose { repair_id: ReconciliationId(181), extent_id: 60 }
Sep 22 23:15:07.150 INFO [2] received reconcile message
Sep 22 23:15:07.150 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(181), op: ExtentClose { repair_id: ReconciliationId(181), extent_id: 60 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:07.150 INFO [2] client ExtentClose { repair_id: ReconciliationId(181), extent_id: 60 }
Sep 22 23:15:07.150 DEBG 181 Close extent 60
Sep 22 23:15:07.151 DEBG 181 Close extent 60
Sep 22 23:15:07.151 DEBG 181 Close extent 60
Sep 22 23:15:07.151 DEBG [2] It's time to notify for 181
Sep 22 23:15:07.151 INFO Completion from [2] id:181 status:true
Sep 22 23:15:07.151 INFO [182/752] Repair commands completed
Sep 22 23:15:07.151 INFO Pop front: ReconcileIO { id: ReconciliationId(182), op: ExtentRepair { repair_id: ReconciliationId(182), extent_id: 60, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:07.151 INFO Sent repair work, now wait for resp
Sep 22 23:15:07.151 INFO [0] received reconcile message
Sep 22 23:15:07.151 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(182), op: ExtentRepair { repair_id: ReconciliationId(182), extent_id: 60, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:07.152 INFO [0] client ExtentRepair { repair_id: ReconciliationId(182), extent_id: 60, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:07.152 INFO [0] Sending repair request ReconciliationId(182)
Sep 22 23:15:07.152 INFO [1] received reconcile message
Sep 22 23:15:07.152 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(182), op: ExtentRepair { repair_id: ReconciliationId(182), extent_id: 60, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:07.152 INFO [1] client ExtentRepair { repair_id: ReconciliationId(182), extent_id: 60, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:07.152 INFO [1] No action required ReconciliationId(182)
Sep 22 23:15:07.152 INFO [2] received reconcile message
Sep 22 23:15:07.152 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(182), op: ExtentRepair { repair_id: ReconciliationId(182), extent_id: 60, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:07.152 INFO [2] client ExtentRepair { repair_id: ReconciliationId(182), extent_id: 60, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:07.152 INFO [2] No action required ReconciliationId(182)
Sep 22 23:15:07.152 DEBG 182 Repair extent 60 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:07.152 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/03C.copy"
Sep 22 23:15:07.214 INFO accepted connection, remote_addr: 127.0.0.1:50194, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.214 TRCE incoming request, uri: /extent/60/files, method: GET, req_id: 3dc11b21-77bb-4afe-b2ed-9445947b807e, remote_addr: 127.0.0.1:50194, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.214 INFO request completed, latency_us: 224, response_code: 200, uri: /extent/60/files, method: GET, req_id: 3dc11b21-77bb-4afe-b2ed-9445947b807e, remote_addr: 127.0.0.1:50194, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.214 INFO eid:60 Found repair files: ["03C", "03C.db"]
Sep 22 23:15:07.215 TRCE incoming request, uri: /newextent/60/data, method: GET, req_id: 477c6cb1-62c0-427a-83b7-af96b54c3e15, remote_addr: 127.0.0.1:50194, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.215 INFO request completed, latency_us: 321, response_code: 200, uri: /newextent/60/data, method: GET, req_id: 477c6cb1-62c0-427a-83b7-af96b54c3e15, remote_addr: 127.0.0.1:50194, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.220 TRCE incoming request, uri: /newextent/60/db, method: GET, req_id: 4745aaea-af7f-4bfa-8baf-83a3fc8a5512, remote_addr: 127.0.0.1:50194, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.220 INFO request completed, latency_us: 291, response_code: 200, uri: /newextent/60/db, method: GET, req_id: 4745aaea-af7f-4bfa-8baf-83a3fc8a5512, remote_addr: 127.0.0.1:50194, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.221 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/03C.copy" to "/tmp/downstairs-vrx8aK6L/00/000/03C.replace"
Sep 22 23:15:07.222 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:07.222 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/03C.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:07.223 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/03C"
Sep 22 23:15:07.223 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/03C.db"
Sep 22 23:15:07.223 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:07.223 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/03C.replace" to "/tmp/downstairs-vrx8aK6L/00/000/03C.completed"
Sep 22 23:15:07.223 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:07.223 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:07.223 DEBG [0] It's time to notify for 182
Sep 22 23:15:07.223 INFO Completion from [0] id:182 status:true
Sep 22 23:15:07.223 INFO [183/752] Repair commands completed
Sep 22 23:15:07.223 INFO Pop front: ReconcileIO { id: ReconciliationId(183), op: ExtentReopen { repair_id: ReconciliationId(183), extent_id: 60 }, state: ClientData([New, New, New]) }
Sep 22 23:15:07.223 INFO Sent repair work, now wait for resp
Sep 22 23:15:07.223 INFO [0] received reconcile message
Sep 22 23:15:07.223 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(183), op: ExtentReopen { repair_id: ReconciliationId(183), extent_id: 60 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:07.223 INFO [0] client ExtentReopen { repair_id: ReconciliationId(183), extent_id: 60 }
Sep 22 23:15:07.223 INFO [1] received reconcile message
Sep 22 23:15:07.223 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(183), op: ExtentReopen { repair_id: ReconciliationId(183), extent_id: 60 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:07.224 INFO [1] client ExtentReopen { repair_id: ReconciliationId(183), extent_id: 60 }
Sep 22 23:15:07.224 INFO [2] received reconcile message
Sep 22 23:15:07.224 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(183), op: ExtentReopen { repair_id: ReconciliationId(183), extent_id: 60 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:07.224 INFO [2] client ExtentReopen { repair_id: ReconciliationId(183), extent_id: 60 }
Sep 22 23:15:07.224 DEBG 183 Reopen extent 60
Sep 22 23:15:07.224 DEBG 183 Reopen extent 60
Sep 22 23:15:07.225 DEBG 183 Reopen extent 60
Sep 22 23:15:07.225 DEBG [2] It's time to notify for 183
Sep 22 23:15:07.226 INFO Completion from [2] id:183 status:true
Sep 22 23:15:07.226 INFO [184/752] Repair commands completed
Sep 22 23:15:07.226 INFO Pop front: ReconcileIO { id: ReconciliationId(184), op: ExtentFlush { repair_id: ReconciliationId(184), extent_id: 7, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:07.226 INFO Sent repair work, now wait for resp
Sep 22 23:15:07.226 INFO [0] received reconcile message
Sep 22 23:15:07.226 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(184), op: ExtentFlush { repair_id: ReconciliationId(184), extent_id: 7, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:07.226 INFO [0] client ExtentFlush { repair_id: ReconciliationId(184), extent_id: 7, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:07.226 INFO [1] received reconcile message
Sep 22 23:15:07.226 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(184), op: ExtentFlush { repair_id: ReconciliationId(184), extent_id: 7, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:07.226 INFO [1] client ExtentFlush { repair_id: ReconciliationId(184), extent_id: 7, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:07.226 INFO [2] received reconcile message
Sep 22 23:15:07.226 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(184), op: ExtentFlush { repair_id: ReconciliationId(184), extent_id: 7, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:07.226 INFO [2] client ExtentFlush { repair_id: ReconciliationId(184), extent_id: 7, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:07.226 DEBG 184 Flush extent 7 with f:2 g:2
Sep 22 23:15:07.226 DEBG Flush just extent 7 with f:2 and g:2
Sep 22 23:15:07.226 DEBG [1] It's time to notify for 184
Sep 22 23:15:07.226 INFO Completion from [1] id:184 status:true
Sep 22 23:15:07.226 INFO [185/752] Repair commands completed
Sep 22 23:15:07.226 INFO Pop front: ReconcileIO { id: ReconciliationId(185), op: ExtentClose { repair_id: ReconciliationId(185), extent_id: 7 }, state: ClientData([New, New, New]) }
Sep 22 23:15:07.226 INFO Sent repair work, now wait for resp
Sep 22 23:15:07.226 INFO [0] received reconcile message
Sep 22 23:15:07.226 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(185), op: ExtentClose { repair_id: ReconciliationId(185), extent_id: 7 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:07.226 INFO [0] client ExtentClose { repair_id: ReconciliationId(185), extent_id: 7 }
Sep 22 23:15:07.226 INFO [1] received reconcile message
Sep 22 23:15:07.226 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(185), op: ExtentClose { repair_id: ReconciliationId(185), extent_id: 7 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:07.226 INFO [1] client ExtentClose { repair_id: ReconciliationId(185), extent_id: 7 }
Sep 22 23:15:07.226 INFO [2] received reconcile message
Sep 22 23:15:07.226 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(185), op: ExtentClose { repair_id: ReconciliationId(185), extent_id: 7 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:07.226 INFO [2] client ExtentClose { repair_id: ReconciliationId(185), extent_id: 7 }
Sep 22 23:15:07.227 DEBG 185 Close extent 7
Sep 22 23:15:07.227 DEBG 185 Close extent 7
Sep 22 23:15:07.227 DEBG 185 Close extent 7
Sep 22 23:15:07.227 DEBG [2] It's time to notify for 185
Sep 22 23:15:07.228 INFO Completion from [2] id:185 status:true
Sep 22 23:15:07.228 INFO [186/752] Repair commands completed
Sep 22 23:15:07.228 INFO Pop front: ReconcileIO { id: ReconciliationId(186), op: ExtentRepair { repair_id: ReconciliationId(186), extent_id: 7, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:07.228 INFO Sent repair work, now wait for resp
Sep 22 23:15:07.228 INFO [0] received reconcile message
Sep 22 23:15:07.228 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(186), op: ExtentRepair { repair_id: ReconciliationId(186), extent_id: 7, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:07.228 INFO [0] client ExtentRepair { repair_id: ReconciliationId(186), extent_id: 7, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:07.228 INFO [0] Sending repair request ReconciliationId(186)
Sep 22 23:15:07.228 INFO [1] received reconcile message
Sep 22 23:15:07.228 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(186), op: ExtentRepair { repair_id: ReconciliationId(186), extent_id: 7, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:07.228 INFO [1] client ExtentRepair { repair_id: ReconciliationId(186), extent_id: 7, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:07.228 INFO [1] No action required ReconciliationId(186)
Sep 22 23:15:07.228 INFO [2] received reconcile message
Sep 22 23:15:07.228 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(186), op: ExtentRepair { repair_id: ReconciliationId(186), extent_id: 7, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:07.228 INFO [2] client ExtentRepair { repair_id: ReconciliationId(186), extent_id: 7, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:07.228 INFO [2] No action required ReconciliationId(186)
Sep 22 23:15:07.228 DEBG 186 Repair extent 7 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:07.228 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/007.copy"
Sep 22 23:15:07.293 INFO accepted connection, remote_addr: 127.0.0.1:40181, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.293 TRCE incoming request, uri: /extent/7/files, method: GET, req_id: 8fee3a6c-e887-4386-ae7e-725bb42a67f3, remote_addr: 127.0.0.1:40181, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.293 INFO request completed, latency_us: 216, response_code: 200, uri: /extent/7/files, method: GET, req_id: 8fee3a6c-e887-4386-ae7e-725bb42a67f3, remote_addr: 127.0.0.1:40181, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.293 INFO eid:7 Found repair files: ["007", "007.db"]
Sep 22 23:15:07.294 TRCE incoming request, uri: /newextent/7/data, method: GET, req_id: 4be306dd-74f7-4fe4-88a5-b0085100dc07, remote_addr: 127.0.0.1:40181, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.294 INFO request completed, latency_us: 313, response_code: 200, uri: /newextent/7/data, method: GET, req_id: 4be306dd-74f7-4fe4-88a5-b0085100dc07, remote_addr: 127.0.0.1:40181, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.299 TRCE incoming request, uri: /newextent/7/db, method: GET, req_id: 205f8a30-cbd7-454a-a9e9-c2192a511053, remote_addr: 127.0.0.1:40181, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.299 INFO request completed, latency_us: 298, response_code: 200, uri: /newextent/7/db, method: GET, req_id: 205f8a30-cbd7-454a-a9e9-c2192a511053, remote_addr: 127.0.0.1:40181, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.301 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/007.copy" to "/tmp/downstairs-vrx8aK6L/00/000/007.replace"
Sep 22 23:15:07.301 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:07.302 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/007.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:07.302 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/007"
Sep 22 23:15:07.302 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/007.db"
Sep 22 23:15:07.302 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:07.302 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/007.replace" to "/tmp/downstairs-vrx8aK6L/00/000/007.completed"
Sep 22 23:15:07.302 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:07.302 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:07.302 DEBG [0] It's time to notify for 186
Sep 22 23:15:07.302 INFO Completion from [0] id:186 status:true
Sep 22 23:15:07.302 INFO [187/752] Repair commands completed
Sep 22 23:15:07.302 INFO Pop front: ReconcileIO { id: ReconciliationId(187), op: ExtentReopen { repair_id: ReconciliationId(187), extent_id: 7 }, state: ClientData([New, New, New]) }
Sep 22 23:15:07.302 INFO Sent repair work, now wait for resp
Sep 22 23:15:07.302 INFO [0] received reconcile message
Sep 22 23:15:07.302 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(187), op: ExtentReopen { repair_id: ReconciliationId(187), extent_id: 7 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:07.302 INFO [0] client ExtentReopen { repair_id: ReconciliationId(187), extent_id: 7 }
Sep 22 23:15:07.303 INFO [1] received reconcile message
Sep 22 23:15:07.303 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(187), op: ExtentReopen { repair_id: ReconciliationId(187), extent_id: 7 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:07.303 INFO [1] client ExtentReopen { repair_id: ReconciliationId(187), extent_id: 7 }
Sep 22 23:15:07.303 INFO [2] received reconcile message
Sep 22 23:15:07.303 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(187), op: ExtentReopen { repair_id: ReconciliationId(187), extent_id: 7 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:07.303 INFO [2] client ExtentReopen { repair_id: ReconciliationId(187), extent_id: 7 }
Sep 22 23:15:07.303 DEBG 187 Reopen extent 7
Sep 22 23:15:07.303 DEBG 187 Reopen extent 7
Sep 22 23:15:07.304 DEBG 187 Reopen extent 7
Sep 22 23:15:07.305 DEBG [2] It's time to notify for 187
Sep 22 23:15:07.305 INFO Completion from [2] id:187 status:true
Sep 22 23:15:07.305 INFO [188/752] Repair commands completed
Sep 22 23:15:07.305 INFO Pop front: ReconcileIO { id: ReconciliationId(188), op: ExtentFlush { repair_id: ReconciliationId(188), extent_id: 139, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:07.305 INFO Sent repair work, now wait for resp
Sep 22 23:15:07.305 INFO [0] received reconcile message
Sep 22 23:15:07.305 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(188), op: ExtentFlush { repair_id: ReconciliationId(188), extent_id: 139, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:07.305 INFO [0] client ExtentFlush { repair_id: ReconciliationId(188), extent_id: 139, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:07.305 INFO [1] received reconcile message
Sep 22 23:15:07.305 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(188), op: ExtentFlush { repair_id: ReconciliationId(188), extent_id: 139, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:07.305 INFO [1] client ExtentFlush { repair_id: ReconciliationId(188), extent_id: 139, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:07.305 INFO [2] received reconcile message
Sep 22 23:15:07.305 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(188), op: ExtentFlush { repair_id: ReconciliationId(188), extent_id: 139, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:07.305 INFO [2] client ExtentFlush { repair_id: ReconciliationId(188), extent_id: 139, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:07.305 DEBG 188 Flush extent 139 with f:2 g:2
Sep 22 23:15:07.305 DEBG Flush just extent 139 with f:2 and g:2
Sep 22 23:15:07.305 DEBG [1] It's time to notify for 188
Sep 22 23:15:07.305 INFO Completion from [1] id:188 status:true
Sep 22 23:15:07.305 INFO [189/752] Repair commands completed
Sep 22 23:15:07.305 INFO Pop front: ReconcileIO { id: ReconciliationId(189), op: ExtentClose { repair_id: ReconciliationId(189), extent_id: 139 }, state: ClientData([New, New, New]) }
Sep 22 23:15:07.305 INFO Sent repair work, now wait for resp
Sep 22 23:15:07.305 INFO [0] received reconcile message
Sep 22 23:15:07.305 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(189), op: ExtentClose { repair_id: ReconciliationId(189), extent_id: 139 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:07.305 INFO [0] client ExtentClose { repair_id: ReconciliationId(189), extent_id: 139 }
Sep 22 23:15:07.305 INFO [1] received reconcile message
Sep 22 23:15:07.305 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(189), op: ExtentClose { repair_id: ReconciliationId(189), extent_id: 139 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:07.305 INFO [1] client ExtentClose { repair_id: ReconciliationId(189), extent_id: 139 }
Sep 22 23:15:07.305 INFO [2] received reconcile message
Sep 22 23:15:07.305 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(189), op: ExtentClose { repair_id: ReconciliationId(189), extent_id: 139 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:07.305 INFO [2] client ExtentClose { repair_id: ReconciliationId(189), extent_id: 139 }
Sep 22 23:15:07.306 DEBG 189 Close extent 139
Sep 22 23:15:07.306 DEBG 189 Close extent 139
Sep 22 23:15:07.306 DEBG 189 Close extent 139
Sep 22 23:15:07.306 DEBG [2] It's time to notify for 189
Sep 22 23:15:07.307 INFO Completion from [2] id:189 status:true
Sep 22 23:15:07.307 INFO [190/752] Repair commands completed
Sep 22 23:15:07.307 INFO Pop front: ReconcileIO { id: ReconciliationId(190), op: ExtentRepair { repair_id: ReconciliationId(190), extent_id: 139, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:07.307 INFO Sent repair work, now wait for resp
Sep 22 23:15:07.307 INFO [0] received reconcile message
Sep 22 23:15:07.307 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(190), op: ExtentRepair { repair_id: ReconciliationId(190), extent_id: 139, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:07.307 INFO [0] client ExtentRepair { repair_id: ReconciliationId(190), extent_id: 139, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:07.307 INFO [0] Sending repair request ReconciliationId(190)
Sep 22 23:15:07.307 INFO [1] received reconcile message
Sep 22 23:15:07.307 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(190), op: ExtentRepair { repair_id: ReconciliationId(190), extent_id: 139, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:07.307 INFO [1] client ExtentRepair { repair_id: ReconciliationId(190), extent_id: 139, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:07.307 INFO [1] No action required ReconciliationId(190)
Sep 22 23:15:07.307 INFO [2] received reconcile message
Sep 22 23:15:07.307 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(190), op: ExtentRepair { repair_id: ReconciliationId(190), extent_id: 139, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:07.307 INFO [2] client ExtentRepair { repair_id: ReconciliationId(190), extent_id: 139, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:07.307 INFO [2] No action required ReconciliationId(190)
Sep 22 23:15:07.307 DEBG 190 Repair extent 139 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:07.307 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/08B.copy"
Sep 22 23:15:07.370 INFO accepted connection, remote_addr: 127.0.0.1:57559, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.371 TRCE incoming request, uri: /extent/139/files, method: GET, req_id: 4db907e1-aa75-4bbd-a43f-11c76a754ef2, remote_addr: 127.0.0.1:57559, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.371 INFO request completed, latency_us: 230, response_code: 200, uri: /extent/139/files, method: GET, req_id: 4db907e1-aa75-4bbd-a43f-11c76a754ef2, remote_addr: 127.0.0.1:57559, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.371 INFO eid:139 Found repair files: ["08B", "08B.db"]
Sep 22 23:15:07.371 TRCE incoming request, uri: /newextent/139/data, method: GET, req_id: 01fb575e-e85b-4fc4-b43e-2ece49be8b0a, remote_addr: 127.0.0.1:57559, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.372 INFO request completed, latency_us: 340, response_code: 200, uri: /newextent/139/data, method: GET, req_id: 01fb575e-e85b-4fc4-b43e-2ece49be8b0a, remote_addr: 127.0.0.1:57559, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.377 TRCE incoming request, uri: /newextent/139/db, method: GET, req_id: f607117f-4ab6-4c84-b3bf-ed29b62420be, remote_addr: 127.0.0.1:57559, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.377 INFO request completed, latency_us: 290, response_code: 200, uri: /newextent/139/db, method: GET, req_id: f607117f-4ab6-4c84-b3bf-ed29b62420be, remote_addr: 127.0.0.1:57559, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.378 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/08B.copy" to "/tmp/downstairs-vrx8aK6L/00/000/08B.replace"
Sep 22 23:15:07.378 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:07.379 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/08B.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:07.379 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/08B"
Sep 22 23:15:07.379 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/08B.db"
Sep 22 23:15:07.379 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:07.379 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/08B.replace" to "/tmp/downstairs-vrx8aK6L/00/000/08B.completed"
Sep 22 23:15:07.379 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:07.380 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:07.380 DEBG [0] It's time to notify for 190
Sep 22 23:15:07.380 INFO Completion from [0] id:190 status:true
Sep 22 23:15:07.380 INFO [191/752] Repair commands completed
Sep 22 23:15:07.380 INFO Pop front: ReconcileIO { id: ReconciliationId(191), op: ExtentReopen { repair_id: ReconciliationId(191), extent_id: 139 }, state: ClientData([New, New, New]) }
Sep 22 23:15:07.380 INFO Sent repair work, now wait for resp
Sep 22 23:15:07.380 INFO [0] received reconcile message
Sep 22 23:15:07.380 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(191), op: ExtentReopen { repair_id: ReconciliationId(191), extent_id: 139 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:07.380 INFO [0] client ExtentReopen { repair_id: ReconciliationId(191), extent_id: 139 }
Sep 22 23:15:07.380 INFO [1] received reconcile message
Sep 22 23:15:07.380 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(191), op: ExtentReopen { repair_id: ReconciliationId(191), extent_id: 139 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:07.380 INFO [1] client ExtentReopen { repair_id: ReconciliationId(191), extent_id: 139 }
Sep 22 23:15:07.380 INFO [2] received reconcile message
Sep 22 23:15:07.380 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(191), op: ExtentReopen { repair_id: ReconciliationId(191), extent_id: 139 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:07.380 INFO [2] client ExtentReopen { repair_id: ReconciliationId(191), extent_id: 139 }
Sep 22 23:15:07.380 DEBG 191 Reopen extent 139
Sep 22 23:15:07.381 DEBG 191 Reopen extent 139
Sep 22 23:15:07.382 DEBG 191 Reopen extent 139
Sep 22 23:15:07.382 DEBG [2] It's time to notify for 191
Sep 22 23:15:07.382 INFO Completion from [2] id:191 status:true
Sep 22 23:15:07.382 INFO [192/752] Repair commands completed
Sep 22 23:15:07.382 INFO Pop front: ReconcileIO { id: ReconciliationId(192), op: ExtentFlush { repair_id: ReconciliationId(192), extent_id: 163, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:07.382 INFO Sent repair work, now wait for resp
Sep 22 23:15:07.382 INFO [0] received reconcile message
Sep 22 23:15:07.382 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(192), op: ExtentFlush { repair_id: ReconciliationId(192), extent_id: 163, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:07.382 INFO [0] client ExtentFlush { repair_id: ReconciliationId(192), extent_id: 163, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:07.382 INFO [1] received reconcile message
Sep 22 23:15:07.382 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(192), op: ExtentFlush { repair_id: ReconciliationId(192), extent_id: 163, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:07.382 INFO [1] client ExtentFlush { repair_id: ReconciliationId(192), extent_id: 163, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:07.382 INFO [2] received reconcile message
Sep 22 23:15:07.382 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(192), op: ExtentFlush { repair_id: ReconciliationId(192), extent_id: 163, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:07.382 INFO [2] client ExtentFlush { repair_id: ReconciliationId(192), extent_id: 163, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:07.383 DEBG 192 Flush extent 163 with f:2 g:2
Sep 22 23:15:07.383 DEBG Flush just extent 163 with f:2 and g:2
Sep 22 23:15:07.383 DEBG [1] It's time to notify for 192
Sep 22 23:15:07.383 INFO Completion from [1] id:192 status:true
Sep 22 23:15:07.383 INFO [193/752] Repair commands completed
Sep 22 23:15:07.383 INFO Pop front: ReconcileIO { id: ReconciliationId(193), op: ExtentClose { repair_id: ReconciliationId(193), extent_id: 163 }, state: ClientData([New, New, New]) }
Sep 22 23:15:07.383 INFO Sent repair work, now wait for resp
Sep 22 23:15:07.383 INFO [0] received reconcile message
Sep 22 23:15:07.383 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(193), op: ExtentClose { repair_id: ReconciliationId(193), extent_id: 163 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:07.383 INFO [0] client ExtentClose { repair_id: ReconciliationId(193), extent_id: 163 }
Sep 22 23:15:07.383 INFO [1] received reconcile message
Sep 22 23:15:07.383 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(193), op: ExtentClose { repair_id: ReconciliationId(193), extent_id: 163 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:07.383 INFO [1] client ExtentClose { repair_id: ReconciliationId(193), extent_id: 163 }
Sep 22 23:15:07.383 INFO [2] received reconcile message
Sep 22 23:15:07.383 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(193), op: ExtentClose { repair_id: ReconciliationId(193), extent_id: 163 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:07.383 INFO [2] client ExtentClose { repair_id: ReconciliationId(193), extent_id: 163 }
Sep 22 23:15:07.383 DEBG 193 Close extent 163
Sep 22 23:15:07.383 DEBG 193 Close extent 163
Sep 22 23:15:07.384 DEBG 193 Close extent 163
Sep 22 23:15:07.384 DEBG [2] It's time to notify for 193
Sep 22 23:15:07.384 INFO Completion from [2] id:193 status:true
Sep 22 23:15:07.384 INFO [194/752] Repair commands completed
Sep 22 23:15:07.384 INFO Pop front: ReconcileIO { id: ReconciliationId(194), op: ExtentRepair { repair_id: ReconciliationId(194), extent_id: 163, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:07.384 INFO Sent repair work, now wait for resp
Sep 22 23:15:07.384 INFO [0] received reconcile message
Sep 22 23:15:07.384 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(194), op: ExtentRepair { repair_id: ReconciliationId(194), extent_id: 163, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:07.384 INFO [0] client ExtentRepair { repair_id: ReconciliationId(194), extent_id: 163, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:07.384 INFO [0] Sending repair request ReconciliationId(194)
Sep 22 23:15:07.384 INFO [1] received reconcile message
Sep 22 23:15:07.384 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(194), op: ExtentRepair { repair_id: ReconciliationId(194), extent_id: 163, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:07.384 INFO [1] client ExtentRepair { repair_id: ReconciliationId(194), extent_id: 163, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:07.384 INFO [1] No action required ReconciliationId(194)
Sep 22 23:15:07.384 INFO [2] received reconcile message
Sep 22 23:15:07.384 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(194), op: ExtentRepair { repair_id: ReconciliationId(194), extent_id: 163, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:07.384 INFO [2] client ExtentRepair { repair_id: ReconciliationId(194), extent_id: 163, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:07.384 INFO [2] No action required ReconciliationId(194)
Sep 22 23:15:07.385 DEBG 194 Repair extent 163 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:07.385 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/0A3.copy"
Sep 22 23:15:07.445 DEBG up_ds_listen was notified
Sep 22 23:15:07.445 DEBG up_ds_listen process 1077
Sep 22 23:15:07.446 DEBG [A] ack job 1077:78, : downstairs
Sep 22 23:15:07.446 DEBG [rc] retire 1077 clears [JobId(1076), JobId(1077)], : downstairs
Sep 22 23:15:07.446 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:07.446 INFO accepted connection, remote_addr: 127.0.0.1:35289, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.446 TRCE incoming request, uri: /extent/163/files, method: GET, req_id: 3c4d2311-2eb1-4b63-bae7-234365e29d49, remote_addr: 127.0.0.1:35289, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.446 INFO request completed, latency_us: 206, response_code: 200, uri: /extent/163/files, method: GET, req_id: 3c4d2311-2eb1-4b63-bae7-234365e29d49, remote_addr: 127.0.0.1:35289, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.447 INFO eid:163 Found repair files: ["0A3", "0A3.db"]
Sep 22 23:15:07.447 TRCE incoming request, uri: /newextent/163/data, method: GET, req_id: 680a42dd-80d6-491f-8f53-ba700bb966b0, remote_addr: 127.0.0.1:35289, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.447 INFO request completed, latency_us: 255, response_code: 200, uri: /newextent/163/data, method: GET, req_id: 680a42dd-80d6-491f-8f53-ba700bb966b0, remote_addr: 127.0.0.1:35289, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.452 TRCE incoming request, uri: /newextent/163/db, method: GET, req_id: e1d61684-de09-4875-9f90-8f6e3106e293, remote_addr: 127.0.0.1:35289, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.452 DEBG IO Flush 1079 has deps [JobId(1078)]
Sep 22 23:15:07.452 INFO request completed, latency_us: 292, response_code: 200, uri: /newextent/163/db, method: GET, req_id: e1d61684-de09-4875-9f90-8f6e3106e293, remote_addr: 127.0.0.1:35289, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.453 INFO [lossy] sleeping 1 second
Sep 22 23:15:07.454 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/0A3.copy" to "/tmp/downstairs-vrx8aK6L/00/000/0A3.replace"
Sep 22 23:15:07.454 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:07.454 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/0A3.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:07.455 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/0A3"
Sep 22 23:15:07.455 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/0A3.db"
Sep 22 23:15:07.455 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:07.455 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/0A3.replace" to "/tmp/downstairs-vrx8aK6L/00/000/0A3.completed"
Sep 22 23:15:07.455 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:07.455 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:07.455 DEBG [0] It's time to notify for 194
Sep 22 23:15:07.455 INFO Completion from [0] id:194 status:true
Sep 22 23:15:07.455 INFO [195/752] Repair commands completed
Sep 22 23:15:07.455 INFO Pop front: ReconcileIO { id: ReconciliationId(195), op: ExtentReopen { repair_id: ReconciliationId(195), extent_id: 163 }, state: ClientData([New, New, New]) }
Sep 22 23:15:07.455 INFO Sent repair work, now wait for resp
Sep 22 23:15:07.455 INFO [0] received reconcile message
Sep 22 23:15:07.455 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(195), op: ExtentReopen { repair_id: ReconciliationId(195), extent_id: 163 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:07.455 INFO [0] client ExtentReopen { repair_id: ReconciliationId(195), extent_id: 163 }
Sep 22 23:15:07.455 INFO [1] received reconcile message
Sep 22 23:15:07.455 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(195), op: ExtentReopen { repair_id: ReconciliationId(195), extent_id: 163 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:07.455 INFO [1] client ExtentReopen { repair_id: ReconciliationId(195), extent_id: 163 }
Sep 22 23:15:07.456 INFO [2] received reconcile message
Sep 22 23:15:07.456 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(195), op: ExtentReopen { repair_id: ReconciliationId(195), extent_id: 163 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:07.456 INFO [2] client ExtentReopen { repair_id: ReconciliationId(195), extent_id: 163 }
Sep 22 23:15:07.456 DEBG 195 Reopen extent 163
Sep 22 23:15:07.456 DEBG 195 Reopen extent 163
Sep 22 23:15:07.457 DEBG 195 Reopen extent 163
Sep 22 23:15:07.457 DEBG [2] It's time to notify for 195
Sep 22 23:15:07.457 INFO Completion from [2] id:195 status:true
Sep 22 23:15:07.457 INFO [196/752] Repair commands completed
Sep 22 23:15:07.457 INFO Pop front: ReconcileIO { id: ReconciliationId(196), op: ExtentFlush { repair_id: ReconciliationId(196), extent_id: 165, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:07.457 INFO Sent repair work, now wait for resp
Sep 22 23:15:07.458 INFO [0] received reconcile message
Sep 22 23:15:07.458 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(196), op: ExtentFlush { repair_id: ReconciliationId(196), extent_id: 165, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:07.458 INFO [0] client ExtentFlush { repair_id: ReconciliationId(196), extent_id: 165, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:07.458 INFO [1] received reconcile message
Sep 22 23:15:07.458 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(196), op: ExtentFlush { repair_id: ReconciliationId(196), extent_id: 165, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:07.458 INFO [1] client ExtentFlush { repair_id: ReconciliationId(196), extent_id: 165, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:07.458 INFO [2] received reconcile message
Sep 22 23:15:07.458 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(196), op: ExtentFlush { repair_id: ReconciliationId(196), extent_id: 165, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:07.458 INFO [2] client ExtentFlush { repair_id: ReconciliationId(196), extent_id: 165, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:07.458 DEBG 196 Flush extent 165 with f:2 g:2
Sep 22 23:15:07.458 DEBG Flush just extent 165 with f:2 and g:2
Sep 22 23:15:07.458 DEBG [1] It's time to notify for 196
Sep 22 23:15:07.458 INFO Completion from [1] id:196 status:true
Sep 22 23:15:07.458 INFO [197/752] Repair commands completed
Sep 22 23:15:07.458 INFO Pop front: ReconcileIO { id: ReconciliationId(197), op: ExtentClose { repair_id: ReconciliationId(197), extent_id: 165 }, state: ClientData([New, New, New]) }
Sep 22 23:15:07.458 INFO Sent repair work, now wait for resp
Sep 22 23:15:07.458 INFO [0] received reconcile message
Sep 22 23:15:07.458 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(197), op: ExtentClose { repair_id: ReconciliationId(197), extent_id: 165 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:07.458 INFO [0] client ExtentClose { repair_id: ReconciliationId(197), extent_id: 165 }
Sep 22 23:15:07.458 INFO [1] received reconcile message
Sep 22 23:15:07.458 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(197), op: ExtentClose { repair_id: ReconciliationId(197), extent_id: 165 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:07.458 INFO [1] client ExtentClose { repair_id: ReconciliationId(197), extent_id: 165 }
Sep 22 23:15:07.458 INFO [2] received reconcile message
Sep 22 23:15:07.458 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(197), op: ExtentClose { repair_id: ReconciliationId(197), extent_id: 165 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:07.458 INFO [2] client ExtentClose { repair_id: ReconciliationId(197), extent_id: 165 }
Sep 22 23:15:07.458 DEBG 197 Close extent 165
Sep 22 23:15:07.459 DEBG 197 Close extent 165
Sep 22 23:15:07.459 DEBG 197 Close extent 165
Sep 22 23:15:07.459 DEBG [2] It's time to notify for 197
Sep 22 23:15:07.459 INFO Completion from [2] id:197 status:true
Sep 22 23:15:07.459 INFO [198/752] Repair commands completed
Sep 22 23:15:07.459 INFO Pop front: ReconcileIO { id: ReconciliationId(198), op: ExtentRepair { repair_id: ReconciliationId(198), extent_id: 165, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:07.459 INFO Sent repair work, now wait for resp
Sep 22 23:15:07.459 INFO [0] received reconcile message
Sep 22 23:15:07.459 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(198), op: ExtentRepair { repair_id: ReconciliationId(198), extent_id: 165, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:07.459 INFO [0] client ExtentRepair { repair_id: ReconciliationId(198), extent_id: 165, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:07.459 INFO [0] Sending repair request ReconciliationId(198)
Sep 22 23:15:07.459 INFO [1] received reconcile message
Sep 22 23:15:07.459 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(198), op: ExtentRepair { repair_id: ReconciliationId(198), extent_id: 165, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:07.460 INFO [1] client ExtentRepair { repair_id: ReconciliationId(198), extent_id: 165, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:07.460 INFO [1] No action required ReconciliationId(198)
Sep 22 23:15:07.460 INFO [2] received reconcile message
Sep 22 23:15:07.460 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(198), op: ExtentRepair { repair_id: ReconciliationId(198), extent_id: 165, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:07.460 INFO [2] client ExtentRepair { repair_id: ReconciliationId(198), extent_id: 165, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:07.460 INFO [2] No action required ReconciliationId(198)
Sep 22 23:15:07.460 DEBG 198 Repair extent 165 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:07.460 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/0A5.copy"
Sep 22 23:15:07.523 INFO accepted connection, remote_addr: 127.0.0.1:49460, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.523 TRCE incoming request, uri: /extent/165/files, method: GET, req_id: c24b49bb-a7e7-40f1-a9ab-69f214c06fdf, remote_addr: 127.0.0.1:49460, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.524 INFO request completed, latency_us: 197, response_code: 200, uri: /extent/165/files, method: GET, req_id: c24b49bb-a7e7-40f1-a9ab-69f214c06fdf, remote_addr: 127.0.0.1:49460, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.524 INFO eid:165 Found repair files: ["0A5", "0A5.db"]
Sep 22 23:15:07.524 TRCE incoming request, uri: /newextent/165/data, method: GET, req_id: bb24b161-4847-42d1-bec2-8e762515403f, remote_addr: 127.0.0.1:49460, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.525 INFO request completed, latency_us: 320, response_code: 200, uri: /newextent/165/data, method: GET, req_id: bb24b161-4847-42d1-bec2-8e762515403f, remote_addr: 127.0.0.1:49460, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.530 TRCE incoming request, uri: /newextent/165/db, method: GET, req_id: ea83bd3f-85c1-413a-af0d-49188727ac03, remote_addr: 127.0.0.1:49460, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.530 INFO request completed, latency_us: 291, response_code: 200, uri: /newextent/165/db, method: GET, req_id: ea83bd3f-85c1-413a-af0d-49188727ac03, remote_addr: 127.0.0.1:49460, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.531 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/0A5.copy" to "/tmp/downstairs-vrx8aK6L/00/000/0A5.replace"
Sep 22 23:15:07.531 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:07.532 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/0A5.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:07.532 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/0A5"
Sep 22 23:15:07.532 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/0A5.db"
Sep 22 23:15:07.532 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:07.532 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/0A5.replace" to "/tmp/downstairs-vrx8aK6L/00/000/0A5.completed"
Sep 22 23:15:07.532 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:07.532 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:07.532 DEBG [0] It's time to notify for 198
Sep 22 23:15:07.533 INFO Completion from [0] id:198 status:true
Sep 22 23:15:07.533 INFO [199/752] Repair commands completed
Sep 22 23:15:07.533 INFO Pop front: ReconcileIO { id: ReconciliationId(199), op: ExtentReopen { repair_id: ReconciliationId(199), extent_id: 165 }, state: ClientData([New, New, New]) }
Sep 22 23:15:07.533 INFO Sent repair work, now wait for resp
Sep 22 23:15:07.533 INFO [0] received reconcile message
Sep 22 23:15:07.533 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(199), op: ExtentReopen { repair_id: ReconciliationId(199), extent_id: 165 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:07.533 INFO [0] client ExtentReopen { repair_id: ReconciliationId(199), extent_id: 165 }
Sep 22 23:15:07.533 INFO [1] received reconcile message
Sep 22 23:15:07.533 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(199), op: ExtentReopen { repair_id: ReconciliationId(199), extent_id: 165 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:07.533 INFO [1] client ExtentReopen { repair_id: ReconciliationId(199), extent_id: 165 }
Sep 22 23:15:07.533 INFO [2] received reconcile message
Sep 22 23:15:07.533 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(199), op: ExtentReopen { repair_id: ReconciliationId(199), extent_id: 165 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:07.533 INFO [2] client ExtentReopen { repair_id: ReconciliationId(199), extent_id: 165 }
Sep 22 23:15:07.533 DEBG 199 Reopen extent 165
Sep 22 23:15:07.534 DEBG 199 Reopen extent 165
Sep 22 23:15:07.534 DEBG 199 Reopen extent 165
Sep 22 23:15:07.535 DEBG [2] It's time to notify for 199
Sep 22 23:15:07.535 INFO Completion from [2] id:199 status:true
Sep 22 23:15:07.535 INFO [200/752] Repair commands completed
Sep 22 23:15:07.535 INFO Pop front: ReconcileIO { id: ReconciliationId(200), op: ExtentFlush { repair_id: ReconciliationId(200), extent_id: 71, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:07.535 INFO Sent repair work, now wait for resp
Sep 22 23:15:07.535 INFO [0] received reconcile message
Sep 22 23:15:07.535 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(200), op: ExtentFlush { repair_id: ReconciliationId(200), extent_id: 71, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:07.535 INFO [0] client ExtentFlush { repair_id: ReconciliationId(200), extent_id: 71, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:07.535 INFO [1] received reconcile message
Sep 22 23:15:07.535 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(200), op: ExtentFlush { repair_id: ReconciliationId(200), extent_id: 71, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:07.535 INFO [1] client ExtentFlush { repair_id: ReconciliationId(200), extent_id: 71, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:07.535 INFO [2] received reconcile message
Sep 22 23:15:07.535 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(200), op: ExtentFlush { repair_id: ReconciliationId(200), extent_id: 71, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:07.535 INFO [2] client ExtentFlush { repair_id: ReconciliationId(200), extent_id: 71, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:07.535 DEBG 200 Flush extent 71 with f:2 g:2
Sep 22 23:15:07.535 DEBG Flush just extent 71 with f:2 and g:2
Sep 22 23:15:07.535 DEBG [1] It's time to notify for 200
Sep 22 23:15:07.535 INFO Completion from [1] id:200 status:true
Sep 22 23:15:07.535 INFO [201/752] Repair commands completed
Sep 22 23:15:07.535 INFO Pop front: ReconcileIO { id: ReconciliationId(201), op: ExtentClose { repair_id: ReconciliationId(201), extent_id: 71 }, state: ClientData([New, New, New]) }
Sep 22 23:15:07.535 INFO Sent repair work, now wait for resp
Sep 22 23:15:07.535 INFO [0] received reconcile message
Sep 22 23:15:07.535 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(201), op: ExtentClose { repair_id: ReconciliationId(201), extent_id: 71 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:07.535 INFO [0] client ExtentClose { repair_id: ReconciliationId(201), extent_id: 71 }
Sep 22 23:15:07.535 INFO [1] received reconcile message
Sep 22 23:15:07.535 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(201), op: ExtentClose { repair_id: ReconciliationId(201), extent_id: 71 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:07.535 INFO [1] client ExtentClose { repair_id: ReconciliationId(201), extent_id: 71 }
Sep 22 23:15:07.536 INFO [2] received reconcile message
Sep 22 23:15:07.536 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(201), op: ExtentClose { repair_id: ReconciliationId(201), extent_id: 71 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:07.536 INFO [2] client ExtentClose { repair_id: ReconciliationId(201), extent_id: 71 }
Sep 22 23:15:07.536 DEBG 201 Close extent 71
Sep 22 23:15:07.536 DEBG 201 Close extent 71
Sep 22 23:15:07.536 DEBG 201 Close extent 71
Sep 22 23:15:07.537 DEBG [2] It's time to notify for 201
Sep 22 23:15:07.537 INFO Completion from [2] id:201 status:true
Sep 22 23:15:07.537 INFO [202/752] Repair commands completed
Sep 22 23:15:07.537 INFO Pop front: ReconcileIO { id: ReconciliationId(202), op: ExtentRepair { repair_id: ReconciliationId(202), extent_id: 71, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:07.537 INFO Sent repair work, now wait for resp
Sep 22 23:15:07.537 INFO [0] received reconcile message
Sep 22 23:15:07.537 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(202), op: ExtentRepair { repair_id: ReconciliationId(202), extent_id: 71, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:07.537 INFO [0] client ExtentRepair { repair_id: ReconciliationId(202), extent_id: 71, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:07.537 INFO [0] Sending repair request ReconciliationId(202)
Sep 22 23:15:07.537 INFO [1] received reconcile message
Sep 22 23:15:07.537 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(202), op: ExtentRepair { repair_id: ReconciliationId(202), extent_id: 71, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:07.537 INFO [1] client ExtentRepair { repair_id: ReconciliationId(202), extent_id: 71, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:07.537 INFO [1] No action required ReconciliationId(202)
Sep 22 23:15:07.537 INFO [2] received reconcile message
Sep 22 23:15:07.537 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(202), op: ExtentRepair { repair_id: ReconciliationId(202), extent_id: 71, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:07.537 INFO [2] client ExtentRepair { repair_id: ReconciliationId(202), extent_id: 71, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:07.537 INFO [2] No action required ReconciliationId(202)
Sep 22 23:15:07.537 DEBG 202 Repair extent 71 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:07.537 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/047.copy"
Sep 22 23:15:07.597 INFO accepted connection, remote_addr: 127.0.0.1:56999, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.598 TRCE incoming request, uri: /extent/71/files, method: GET, req_id: 647b4075-1a57-4e43-a49b-f700ed9d6a51, remote_addr: 127.0.0.1:56999, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.598 INFO request completed, latency_us: 193, response_code: 200, uri: /extent/71/files, method: GET, req_id: 647b4075-1a57-4e43-a49b-f700ed9d6a51, remote_addr: 127.0.0.1:56999, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.598 INFO eid:71 Found repair files: ["047", "047.db"]
Sep 22 23:15:07.598 TRCE incoming request, uri: /newextent/71/data, method: GET, req_id: 3844fc9a-02e2-4cb1-9ef6-6133ea20948f, remote_addr: 127.0.0.1:56999, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.599 INFO request completed, latency_us: 327, response_code: 200, uri: /newextent/71/data, method: GET, req_id: 3844fc9a-02e2-4cb1-9ef6-6133ea20948f, remote_addr: 127.0.0.1:56999, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.604 TRCE incoming request, uri: /newextent/71/db, method: GET, req_id: 98d8015c-0582-467e-923e-79055a81464f, remote_addr: 127.0.0.1:56999, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.604 INFO request completed, latency_us: 290, response_code: 200, uri: /newextent/71/db, method: GET, req_id: 98d8015c-0582-467e-923e-79055a81464f, remote_addr: 127.0.0.1:56999, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.605 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/047.copy" to "/tmp/downstairs-vrx8aK6L/00/000/047.replace"
Sep 22 23:15:07.605 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:07.606 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/047.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:07.606 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/047"
Sep 22 23:15:07.606 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/047.db"
Sep 22 23:15:07.606 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:07.606 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/047.replace" to "/tmp/downstairs-vrx8aK6L/00/000/047.completed"
Sep 22 23:15:07.606 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:07.606 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:07.606 DEBG [0] It's time to notify for 202
Sep 22 23:15:07.607 INFO Completion from [0] id:202 status:true
Sep 22 23:15:07.607 INFO [203/752] Repair commands completed
Sep 22 23:15:07.607 INFO Pop front: ReconcileIO { id: ReconciliationId(203), op: ExtentReopen { repair_id: ReconciliationId(203), extent_id: 71 }, state: ClientData([New, New, New]) }
Sep 22 23:15:07.607 INFO Sent repair work, now wait for resp
Sep 22 23:15:07.607 INFO [0] received reconcile message
Sep 22 23:15:07.607 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(203), op: ExtentReopen { repair_id: ReconciliationId(203), extent_id: 71 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:07.607 INFO [0] client ExtentReopen { repair_id: ReconciliationId(203), extent_id: 71 }
Sep 22 23:15:07.607 INFO [1] received reconcile message
Sep 22 23:15:07.607 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(203), op: ExtentReopen { repair_id: ReconciliationId(203), extent_id: 71 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:07.607 INFO [1] client ExtentReopen { repair_id: ReconciliationId(203), extent_id: 71 }
Sep 22 23:15:07.607 INFO [2] received reconcile message
Sep 22 23:15:07.607 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(203), op: ExtentReopen { repair_id: ReconciliationId(203), extent_id: 71 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:07.607 INFO [2] client ExtentReopen { repair_id: ReconciliationId(203), extent_id: 71 }
Sep 22 23:15:07.607 DEBG 203 Reopen extent 71
Sep 22 23:15:07.607 DEBG 203 Reopen extent 71
Sep 22 23:15:07.608 DEBG 203 Reopen extent 71
Sep 22 23:15:07.609 DEBG [2] It's time to notify for 203
Sep 22 23:15:07.609 INFO Completion from [2] id:203 status:true
Sep 22 23:15:07.609 INFO [204/752] Repair commands completed
Sep 22 23:15:07.609 INFO Pop front: ReconcileIO { id: ReconciliationId(204), op: ExtentFlush { repair_id: ReconciliationId(204), extent_id: 141, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:07.609 INFO Sent repair work, now wait for resp
Sep 22 23:15:07.609 INFO [0] received reconcile message
Sep 22 23:15:07.609 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(204), op: ExtentFlush { repair_id: ReconciliationId(204), extent_id: 141, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:07.609 INFO [0] client ExtentFlush { repair_id: ReconciliationId(204), extent_id: 141, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:07.609 INFO [1] received reconcile message
Sep 22 23:15:07.609 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(204), op: ExtentFlush { repair_id: ReconciliationId(204), extent_id: 141, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:07.609 INFO [1] client ExtentFlush { repair_id: ReconciliationId(204), extent_id: 141, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:07.609 INFO [2] received reconcile message
Sep 22 23:15:07.609 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(204), op: ExtentFlush { repair_id: ReconciliationId(204), extent_id: 141, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:07.609 INFO [2] client ExtentFlush { repair_id: ReconciliationId(204), extent_id: 141, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:07.609 DEBG 204 Flush extent 141 with f:2 g:2
Sep 22 23:15:07.609 DEBG Flush just extent 141 with f:2 and g:2
Sep 22 23:15:07.609 DEBG [1] It's time to notify for 204
Sep 22 23:15:07.609 INFO Completion from [1] id:204 status:true
Sep 22 23:15:07.609 INFO [205/752] Repair commands completed
Sep 22 23:15:07.609 INFO Pop front: ReconcileIO { id: ReconciliationId(205), op: ExtentClose { repair_id: ReconciliationId(205), extent_id: 141 }, state: ClientData([New, New, New]) }
Sep 22 23:15:07.609 INFO Sent repair work, now wait for resp
Sep 22 23:15:07.609 INFO [0] received reconcile message
Sep 22 23:15:07.609 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(205), op: ExtentClose { repair_id: ReconciliationId(205), extent_id: 141 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:07.609 INFO [0] client ExtentClose { repair_id: ReconciliationId(205), extent_id: 141 }
Sep 22 23:15:07.609 INFO [1] received reconcile message
Sep 22 23:15:07.609 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(205), op: ExtentClose { repair_id: ReconciliationId(205), extent_id: 141 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:07.609 INFO [1] client ExtentClose { repair_id: ReconciliationId(205), extent_id: 141 }
Sep 22 23:15:07.609 INFO [2] received reconcile message
Sep 22 23:15:07.609 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(205), op: ExtentClose { repair_id: ReconciliationId(205), extent_id: 141 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:07.609 INFO [2] client ExtentClose { repair_id: ReconciliationId(205), extent_id: 141 }
Sep 22 23:15:07.610 DEBG 205 Close extent 141
Sep 22 23:15:07.610 DEBG 205 Close extent 141
Sep 22 23:15:07.610 DEBG 205 Close extent 141
Sep 22 23:15:07.610 DEBG [2] It's time to notify for 205
Sep 22 23:15:07.611 INFO Completion from [2] id:205 status:true
Sep 22 23:15:07.611 INFO [206/752] Repair commands completed
Sep 22 23:15:07.611 INFO Pop front: ReconcileIO { id: ReconciliationId(206), op: ExtentRepair { repair_id: ReconciliationId(206), extent_id: 141, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:07.611 INFO Sent repair work, now wait for resp
Sep 22 23:15:07.611 INFO [0] received reconcile message
Sep 22 23:15:07.611 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(206), op: ExtentRepair { repair_id: ReconciliationId(206), extent_id: 141, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:07.611 INFO [0] client ExtentRepair { repair_id: ReconciliationId(206), extent_id: 141, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:07.611 INFO [0] Sending repair request ReconciliationId(206)
Sep 22 23:15:07.611 INFO [1] received reconcile message
Sep 22 23:15:07.611 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(206), op: ExtentRepair { repair_id: ReconciliationId(206), extent_id: 141, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:07.611 INFO [1] client ExtentRepair { repair_id: ReconciliationId(206), extent_id: 141, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:07.611 INFO [1] No action required ReconciliationId(206)
Sep 22 23:15:07.611 INFO [2] received reconcile message
Sep 22 23:15:07.611 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(206), op: ExtentRepair { repair_id: ReconciliationId(206), extent_id: 141, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:07.611 INFO [2] client ExtentRepair { repair_id: ReconciliationId(206), extent_id: 141, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:07.611 INFO [2] No action required ReconciliationId(206)
Sep 22 23:15:07.611 DEBG 206 Repair extent 141 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:07.611 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/08D.copy"
Sep 22 23:15:07.675 INFO accepted connection, remote_addr: 127.0.0.1:42719, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.675 TRCE incoming request, uri: /extent/141/files, method: GET, req_id: d984cac8-a869-4355-84c9-b52543b196b2, remote_addr: 127.0.0.1:42719, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.676 INFO request completed, latency_us: 211, response_code: 200, uri: /extent/141/files, method: GET, req_id: d984cac8-a869-4355-84c9-b52543b196b2, remote_addr: 127.0.0.1:42719, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.676 INFO eid:141 Found repair files: ["08D", "08D.db"]
Sep 22 23:15:07.676 TRCE incoming request, uri: /newextent/141/data, method: GET, req_id: df7c18b3-ab72-450d-922a-ff13b4ff928f, remote_addr: 127.0.0.1:42719, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.677 INFO request completed, latency_us: 328, response_code: 200, uri: /newextent/141/data, method: GET, req_id: df7c18b3-ab72-450d-922a-ff13b4ff928f, remote_addr: 127.0.0.1:42719, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.682 TRCE incoming request, uri: /newextent/141/db, method: GET, req_id: 77f6cb1b-5fa7-45e3-9624-f0100a9da33e, remote_addr: 127.0.0.1:42719, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.682 INFO request completed, latency_us: 293, response_code: 200, uri: /newextent/141/db, method: GET, req_id: 77f6cb1b-5fa7-45e3-9624-f0100a9da33e, remote_addr: 127.0.0.1:42719, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.683 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/08D.copy" to "/tmp/downstairs-vrx8aK6L/00/000/08D.replace"
Sep 22 23:15:07.683 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:07.684 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/08D.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:07.684 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/08D"
Sep 22 23:15:07.684 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/08D.db"
Sep 22 23:15:07.684 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:07.684 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/08D.replace" to "/tmp/downstairs-vrx8aK6L/00/000/08D.completed"
Sep 22 23:15:07.685 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:07.685 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:07.685 DEBG [0] It's time to notify for 206
Sep 22 23:15:07.685 INFO Completion from [0] id:206 status:true
Sep 22 23:15:07.685 INFO [207/752] Repair commands completed
Sep 22 23:15:07.685 INFO Pop front: ReconcileIO { id: ReconciliationId(207), op: ExtentReopen { repair_id: ReconciliationId(207), extent_id: 141 }, state: ClientData([New, New, New]) }
Sep 22 23:15:07.685 INFO Sent repair work, now wait for resp
Sep 22 23:15:07.685 INFO [0] received reconcile message
Sep 22 23:15:07.685 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(207), op: ExtentReopen { repair_id: ReconciliationId(207), extent_id: 141 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:07.685 INFO [0] client ExtentReopen { repair_id: ReconciliationId(207), extent_id: 141 }
Sep 22 23:15:07.685 INFO [1] received reconcile message
Sep 22 23:15:07.685 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(207), op: ExtentReopen { repair_id: ReconciliationId(207), extent_id: 141 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:07.685 INFO [1] client ExtentReopen { repair_id: ReconciliationId(207), extent_id: 141 }
Sep 22 23:15:07.685 INFO [2] received reconcile message
Sep 22 23:15:07.685 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(207), op: ExtentReopen { repair_id: ReconciliationId(207), extent_id: 141 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:07.685 INFO [2] client ExtentReopen { repair_id: ReconciliationId(207), extent_id: 141 }
Sep 22 23:15:07.685 DEBG 207 Reopen extent 141
Sep 22 23:15:07.686 DEBG Read      :1078 deps:[JobId(1077)] res:true
Sep 22 23:15:07.686 DEBG 207 Reopen extent 141
Sep 22 23:15:07.687 DEBG 207 Reopen extent 141
Sep 22 23:15:07.687 DEBG [2] It's time to notify for 207
Sep 22 23:15:07.687 INFO Completion from [2] id:207 status:true
Sep 22 23:15:07.687 INFO [208/752] Repair commands completed
Sep 22 23:15:07.687 INFO Pop front: ReconcileIO { id: ReconciliationId(208), op: ExtentFlush { repair_id: ReconciliationId(208), extent_id: 114, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:07.687 INFO Sent repair work, now wait for resp
Sep 22 23:15:07.687 INFO [0] received reconcile message
Sep 22 23:15:07.687 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(208), op: ExtentFlush { repair_id: ReconciliationId(208), extent_id: 114, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:07.687 INFO [0] client ExtentFlush { repair_id: ReconciliationId(208), extent_id: 114, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:07.687 INFO [1] received reconcile message
Sep 22 23:15:07.687 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(208), op: ExtentFlush { repair_id: ReconciliationId(208), extent_id: 114, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:07.687 INFO [1] client ExtentFlush { repair_id: ReconciliationId(208), extent_id: 114, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:07.687 INFO [2] received reconcile message
Sep 22 23:15:07.687 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(208), op: ExtentFlush { repair_id: ReconciliationId(208), extent_id: 114, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:07.687 INFO [2] client ExtentFlush { repair_id: ReconciliationId(208), extent_id: 114, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:07.688 DEBG 208 Flush extent 114 with f:2 g:2
Sep 22 23:15:07.688 DEBG Flush just extent 114 with f:2 and g:2
Sep 22 23:15:07.688 DEBG [1] It's time to notify for 208
Sep 22 23:15:07.688 INFO Completion from [1] id:208 status:true
Sep 22 23:15:07.688 INFO [209/752] Repair commands completed
Sep 22 23:15:07.688 INFO Pop front: ReconcileIO { id: ReconciliationId(209), op: ExtentClose { repair_id: ReconciliationId(209), extent_id: 114 }, state: ClientData([New, New, New]) }
Sep 22 23:15:07.688 INFO Sent repair work, now wait for resp
Sep 22 23:15:07.688 INFO [0] received reconcile message
Sep 22 23:15:07.688 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(209), op: ExtentClose { repair_id: ReconciliationId(209), extent_id: 114 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:07.688 INFO [0] client ExtentClose { repair_id: ReconciliationId(209), extent_id: 114 }
Sep 22 23:15:07.688 INFO [1] received reconcile message
Sep 22 23:15:07.688 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(209), op: ExtentClose { repair_id: ReconciliationId(209), extent_id: 114 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:07.688 INFO [1] client ExtentClose { repair_id: ReconciliationId(209), extent_id: 114 }
Sep 22 23:15:07.688 INFO [2] received reconcile message
Sep 22 23:15:07.688 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(209), op: ExtentClose { repair_id: ReconciliationId(209), extent_id: 114 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:07.688 INFO [2] client ExtentClose { repair_id: ReconciliationId(209), extent_id: 114 }
Sep 22 23:15:07.688 DEBG 209 Close extent 114
Sep 22 23:15:07.688 DEBG 209 Close extent 114
Sep 22 23:15:07.689 DEBG 209 Close extent 114
Sep 22 23:15:07.689 DEBG [2] It's time to notify for 209
Sep 22 23:15:07.689 INFO Completion from [2] id:209 status:true
Sep 22 23:15:07.689 INFO [210/752] Repair commands completed
Sep 22 23:15:07.689 INFO Pop front: ReconcileIO { id: ReconciliationId(210), op: ExtentRepair { repair_id: ReconciliationId(210), extent_id: 114, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:07.689 INFO Sent repair work, now wait for resp
Sep 22 23:15:07.689 INFO [0] received reconcile message
Sep 22 23:15:07.689 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(210), op: ExtentRepair { repair_id: ReconciliationId(210), extent_id: 114, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:07.689 INFO [0] client ExtentRepair { repair_id: ReconciliationId(210), extent_id: 114, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:07.689 INFO [0] Sending repair request ReconciliationId(210)
Sep 22 23:15:07.689 INFO [1] received reconcile message
Sep 22 23:15:07.689 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(210), op: ExtentRepair { repair_id: ReconciliationId(210), extent_id: 114, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:07.689 INFO [1] client ExtentRepair { repair_id: ReconciliationId(210), extent_id: 114, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:07.689 INFO [1] No action required ReconciliationId(210)
Sep 22 23:15:07.689 INFO [2] received reconcile message
Sep 22 23:15:07.689 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(210), op: ExtentRepair { repair_id: ReconciliationId(210), extent_id: 114, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:07.689 INFO [2] client ExtentRepair { repair_id: ReconciliationId(210), extent_id: 114, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:07.690 INFO [2] No action required ReconciliationId(210)
Sep 22 23:15:07.690 DEBG 210 Repair extent 114 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:07.690 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/072.copy"
Sep 22 23:15:07.707 INFO [lossy] skipping 1078
Sep 22 23:15:07.707 INFO [lossy] skipping 1078
Sep 22 23:15:07.714 DEBG Read      :1078 deps:[JobId(1077)] res:true
Sep 22 23:15:07.737 INFO [lossy] skipping 1079
Sep 22 23:15:07.737 DEBG Flush     :1079 extent_limit None deps:[JobId(1078)] res:true f:29 g:1
Sep 22 23:15:07.737 INFO [lossy] sleeping 1 second
Sep 22 23:15:07.750 INFO accepted connection, remote_addr: 127.0.0.1:57983, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.750 TRCE incoming request, uri: /extent/114/files, method: GET, req_id: e0183434-1d00-4919-b2a3-77d22d5d5997, remote_addr: 127.0.0.1:57983, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.750 INFO request completed, latency_us: 230, response_code: 200, uri: /extent/114/files, method: GET, req_id: e0183434-1d00-4919-b2a3-77d22d5d5997, remote_addr: 127.0.0.1:57983, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.750 INFO eid:114 Found repair files: ["072", "072.db"]
Sep 22 23:15:07.751 TRCE incoming request, uri: /newextent/114/data, method: GET, req_id: 6a9364e3-6f10-4480-ab72-341cc126b76b, remote_addr: 127.0.0.1:57983, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.751 INFO request completed, latency_us: 322, response_code: 200, uri: /newextent/114/data, method: GET, req_id: 6a9364e3-6f10-4480-ab72-341cc126b76b, remote_addr: 127.0.0.1:57983, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.756 TRCE incoming request, uri: /newextent/114/db, method: GET, req_id: db957bd1-472f-4fbc-94a4-3a19979571e5, remote_addr: 127.0.0.1:57983, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.756 INFO request completed, latency_us: 291, response_code: 200, uri: /newextent/114/db, method: GET, req_id: db957bd1-472f-4fbc-94a4-3a19979571e5, remote_addr: 127.0.0.1:57983, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.758 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/072.copy" to "/tmp/downstairs-vrx8aK6L/00/000/072.replace"
Sep 22 23:15:07.758 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:07.759 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/072.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:07.759 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/072"
Sep 22 23:15:07.759 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/072.db"
Sep 22 23:15:07.759 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:07.759 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/072.replace" to "/tmp/downstairs-vrx8aK6L/00/000/072.completed"
Sep 22 23:15:07.759 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:07.759 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:07.759 DEBG [0] It's time to notify for 210
Sep 22 23:15:07.759 INFO Completion from [0] id:210 status:true
Sep 22 23:15:07.759 INFO [211/752] Repair commands completed
Sep 22 23:15:07.759 INFO Pop front: ReconcileIO { id: ReconciliationId(211), op: ExtentReopen { repair_id: ReconciliationId(211), extent_id: 114 }, state: ClientData([New, New, New]) }
Sep 22 23:15:07.759 INFO Sent repair work, now wait for resp
Sep 22 23:15:07.760 INFO [0] received reconcile message
Sep 22 23:15:07.760 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(211), op: ExtentReopen { repair_id: ReconciliationId(211), extent_id: 114 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:07.760 INFO [0] client ExtentReopen { repair_id: ReconciliationId(211), extent_id: 114 }
Sep 22 23:15:07.760 INFO [1] received reconcile message
Sep 22 23:15:07.760 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(211), op: ExtentReopen { repair_id: ReconciliationId(211), extent_id: 114 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:07.760 INFO [1] client ExtentReopen { repair_id: ReconciliationId(211), extent_id: 114 }
Sep 22 23:15:07.760 INFO [2] received reconcile message
Sep 22 23:15:07.760 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(211), op: ExtentReopen { repair_id: ReconciliationId(211), extent_id: 114 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:07.760 INFO [2] client ExtentReopen { repair_id: ReconciliationId(211), extent_id: 114 }
Sep 22 23:15:07.760 DEBG 211 Reopen extent 114
Sep 22 23:15:07.761 DEBG 211 Reopen extent 114
Sep 22 23:15:07.761 DEBG 211 Reopen extent 114
Sep 22 23:15:07.762 DEBG [2] It's time to notify for 211
Sep 22 23:15:07.762 INFO Completion from [2] id:211 status:true
Sep 22 23:15:07.762 INFO [212/752] Repair commands completed
Sep 22 23:15:07.762 INFO Pop front: ReconcileIO { id: ReconciliationId(212), op: ExtentFlush { repair_id: ReconciliationId(212), extent_id: 107, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:07.762 INFO Sent repair work, now wait for resp
Sep 22 23:15:07.762 INFO [0] received reconcile message
Sep 22 23:15:07.762 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(212), op: ExtentFlush { repair_id: ReconciliationId(212), extent_id: 107, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:07.762 INFO [0] client ExtentFlush { repair_id: ReconciliationId(212), extent_id: 107, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:07.762 INFO [1] received reconcile message
Sep 22 23:15:07.762 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(212), op: ExtentFlush { repair_id: ReconciliationId(212), extent_id: 107, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:07.762 INFO [1] client ExtentFlush { repair_id: ReconciliationId(212), extent_id: 107, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:07.762 INFO [2] received reconcile message
Sep 22 23:15:07.762 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(212), op: ExtentFlush { repair_id: ReconciliationId(212), extent_id: 107, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:07.762 INFO [2] client ExtentFlush { repair_id: ReconciliationId(212), extent_id: 107, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:07.762 DEBG 212 Flush extent 107 with f:2 g:2
Sep 22 23:15:07.762 DEBG Flush just extent 107 with f:2 and g:2
Sep 22 23:15:07.762 DEBG [1] It's time to notify for 212
Sep 22 23:15:07.762 INFO Completion from [1] id:212 status:true
Sep 22 23:15:07.762 INFO [213/752] Repair commands completed
Sep 22 23:15:07.762 INFO Pop front: ReconcileIO { id: ReconciliationId(213), op: ExtentClose { repair_id: ReconciliationId(213), extent_id: 107 }, state: ClientData([New, New, New]) }
Sep 22 23:15:07.762 INFO Sent repair work, now wait for resp
Sep 22 23:15:07.762 INFO [0] received reconcile message
Sep 22 23:15:07.762 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(213), op: ExtentClose { repair_id: ReconciliationId(213), extent_id: 107 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:07.762 INFO [0] client ExtentClose { repair_id: ReconciliationId(213), extent_id: 107 }
Sep 22 23:15:07.762 INFO [1] received reconcile message
Sep 22 23:15:07.762 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(213), op: ExtentClose { repair_id: ReconciliationId(213), extent_id: 107 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:07.762 INFO [1] client ExtentClose { repair_id: ReconciliationId(213), extent_id: 107 }
Sep 22 23:15:07.763 INFO [2] received reconcile message
Sep 22 23:15:07.763 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(213), op: ExtentClose { repair_id: ReconciliationId(213), extent_id: 107 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:07.763 INFO [2] client ExtentClose { repair_id: ReconciliationId(213), extent_id: 107 }
Sep 22 23:15:07.763 DEBG 213 Close extent 107
Sep 22 23:15:07.763 DEBG 213 Close extent 107
Sep 22 23:15:07.763 DEBG 213 Close extent 107
Sep 22 23:15:07.764 DEBG [2] It's time to notify for 213
Sep 22 23:15:07.764 INFO Completion from [2] id:213 status:true
Sep 22 23:15:07.764 INFO [214/752] Repair commands completed
Sep 22 23:15:07.764 INFO Pop front: ReconcileIO { id: ReconciliationId(214), op: ExtentRepair { repair_id: ReconciliationId(214), extent_id: 107, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:07.764 INFO Sent repair work, now wait for resp
Sep 22 23:15:07.764 INFO [0] received reconcile message
Sep 22 23:15:07.764 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(214), op: ExtentRepair { repair_id: ReconciliationId(214), extent_id: 107, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:07.764 INFO [0] client ExtentRepair { repair_id: ReconciliationId(214), extent_id: 107, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:07.764 INFO [0] Sending repair request ReconciliationId(214)
Sep 22 23:15:07.764 INFO [1] received reconcile message
Sep 22 23:15:07.764 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(214), op: ExtentRepair { repair_id: ReconciliationId(214), extent_id: 107, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:07.764 INFO [1] client ExtentRepair { repair_id: ReconciliationId(214), extent_id: 107, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:07.764 INFO [1] No action required ReconciliationId(214)
Sep 22 23:15:07.764 INFO [2] received reconcile message
Sep 22 23:15:07.764 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(214), op: ExtentRepair { repair_id: ReconciliationId(214), extent_id: 107, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:07.764 INFO [2] client ExtentRepair { repair_id: ReconciliationId(214), extent_id: 107, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:07.764 INFO [2] No action required ReconciliationId(214)
Sep 22 23:15:07.764 DEBG 214 Repair extent 107 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:07.764 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/06B.copy"
Sep 22 23:15:07.786 DEBG Flush     :1079 extent_limit None deps:[JobId(1078)] res:true f:29 g:1
Sep 22 23:15:07.786 INFO [lossy] sleeping 1 second
Sep 22 23:15:07.829 INFO accepted connection, remote_addr: 127.0.0.1:61335, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.829 TRCE incoming request, uri: /extent/107/files, method: GET, req_id: bc890764-1cf6-41d7-9420-7b1a70592f84, remote_addr: 127.0.0.1:61335, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.829 INFO request completed, latency_us: 231, response_code: 200, uri: /extent/107/files, method: GET, req_id: bc890764-1cf6-41d7-9420-7b1a70592f84, remote_addr: 127.0.0.1:61335, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.829 INFO eid:107 Found repair files: ["06B", "06B.db"]
Sep 22 23:15:07.830 TRCE incoming request, uri: /newextent/107/data, method: GET, req_id: d9c05532-0e9b-496b-9399-af02db856432, remote_addr: 127.0.0.1:61335, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.830 INFO request completed, latency_us: 325, response_code: 200, uri: /newextent/107/data, method: GET, req_id: d9c05532-0e9b-496b-9399-af02db856432, remote_addr: 127.0.0.1:61335, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.835 TRCE incoming request, uri: /newextent/107/db, method: GET, req_id: 0ca9d7fc-feed-41f8-8070-ef0cec295249, remote_addr: 127.0.0.1:61335, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.835 INFO request completed, latency_us: 291, response_code: 200, uri: /newextent/107/db, method: GET, req_id: 0ca9d7fc-feed-41f8-8070-ef0cec295249, remote_addr: 127.0.0.1:61335, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.837 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/06B.copy" to "/tmp/downstairs-vrx8aK6L/00/000/06B.replace"
Sep 22 23:15:07.837 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:07.837 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/06B.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:07.838 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/06B"
Sep 22 23:15:07.838 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/06B.db"
Sep 22 23:15:07.838 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:07.838 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/06B.replace" to "/tmp/downstairs-vrx8aK6L/00/000/06B.completed"
Sep 22 23:15:07.838 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:07.838 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:07.838 DEBG [0] It's time to notify for 214
Sep 22 23:15:07.838 INFO Completion from [0] id:214 status:true
Sep 22 23:15:07.838 INFO [215/752] Repair commands completed
Sep 22 23:15:07.838 INFO Pop front: ReconcileIO { id: ReconciliationId(215), op: ExtentReopen { repair_id: ReconciliationId(215), extent_id: 107 }, state: ClientData([New, New, New]) }
Sep 22 23:15:07.838 INFO Sent repair work, now wait for resp
Sep 22 23:15:07.838 INFO [0] received reconcile message
Sep 22 23:15:07.838 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(215), op: ExtentReopen { repair_id: ReconciliationId(215), extent_id: 107 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:07.838 INFO [0] client ExtentReopen { repair_id: ReconciliationId(215), extent_id: 107 }
Sep 22 23:15:07.838 INFO [1] received reconcile message
Sep 22 23:15:07.838 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(215), op: ExtentReopen { repair_id: ReconciliationId(215), extent_id: 107 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:07.839 INFO [1] client ExtentReopen { repair_id: ReconciliationId(215), extent_id: 107 }
Sep 22 23:15:07.839 INFO [2] received reconcile message
Sep 22 23:15:07.839 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(215), op: ExtentReopen { repair_id: ReconciliationId(215), extent_id: 107 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:07.839 INFO [2] client ExtentReopen { repair_id: ReconciliationId(215), extent_id: 107 }
Sep 22 23:15:07.839 DEBG 215 Reopen extent 107
Sep 22 23:15:07.839 DEBG 215 Reopen extent 107
Sep 22 23:15:07.840 DEBG 215 Reopen extent 107
Sep 22 23:15:07.841 DEBG [2] It's time to notify for 215
Sep 22 23:15:07.841 INFO Completion from [2] id:215 status:true
Sep 22 23:15:07.841 INFO [216/752] Repair commands completed
Sep 22 23:15:07.841 INFO Pop front: ReconcileIO { id: ReconciliationId(216), op: ExtentFlush { repair_id: ReconciliationId(216), extent_id: 150, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:07.841 INFO Sent repair work, now wait for resp
Sep 22 23:15:07.841 INFO [0] received reconcile message
Sep 22 23:15:07.841 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(216), op: ExtentFlush { repair_id: ReconciliationId(216), extent_id: 150, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:07.841 INFO [0] client ExtentFlush { repair_id: ReconciliationId(216), extent_id: 150, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:07.841 INFO [1] received reconcile message
Sep 22 23:15:07.841 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(216), op: ExtentFlush { repair_id: ReconciliationId(216), extent_id: 150, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:07.841 INFO [1] client ExtentFlush { repair_id: ReconciliationId(216), extent_id: 150, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:07.841 INFO [2] received reconcile message
Sep 22 23:15:07.841 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(216), op: ExtentFlush { repair_id: ReconciliationId(216), extent_id: 150, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:07.841 INFO [2] client ExtentFlush { repair_id: ReconciliationId(216), extent_id: 150, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:07.841 DEBG 216 Flush extent 150 with f:2 g:2
Sep 22 23:15:07.841 DEBG Flush just extent 150 with f:2 and g:2
Sep 22 23:15:07.841 DEBG [1] It's time to notify for 216
Sep 22 23:15:07.841 INFO Completion from [1] id:216 status:true
Sep 22 23:15:07.841 INFO [217/752] Repair commands completed
Sep 22 23:15:07.841 INFO Pop front: ReconcileIO { id: ReconciliationId(217), op: ExtentClose { repair_id: ReconciliationId(217), extent_id: 150 }, state: ClientData([New, New, New]) }
Sep 22 23:15:07.841 INFO Sent repair work, now wait for resp
Sep 22 23:15:07.841 INFO [0] received reconcile message
Sep 22 23:15:07.841 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(217), op: ExtentClose { repair_id: ReconciliationId(217), extent_id: 150 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:07.841 INFO [0] client ExtentClose { repair_id: ReconciliationId(217), extent_id: 150 }
Sep 22 23:15:07.841 INFO [1] received reconcile message
Sep 22 23:15:07.841 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(217), op: ExtentClose { repair_id: ReconciliationId(217), extent_id: 150 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:07.841 INFO [1] client ExtentClose { repair_id: ReconciliationId(217), extent_id: 150 }
Sep 22 23:15:07.841 INFO [2] received reconcile message
Sep 22 23:15:07.841 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(217), op: ExtentClose { repair_id: ReconciliationId(217), extent_id: 150 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:07.841 INFO [2] client ExtentClose { repair_id: ReconciliationId(217), extent_id: 150 }
Sep 22 23:15:07.842 DEBG 217 Close extent 150
Sep 22 23:15:07.842 DEBG 217 Close extent 150
Sep 22 23:15:07.842 DEBG 217 Close extent 150
Sep 22 23:15:07.842 DEBG [2] It's time to notify for 217
Sep 22 23:15:07.842 INFO Completion from [2] id:217 status:true
Sep 22 23:15:07.843 INFO [218/752] Repair commands completed
Sep 22 23:15:07.843 INFO Pop front: ReconcileIO { id: ReconciliationId(218), op: ExtentRepair { repair_id: ReconciliationId(218), extent_id: 150, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:07.843 INFO Sent repair work, now wait for resp
Sep 22 23:15:07.843 INFO [0] received reconcile message
Sep 22 23:15:07.843 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(218), op: ExtentRepair { repair_id: ReconciliationId(218), extent_id: 150, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:07.843 INFO [0] client ExtentRepair { repair_id: ReconciliationId(218), extent_id: 150, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:07.843 INFO [0] Sending repair request ReconciliationId(218)
Sep 22 23:15:07.843 INFO [1] received reconcile message
Sep 22 23:15:07.843 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(218), op: ExtentRepair { repair_id: ReconciliationId(218), extent_id: 150, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:07.843 INFO [1] client ExtentRepair { repair_id: ReconciliationId(218), extent_id: 150, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:07.843 INFO [1] No action required ReconciliationId(218)
Sep 22 23:15:07.843 INFO [2] received reconcile message
Sep 22 23:15:07.843 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(218), op: ExtentRepair { repair_id: ReconciliationId(218), extent_id: 150, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:07.843 INFO [2] client ExtentRepair { repair_id: ReconciliationId(218), extent_id: 150, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:07.843 INFO [2] No action required ReconciliationId(218)
Sep 22 23:15:07.843 DEBG 218 Repair extent 150 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:07.843 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/096.copy"
Sep 22 23:15:07.906 INFO accepted connection, remote_addr: 127.0.0.1:56142, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.906 TRCE incoming request, uri: /extent/150/files, method: GET, req_id: 485308e5-feac-4a15-a065-bbb6ab5eff82, remote_addr: 127.0.0.1:56142, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.906 INFO request completed, latency_us: 213, response_code: 200, uri: /extent/150/files, method: GET, req_id: 485308e5-feac-4a15-a065-bbb6ab5eff82, remote_addr: 127.0.0.1:56142, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.907 INFO eid:150 Found repair files: ["096", "096.db"]
Sep 22 23:15:07.907 TRCE incoming request, uri: /newextent/150/data, method: GET, req_id: 25fa0698-c715-466e-931c-f2cec1cdcf94, remote_addr: 127.0.0.1:56142, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.907 INFO request completed, latency_us: 256, response_code: 200, uri: /newextent/150/data, method: GET, req_id: 25fa0698-c715-466e-931c-f2cec1cdcf94, remote_addr: 127.0.0.1:56142, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.912 TRCE incoming request, uri: /newextent/150/db, method: GET, req_id: ba231f0f-72b5-4e75-bf5b-c910f85870ad, remote_addr: 127.0.0.1:56142, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.913 INFO request completed, latency_us: 298, response_code: 200, uri: /newextent/150/db, method: GET, req_id: ba231f0f-72b5-4e75-bf5b-c910f85870ad, remote_addr: 127.0.0.1:56142, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.914 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/096.copy" to "/tmp/downstairs-vrx8aK6L/00/000/096.replace"
Sep 22 23:15:07.914 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:07.915 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/096.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:07.915 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/096"
Sep 22 23:15:07.915 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/096.db"
Sep 22 23:15:07.915 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:07.915 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/096.replace" to "/tmp/downstairs-vrx8aK6L/00/000/096.completed"
Sep 22 23:15:07.915 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:07.915 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:07.915 DEBG [0] It's time to notify for 218
Sep 22 23:15:07.916 INFO Completion from [0] id:218 status:true
Sep 22 23:15:07.916 INFO [219/752] Repair commands completed
Sep 22 23:15:07.916 INFO Pop front: ReconcileIO { id: ReconciliationId(219), op: ExtentReopen { repair_id: ReconciliationId(219), extent_id: 150 }, state: ClientData([New, New, New]) }
Sep 22 23:15:07.916 INFO Sent repair work, now wait for resp
Sep 22 23:15:07.916 INFO [0] received reconcile message
Sep 22 23:15:07.916 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(219), op: ExtentReopen { repair_id: ReconciliationId(219), extent_id: 150 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:07.916 INFO [0] client ExtentReopen { repair_id: ReconciliationId(219), extent_id: 150 }
Sep 22 23:15:07.916 INFO [1] received reconcile message
Sep 22 23:15:07.916 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(219), op: ExtentReopen { repair_id: ReconciliationId(219), extent_id: 150 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:07.916 INFO [1] client ExtentReopen { repair_id: ReconciliationId(219), extent_id: 150 }
Sep 22 23:15:07.916 INFO [2] received reconcile message
Sep 22 23:15:07.916 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(219), op: ExtentReopen { repair_id: ReconciliationId(219), extent_id: 150 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:07.916 INFO [2] client ExtentReopen { repair_id: ReconciliationId(219), extent_id: 150 }
Sep 22 23:15:07.916 DEBG 219 Reopen extent 150
Sep 22 23:15:07.917 DEBG 219 Reopen extent 150
Sep 22 23:15:07.917 DEBG 219 Reopen extent 150
Sep 22 23:15:07.918 DEBG [2] It's time to notify for 219
Sep 22 23:15:07.918 INFO Completion from [2] id:219 status:true
Sep 22 23:15:07.918 INFO [220/752] Repair commands completed
Sep 22 23:15:07.918 INFO Pop front: ReconcileIO { id: ReconciliationId(220), op: ExtentFlush { repair_id: ReconciliationId(220), extent_id: 164, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:07.918 INFO Sent repair work, now wait for resp
Sep 22 23:15:07.918 INFO [0] received reconcile message
Sep 22 23:15:07.918 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(220), op: ExtentFlush { repair_id: ReconciliationId(220), extent_id: 164, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:07.918 INFO [0] client ExtentFlush { repair_id: ReconciliationId(220), extent_id: 164, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:07.918 INFO [1] received reconcile message
Sep 22 23:15:07.918 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(220), op: ExtentFlush { repair_id: ReconciliationId(220), extent_id: 164, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:07.918 INFO [1] client ExtentFlush { repair_id: ReconciliationId(220), extent_id: 164, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:07.918 INFO [2] received reconcile message
Sep 22 23:15:07.918 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(220), op: ExtentFlush { repair_id: ReconciliationId(220), extent_id: 164, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:07.918 INFO [2] client ExtentFlush { repair_id: ReconciliationId(220), extent_id: 164, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:07.918 DEBG 220 Flush extent 164 with f:2 g:2
Sep 22 23:15:07.918 DEBG Flush just extent 164 with f:2 and g:2
Sep 22 23:15:07.918 DEBG [1] It's time to notify for 220
Sep 22 23:15:07.918 INFO Completion from [1] id:220 status:true
Sep 22 23:15:07.918 INFO [221/752] Repair commands completed
Sep 22 23:15:07.918 INFO Pop front: ReconcileIO { id: ReconciliationId(221), op: ExtentClose { repair_id: ReconciliationId(221), extent_id: 164 }, state: ClientData([New, New, New]) }
Sep 22 23:15:07.918 INFO Sent repair work, now wait for resp
Sep 22 23:15:07.918 INFO [0] received reconcile message
Sep 22 23:15:07.918 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(221), op: ExtentClose { repair_id: ReconciliationId(221), extent_id: 164 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:07.918 INFO [0] client ExtentClose { repair_id: ReconciliationId(221), extent_id: 164 }
Sep 22 23:15:07.918 INFO [1] received reconcile message
Sep 22 23:15:07.918 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(221), op: ExtentClose { repair_id: ReconciliationId(221), extent_id: 164 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:07.919 INFO [1] client ExtentClose { repair_id: ReconciliationId(221), extent_id: 164 }
Sep 22 23:15:07.919 INFO [2] received reconcile message
Sep 22 23:15:07.919 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(221), op: ExtentClose { repair_id: ReconciliationId(221), extent_id: 164 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:07.919 INFO [2] client ExtentClose { repair_id: ReconciliationId(221), extent_id: 164 }
Sep 22 23:15:07.919 DEBG 221 Close extent 164
Sep 22 23:15:07.919 DEBG 221 Close extent 164
Sep 22 23:15:07.919 DEBG 221 Close extent 164
Sep 22 23:15:07.920 DEBG [2] It's time to notify for 221
Sep 22 23:15:07.920 INFO Completion from [2] id:221 status:true
Sep 22 23:15:07.920 INFO [222/752] Repair commands completed
Sep 22 23:15:07.920 INFO Pop front: ReconcileIO { id: ReconciliationId(222), op: ExtentRepair { repair_id: ReconciliationId(222), extent_id: 164, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:07.920 INFO Sent repair work, now wait for resp
Sep 22 23:15:07.920 INFO [0] received reconcile message
Sep 22 23:15:07.920 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(222), op: ExtentRepair { repair_id: ReconciliationId(222), extent_id: 164, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:07.920 INFO [0] client ExtentRepair { repair_id: ReconciliationId(222), extent_id: 164, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:07.920 INFO [0] Sending repair request ReconciliationId(222)
Sep 22 23:15:07.920 INFO [1] received reconcile message
Sep 22 23:15:07.920 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(222), op: ExtentRepair { repair_id: ReconciliationId(222), extent_id: 164, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:07.920 INFO [1] client ExtentRepair { repair_id: ReconciliationId(222), extent_id: 164, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:07.920 INFO [1] No action required ReconciliationId(222)
Sep 22 23:15:07.920 INFO [2] received reconcile message
Sep 22 23:15:07.920 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(222), op: ExtentRepair { repair_id: ReconciliationId(222), extent_id: 164, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:07.920 INFO [2] client ExtentRepair { repair_id: ReconciliationId(222), extent_id: 164, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:07.920 INFO [2] No action required ReconciliationId(222)
Sep 22 23:15:07.920 DEBG 222 Repair extent 164 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:07.920 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/0A4.copy"
Sep 22 23:15:07.982 INFO accepted connection, remote_addr: 127.0.0.1:41953, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.982 TRCE incoming request, uri: /extent/164/files, method: GET, req_id: f1347db1-6abe-4744-a5de-1498938e6d10, remote_addr: 127.0.0.1:41953, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.982 INFO request completed, latency_us: 230, response_code: 200, uri: /extent/164/files, method: GET, req_id: f1347db1-6abe-4744-a5de-1498938e6d10, remote_addr: 127.0.0.1:41953, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.983 INFO eid:164 Found repair files: ["0A4", "0A4.db"]
Sep 22 23:15:07.983 TRCE incoming request, uri: /newextent/164/data, method: GET, req_id: cdc6ddd0-c5e9-4729-9fcd-3d9ecfb05af6, remote_addr: 127.0.0.1:41953, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.983 INFO request completed, latency_us: 340, response_code: 200, uri: /newextent/164/data, method: GET, req_id: cdc6ddd0-c5e9-4729-9fcd-3d9ecfb05af6, remote_addr: 127.0.0.1:41953, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.988 TRCE incoming request, uri: /newextent/164/db, method: GET, req_id: 6cb19581-325a-499e-99cc-96959db5020c, remote_addr: 127.0.0.1:41953, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.988 INFO request completed, latency_us: 290, response_code: 200, uri: /newextent/164/db, method: GET, req_id: 6cb19581-325a-499e-99cc-96959db5020c, remote_addr: 127.0.0.1:41953, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:07.990 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/0A4.copy" to "/tmp/downstairs-vrx8aK6L/00/000/0A4.replace"
Sep 22 23:15:07.990 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:07.990 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/0A4.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:07.991 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/0A4"
Sep 22 23:15:07.991 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/0A4.db"
Sep 22 23:15:07.991 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:07.991 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/0A4.replace" to "/tmp/downstairs-vrx8aK6L/00/000/0A4.completed"
Sep 22 23:15:07.991 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:07.991 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:07.991 DEBG [0] It's time to notify for 222
Sep 22 23:15:07.991 INFO Completion from [0] id:222 status:true
Sep 22 23:15:07.991 INFO [223/752] Repair commands completed
Sep 22 23:15:07.991 INFO Pop front: ReconcileIO { id: ReconciliationId(223), op: ExtentReopen { repair_id: ReconciliationId(223), extent_id: 164 }, state: ClientData([New, New, New]) }
Sep 22 23:15:07.991 INFO Sent repair work, now wait for resp
Sep 22 23:15:07.991 INFO [0] received reconcile message
Sep 22 23:15:07.991 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(223), op: ExtentReopen { repair_id: ReconciliationId(223), extent_id: 164 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:07.991 INFO [0] client ExtentReopen { repair_id: ReconciliationId(223), extent_id: 164 }
Sep 22 23:15:07.991 INFO [1] received reconcile message
Sep 22 23:15:07.991 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(223), op: ExtentReopen { repair_id: ReconciliationId(223), extent_id: 164 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:07.991 INFO [1] client ExtentReopen { repair_id: ReconciliationId(223), extent_id: 164 }
Sep 22 23:15:07.992 INFO [2] received reconcile message
Sep 22 23:15:07.992 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(223), op: ExtentReopen { repair_id: ReconciliationId(223), extent_id: 164 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:07.992 INFO [2] client ExtentReopen { repair_id: ReconciliationId(223), extent_id: 164 }
Sep 22 23:15:07.992 DEBG 223 Reopen extent 164
Sep 22 23:15:07.992 DEBG 223 Reopen extent 164
Sep 22 23:15:07.993 DEBG 223 Reopen extent 164
Sep 22 23:15:07.993 DEBG [2] It's time to notify for 223
Sep 22 23:15:07.994 INFO Completion from [2] id:223 status:true
Sep 22 23:15:07.994 INFO [224/752] Repair commands completed
Sep 22 23:15:07.994 INFO Pop front: ReconcileIO { id: ReconciliationId(224), op: ExtentFlush { repair_id: ReconciliationId(224), extent_id: 6, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:07.994 INFO Sent repair work, now wait for resp
Sep 22 23:15:07.994 INFO [0] received reconcile message
Sep 22 23:15:07.994 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(224), op: ExtentFlush { repair_id: ReconciliationId(224), extent_id: 6, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:07.994 INFO [0] client ExtentFlush { repair_id: ReconciliationId(224), extent_id: 6, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:07.994 INFO [1] received reconcile message
Sep 22 23:15:07.994 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(224), op: ExtentFlush { repair_id: ReconciliationId(224), extent_id: 6, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:07.994 INFO [1] client ExtentFlush { repair_id: ReconciliationId(224), extent_id: 6, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:07.994 INFO [2] received reconcile message
Sep 22 23:15:07.994 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(224), op: ExtentFlush { repair_id: ReconciliationId(224), extent_id: 6, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:07.994 INFO [2] client ExtentFlush { repair_id: ReconciliationId(224), extent_id: 6, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:07.994 DEBG 224 Flush extent 6 with f:2 g:2
Sep 22 23:15:07.994 DEBG Flush just extent 6 with f:2 and g:2
Sep 22 23:15:07.994 DEBG [1] It's time to notify for 224
Sep 22 23:15:07.994 INFO Completion from [1] id:224 status:true
Sep 22 23:15:07.994 INFO [225/752] Repair commands completed
Sep 22 23:15:07.994 INFO Pop front: ReconcileIO { id: ReconciliationId(225), op: ExtentClose { repair_id: ReconciliationId(225), extent_id: 6 }, state: ClientData([New, New, New]) }
Sep 22 23:15:07.994 INFO Sent repair work, now wait for resp
Sep 22 23:15:07.994 INFO [0] received reconcile message
Sep 22 23:15:07.994 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(225), op: ExtentClose { repair_id: ReconciliationId(225), extent_id: 6 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:07.994 INFO [0] client ExtentClose { repair_id: ReconciliationId(225), extent_id: 6 }
Sep 22 23:15:07.994 INFO [1] received reconcile message
Sep 22 23:15:07.994 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(225), op: ExtentClose { repair_id: ReconciliationId(225), extent_id: 6 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:07.994 INFO [1] client ExtentClose { repair_id: ReconciliationId(225), extent_id: 6 }
Sep 22 23:15:07.994 INFO [2] received reconcile message
Sep 22 23:15:07.994 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(225), op: ExtentClose { repair_id: ReconciliationId(225), extent_id: 6 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:07.994 INFO [2] client ExtentClose { repair_id: ReconciliationId(225), extent_id: 6 }
Sep 22 23:15:07.995 DEBG 225 Close extent 6
Sep 22 23:15:07.995 DEBG 225 Close extent 6
Sep 22 23:15:07.995 DEBG 225 Close extent 6
Sep 22 23:15:07.995 DEBG [2] It's time to notify for 225
Sep 22 23:15:07.995 INFO Completion from [2] id:225 status:true
Sep 22 23:15:07.996 INFO [226/752] Repair commands completed
Sep 22 23:15:07.996 INFO Pop front: ReconcileIO { id: ReconciliationId(226), op: ExtentRepair { repair_id: ReconciliationId(226), extent_id: 6, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:07.996 INFO Sent repair work, now wait for resp
Sep 22 23:15:07.996 INFO [0] received reconcile message
Sep 22 23:15:07.996 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(226), op: ExtentRepair { repair_id: ReconciliationId(226), extent_id: 6, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:07.996 INFO [0] client ExtentRepair { repair_id: ReconciliationId(226), extent_id: 6, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:07.996 INFO [0] Sending repair request ReconciliationId(226)
Sep 22 23:15:07.996 INFO [1] received reconcile message
Sep 22 23:15:07.996 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(226), op: ExtentRepair { repair_id: ReconciliationId(226), extent_id: 6, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:07.996 INFO [1] client ExtentRepair { repair_id: ReconciliationId(226), extent_id: 6, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:07.996 INFO [1] No action required ReconciliationId(226)
Sep 22 23:15:07.996 INFO [2] received reconcile message
Sep 22 23:15:07.996 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(226), op: ExtentRepair { repair_id: ReconciliationId(226), extent_id: 6, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:07.996 INFO [2] client ExtentRepair { repair_id: ReconciliationId(226), extent_id: 6, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:07.996 INFO [2] No action required ReconciliationId(226)
Sep 22 23:15:07.996 DEBG 226 Repair extent 6 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:07.996 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/006.copy"
Sep 22 23:15:08.062 INFO accepted connection, remote_addr: 127.0.0.1:53023, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.062 TRCE incoming request, uri: /extent/6/files, method: GET, req_id: 3c6e4bb7-3232-4151-b908-91455dd1ddf8, remote_addr: 127.0.0.1:53023, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.062 INFO request completed, latency_us: 273, response_code: 200, uri: /extent/6/files, method: GET, req_id: 3c6e4bb7-3232-4151-b908-91455dd1ddf8, remote_addr: 127.0.0.1:53023, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.063 INFO eid:6 Found repair files: ["006", "006.db"]
Sep 22 23:15:08.063 TRCE incoming request, uri: /newextent/6/data, method: GET, req_id: d601c36f-2536-4edf-8e17-04ad840d7ed9, remote_addr: 127.0.0.1:53023, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.063 INFO request completed, latency_us: 354, response_code: 200, uri: /newextent/6/data, method: GET, req_id: d601c36f-2536-4edf-8e17-04ad840d7ed9, remote_addr: 127.0.0.1:53023, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.068 TRCE incoming request, uri: /newextent/6/db, method: GET, req_id: d33e26a8-722a-41a1-affd-84a567874bef, remote_addr: 127.0.0.1:53023, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.069 INFO request completed, latency_us: 291, response_code: 200, uri: /newextent/6/db, method: GET, req_id: d33e26a8-722a-41a1-affd-84a567874bef, remote_addr: 127.0.0.1:53023, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.070 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/006.copy" to "/tmp/downstairs-vrx8aK6L/00/000/006.replace"
Sep 22 23:15:08.070 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:08.071 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/006.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:08.071 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/006"
Sep 22 23:15:08.071 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/006.db"
Sep 22 23:15:08.071 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:08.071 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/006.replace" to "/tmp/downstairs-vrx8aK6L/00/000/006.completed"
Sep 22 23:15:08.071 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:08.072 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:08.072 DEBG [0] It's time to notify for 226
Sep 22 23:15:08.072 INFO Completion from [0] id:226 status:true
Sep 22 23:15:08.072 INFO [227/752] Repair commands completed
Sep 22 23:15:08.072 INFO Pop front: ReconcileIO { id: ReconciliationId(227), op: ExtentReopen { repair_id: ReconciliationId(227), extent_id: 6 }, state: ClientData([New, New, New]) }
Sep 22 23:15:08.072 INFO Sent repair work, now wait for resp
Sep 22 23:15:08.072 INFO [0] received reconcile message
Sep 22 23:15:08.072 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(227), op: ExtentReopen { repair_id: ReconciliationId(227), extent_id: 6 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:08.072 INFO [0] client ExtentReopen { repair_id: ReconciliationId(227), extent_id: 6 }
Sep 22 23:15:08.072 INFO [1] received reconcile message
Sep 22 23:15:08.072 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(227), op: ExtentReopen { repair_id: ReconciliationId(227), extent_id: 6 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:08.072 INFO [1] client ExtentReopen { repair_id: ReconciliationId(227), extent_id: 6 }
Sep 22 23:15:08.072 INFO [2] received reconcile message
Sep 22 23:15:08.072 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(227), op: ExtentReopen { repair_id: ReconciliationId(227), extent_id: 6 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:08.072 INFO [2] client ExtentReopen { repair_id: ReconciliationId(227), extent_id: 6 }
Sep 22 23:15:08.072 DEBG 227 Reopen extent 6
Sep 22 23:15:08.073 DEBG 227 Reopen extent 6
Sep 22 23:15:08.074 DEBG 227 Reopen extent 6
Sep 22 23:15:08.074 DEBG [2] It's time to notify for 227
Sep 22 23:15:08.074 INFO Completion from [2] id:227 status:true
Sep 22 23:15:08.074 INFO [228/752] Repair commands completed
Sep 22 23:15:08.074 INFO Pop front: ReconcileIO { id: ReconciliationId(228), op: ExtentFlush { repair_id: ReconciliationId(228), extent_id: 52, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:08.075 INFO Sent repair work, now wait for resp
Sep 22 23:15:08.075 INFO [0] received reconcile message
Sep 22 23:15:08.075 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(228), op: ExtentFlush { repair_id: ReconciliationId(228), extent_id: 52, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:08.075 INFO [0] client ExtentFlush { repair_id: ReconciliationId(228), extent_id: 52, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:08.075 INFO [1] received reconcile message
Sep 22 23:15:08.075 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(228), op: ExtentFlush { repair_id: ReconciliationId(228), extent_id: 52, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:08.075 INFO [1] client ExtentFlush { repair_id: ReconciliationId(228), extent_id: 52, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:08.075 INFO [2] received reconcile message
Sep 22 23:15:08.075 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(228), op: ExtentFlush { repair_id: ReconciliationId(228), extent_id: 52, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:08.075 INFO [2] client ExtentFlush { repair_id: ReconciliationId(228), extent_id: 52, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:08.075 DEBG 228 Flush extent 52 with f:2 g:2
Sep 22 23:15:08.075 DEBG Flush just extent 52 with f:2 and g:2
Sep 22 23:15:08.075 DEBG [1] It's time to notify for 228
Sep 22 23:15:08.075 INFO Completion from [1] id:228 status:true
Sep 22 23:15:08.075 INFO [229/752] Repair commands completed
Sep 22 23:15:08.075 INFO Pop front: ReconcileIO { id: ReconciliationId(229), op: ExtentClose { repair_id: ReconciliationId(229), extent_id: 52 }, state: ClientData([New, New, New]) }
Sep 22 23:15:08.075 INFO Sent repair work, now wait for resp
Sep 22 23:15:08.075 INFO [0] received reconcile message
Sep 22 23:15:08.075 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(229), op: ExtentClose { repair_id: ReconciliationId(229), extent_id: 52 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:08.075 INFO [0] client ExtentClose { repair_id: ReconciliationId(229), extent_id: 52 }
Sep 22 23:15:08.075 INFO [1] received reconcile message
Sep 22 23:15:08.075 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(229), op: ExtentClose { repair_id: ReconciliationId(229), extent_id: 52 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:08.075 INFO [1] client ExtentClose { repair_id: ReconciliationId(229), extent_id: 52 }
Sep 22 23:15:08.075 INFO [2] received reconcile message
Sep 22 23:15:08.075 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(229), op: ExtentClose { repair_id: ReconciliationId(229), extent_id: 52 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:08.075 INFO [2] client ExtentClose { repair_id: ReconciliationId(229), extent_id: 52 }
Sep 22 23:15:08.075 DEBG 229 Close extent 52
Sep 22 23:15:08.076 DEBG 229 Close extent 52
Sep 22 23:15:08.076 DEBG 229 Close extent 52
Sep 22 23:15:08.076 DEBG [2] It's time to notify for 229
Sep 22 23:15:08.076 INFO Completion from [2] id:229 status:true
Sep 22 23:15:08.076 INFO [230/752] Repair commands completed
Sep 22 23:15:08.076 INFO Pop front: ReconcileIO { id: ReconciliationId(230), op: ExtentRepair { repair_id: ReconciliationId(230), extent_id: 52, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:08.076 INFO Sent repair work, now wait for resp
Sep 22 23:15:08.077 INFO [0] received reconcile message
Sep 22 23:15:08.077 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(230), op: ExtentRepair { repair_id: ReconciliationId(230), extent_id: 52, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:08.077 INFO [0] client ExtentRepair { repair_id: ReconciliationId(230), extent_id: 52, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:08.077 INFO [0] Sending repair request ReconciliationId(230)
Sep 22 23:15:08.077 INFO [1] received reconcile message
Sep 22 23:15:08.077 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(230), op: ExtentRepair { repair_id: ReconciliationId(230), extent_id: 52, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:08.077 INFO [1] client ExtentRepair { repair_id: ReconciliationId(230), extent_id: 52, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:08.077 INFO [1] No action required ReconciliationId(230)
Sep 22 23:15:08.077 INFO [2] received reconcile message
Sep 22 23:15:08.077 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(230), op: ExtentRepair { repair_id: ReconciliationId(230), extent_id: 52, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:08.077 INFO [2] client ExtentRepair { repair_id: ReconciliationId(230), extent_id: 52, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:08.077 INFO [2] No action required ReconciliationId(230)
Sep 22 23:15:08.077 DEBG 230 Repair extent 52 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:08.077 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/034.copy"
Sep 22 23:15:08.115 DEBG [1] Read AckReady 1078, : downstairs
Sep 22 23:15:08.117 DEBG up_ds_listen was notified
Sep 22 23:15:08.117 DEBG up_ds_listen process 1078
Sep 22 23:15:08.117 DEBG [A] ack job 1078:79, : downstairs
Sep 22 23:15:08.139 INFO accepted connection, remote_addr: 127.0.0.1:40497, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.139 TRCE incoming request, uri: /extent/52/files, method: GET, req_id: 6f0dc614-ab4b-4240-9a0e-ac44346765fb, remote_addr: 127.0.0.1:40497, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.140 INFO request completed, latency_us: 236, response_code: 200, uri: /extent/52/files, method: GET, req_id: 6f0dc614-ab4b-4240-9a0e-ac44346765fb, remote_addr: 127.0.0.1:40497, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.140 INFO eid:52 Found repair files: ["034", "034.db"]
Sep 22 23:15:08.140 TRCE incoming request, uri: /newextent/52/data, method: GET, req_id: 8012db34-2bfd-4f35-9899-fccf1fb7484b, remote_addr: 127.0.0.1:40497, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.141 INFO request completed, latency_us: 343, response_code: 200, uri: /newextent/52/data, method: GET, req_id: 8012db34-2bfd-4f35-9899-fccf1fb7484b, remote_addr: 127.0.0.1:40497, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.146 TRCE incoming request, uri: /newextent/52/db, method: GET, req_id: ff883fb6-6497-4197-ab81-eee86b2297d9, remote_addr: 127.0.0.1:40497, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.146 INFO request completed, latency_us: 296, response_code: 200, uri: /newextent/52/db, method: GET, req_id: ff883fb6-6497-4197-ab81-eee86b2297d9, remote_addr: 127.0.0.1:40497, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.147 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/034.copy" to "/tmp/downstairs-vrx8aK6L/00/000/034.replace"
Sep 22 23:15:08.147 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:08.148 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/034.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:08.148 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/034"
Sep 22 23:15:08.148 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/034.db"
Sep 22 23:15:08.149 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:08.149 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/034.replace" to "/tmp/downstairs-vrx8aK6L/00/000/034.completed"
Sep 22 23:15:08.149 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:08.149 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:08.149 DEBG [0] It's time to notify for 230
Sep 22 23:15:08.149 INFO Completion from [0] id:230 status:true
Sep 22 23:15:08.149 INFO [231/752] Repair commands completed
Sep 22 23:15:08.149 INFO Pop front: ReconcileIO { id: ReconciliationId(231), op: ExtentReopen { repair_id: ReconciliationId(231), extent_id: 52 }, state: ClientData([New, New, New]) }
Sep 22 23:15:08.149 INFO Sent repair work, now wait for resp
Sep 22 23:15:08.149 INFO [0] received reconcile message
Sep 22 23:15:08.149 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(231), op: ExtentReopen { repair_id: ReconciliationId(231), extent_id: 52 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:08.149 INFO [0] client ExtentReopen { repair_id: ReconciliationId(231), extent_id: 52 }
Sep 22 23:15:08.149 INFO [1] received reconcile message
Sep 22 23:15:08.149 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(231), op: ExtentReopen { repair_id: ReconciliationId(231), extent_id: 52 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:08.149 INFO [1] client ExtentReopen { repair_id: ReconciliationId(231), extent_id: 52 }
Sep 22 23:15:08.149 INFO [2] received reconcile message
Sep 22 23:15:08.149 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(231), op: ExtentReopen { repair_id: ReconciliationId(231), extent_id: 52 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:08.149 INFO [2] client ExtentReopen { repair_id: ReconciliationId(231), extent_id: 52 }
Sep 22 23:15:08.149 DEBG 231 Reopen extent 52
Sep 22 23:15:08.150 DEBG 231 Reopen extent 52
Sep 22 23:15:08.151 DEBG 231 Reopen extent 52
Sep 22 23:15:08.151 DEBG [2] It's time to notify for 231
Sep 22 23:15:08.151 INFO Completion from [2] id:231 status:true
Sep 22 23:15:08.151 INFO [232/752] Repair commands completed
Sep 22 23:15:08.151 INFO Pop front: ReconcileIO { id: ReconciliationId(232), op: ExtentFlush { repair_id: ReconciliationId(232), extent_id: 19, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:08.151 INFO Sent repair work, now wait for resp
Sep 22 23:15:08.151 INFO [0] received reconcile message
Sep 22 23:15:08.151 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(232), op: ExtentFlush { repair_id: ReconciliationId(232), extent_id: 19, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:08.151 INFO [0] client ExtentFlush { repair_id: ReconciliationId(232), extent_id: 19, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:08.152 INFO [1] received reconcile message
Sep 22 23:15:08.152 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(232), op: ExtentFlush { repair_id: ReconciliationId(232), extent_id: 19, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:08.152 INFO [1] client ExtentFlush { repair_id: ReconciliationId(232), extent_id: 19, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:08.152 INFO [2] received reconcile message
Sep 22 23:15:08.152 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(232), op: ExtentFlush { repair_id: ReconciliationId(232), extent_id: 19, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:08.152 INFO [2] client ExtentFlush { repair_id: ReconciliationId(232), extent_id: 19, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:08.152 DEBG 232 Flush extent 19 with f:2 g:2
Sep 22 23:15:08.152 DEBG Flush just extent 19 with f:2 and g:2
Sep 22 23:15:08.152 DEBG [1] It's time to notify for 232
Sep 22 23:15:08.152 INFO Completion from [1] id:232 status:true
Sep 22 23:15:08.152 INFO [233/752] Repair commands completed
Sep 22 23:15:08.152 INFO Pop front: ReconcileIO { id: ReconciliationId(233), op: ExtentClose { repair_id: ReconciliationId(233), extent_id: 19 }, state: ClientData([New, New, New]) }
Sep 22 23:15:08.152 INFO Sent repair work, now wait for resp
Sep 22 23:15:08.152 INFO [0] received reconcile message
Sep 22 23:15:08.152 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(233), op: ExtentClose { repair_id: ReconciliationId(233), extent_id: 19 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:08.152 INFO [0] client ExtentClose { repair_id: ReconciliationId(233), extent_id: 19 }
Sep 22 23:15:08.152 INFO [1] received reconcile message
Sep 22 23:15:08.152 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(233), op: ExtentClose { repair_id: ReconciliationId(233), extent_id: 19 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:08.152 INFO [1] client ExtentClose { repair_id: ReconciliationId(233), extent_id: 19 }
Sep 22 23:15:08.152 INFO [2] received reconcile message
Sep 22 23:15:08.152 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(233), op: ExtentClose { repair_id: ReconciliationId(233), extent_id: 19 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:08.152 INFO [2] client ExtentClose { repair_id: ReconciliationId(233), extent_id: 19 }
Sep 22 23:15:08.152 DEBG 233 Close extent 19
Sep 22 23:15:08.153 DEBG 233 Close extent 19
Sep 22 23:15:08.153 DEBG 233 Close extent 19
Sep 22 23:15:08.153 DEBG [2] It's time to notify for 233
Sep 22 23:15:08.153 INFO Completion from [2] id:233 status:true
Sep 22 23:15:08.153 INFO [234/752] Repair commands completed
Sep 22 23:15:08.153 INFO Pop front: ReconcileIO { id: ReconciliationId(234), op: ExtentRepair { repair_id: ReconciliationId(234), extent_id: 19, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:08.153 INFO Sent repair work, now wait for resp
Sep 22 23:15:08.153 INFO [0] received reconcile message
Sep 22 23:15:08.153 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(234), op: ExtentRepair { repair_id: ReconciliationId(234), extent_id: 19, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:08.153 INFO [0] client ExtentRepair { repair_id: ReconciliationId(234), extent_id: 19, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:08.153 INFO [0] Sending repair request ReconciliationId(234)
Sep 22 23:15:08.153 INFO [1] received reconcile message
Sep 22 23:15:08.154 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(234), op: ExtentRepair { repair_id: ReconciliationId(234), extent_id: 19, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:08.154 INFO [1] client ExtentRepair { repair_id: ReconciliationId(234), extent_id: 19, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:08.154 INFO [1] No action required ReconciliationId(234)
Sep 22 23:15:08.154 INFO [2] received reconcile message
Sep 22 23:15:08.154 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(234), op: ExtentRepair { repair_id: ReconciliationId(234), extent_id: 19, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:08.154 INFO [2] client ExtentRepair { repair_id: ReconciliationId(234), extent_id: 19, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:08.154 INFO [2] No action required ReconciliationId(234)
Sep 22 23:15:08.154 DEBG 234 Repair extent 19 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:08.154 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/013.copy"
Sep 22 23:15:08.170 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:08.216 INFO accepted connection, remote_addr: 127.0.0.1:39244, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.216 TRCE incoming request, uri: /extent/19/files, method: GET, req_id: 9b0db1c6-7264-421b-a456-30ef20c327e0, remote_addr: 127.0.0.1:39244, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.217 INFO request completed, latency_us: 216, response_code: 200, uri: /extent/19/files, method: GET, req_id: 9b0db1c6-7264-421b-a456-30ef20c327e0, remote_addr: 127.0.0.1:39244, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.217 INFO eid:19 Found repair files: ["013", "013.db"]
Sep 22 23:15:08.217 TRCE incoming request, uri: /newextent/19/data, method: GET, req_id: dd66a06d-be55-4f7a-954c-c287f356db99, remote_addr: 127.0.0.1:39244, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.217 INFO request completed, latency_us: 329, response_code: 200, uri: /newextent/19/data, method: GET, req_id: dd66a06d-be55-4f7a-954c-c287f356db99, remote_addr: 127.0.0.1:39244, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.222 TRCE incoming request, uri: /newextent/19/db, method: GET, req_id: 5ce3ed06-6c58-4a2b-8693-faa45cbb4e78, remote_addr: 127.0.0.1:39244, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.223 INFO request completed, latency_us: 292, response_code: 200, uri: /newextent/19/db, method: GET, req_id: 5ce3ed06-6c58-4a2b-8693-faa45cbb4e78, remote_addr: 127.0.0.1:39244, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.224 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/013.copy" to "/tmp/downstairs-vrx8aK6L/00/000/013.replace"
Sep 22 23:15:08.224 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:08.225 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/013.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:08.225 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/013"
Sep 22 23:15:08.225 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/013.db"
Sep 22 23:15:08.225 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:08.225 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/013.replace" to "/tmp/downstairs-vrx8aK6L/00/000/013.completed"
Sep 22 23:15:08.225 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:08.225 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:08.226 DEBG [0] It's time to notify for 234
Sep 22 23:15:08.226 INFO Completion from [0] id:234 status:true
Sep 22 23:15:08.226 INFO [235/752] Repair commands completed
Sep 22 23:15:08.226 INFO Pop front: ReconcileIO { id: ReconciliationId(235), op: ExtentReopen { repair_id: ReconciliationId(235), extent_id: 19 }, state: ClientData([New, New, New]) }
Sep 22 23:15:08.226 INFO Sent repair work, now wait for resp
Sep 22 23:15:08.226 INFO [0] received reconcile message
Sep 22 23:15:08.226 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(235), op: ExtentReopen { repair_id: ReconciliationId(235), extent_id: 19 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:08.226 INFO [0] client ExtentReopen { repair_id: ReconciliationId(235), extent_id: 19 }
Sep 22 23:15:08.226 INFO [1] received reconcile message
Sep 22 23:15:08.226 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(235), op: ExtentReopen { repair_id: ReconciliationId(235), extent_id: 19 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:08.226 INFO [1] client ExtentReopen { repair_id: ReconciliationId(235), extent_id: 19 }
Sep 22 23:15:08.226 INFO [2] received reconcile message
Sep 22 23:15:08.226 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(235), op: ExtentReopen { repair_id: ReconciliationId(235), extent_id: 19 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:08.226 INFO [2] client ExtentReopen { repair_id: ReconciliationId(235), extent_id: 19 }
Sep 22 23:15:08.226 DEBG 235 Reopen extent 19
Sep 22 23:15:08.227 DEBG 235 Reopen extent 19
Sep 22 23:15:08.227 DEBG 235 Reopen extent 19
Sep 22 23:15:08.228 DEBG [2] It's time to notify for 235
Sep 22 23:15:08.228 INFO Completion from [2] id:235 status:true
Sep 22 23:15:08.228 INFO [236/752] Repair commands completed
Sep 22 23:15:08.228 INFO Pop front: ReconcileIO { id: ReconciliationId(236), op: ExtentFlush { repair_id: ReconciliationId(236), extent_id: 79, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:08.228 INFO Sent repair work, now wait for resp
Sep 22 23:15:08.228 INFO [0] received reconcile message
Sep 22 23:15:08.228 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(236), op: ExtentFlush { repair_id: ReconciliationId(236), extent_id: 79, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:08.228 INFO [0] client ExtentFlush { repair_id: ReconciliationId(236), extent_id: 79, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:08.228 INFO [1] received reconcile message
Sep 22 23:15:08.228 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(236), op: ExtentFlush { repair_id: ReconciliationId(236), extent_id: 79, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:08.228 INFO [1] client ExtentFlush { repair_id: ReconciliationId(236), extent_id: 79, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:08.228 INFO [2] received reconcile message
Sep 22 23:15:08.228 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(236), op: ExtentFlush { repair_id: ReconciliationId(236), extent_id: 79, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:08.228 INFO [2] client ExtentFlush { repair_id: ReconciliationId(236), extent_id: 79, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:08.228 DEBG 236 Flush extent 79 with f:2 g:2
Sep 22 23:15:08.228 DEBG Flush just extent 79 with f:2 and g:2
Sep 22 23:15:08.229 DEBG [1] It's time to notify for 236
Sep 22 23:15:08.229 INFO Completion from [1] id:236 status:true
Sep 22 23:15:08.229 INFO [237/752] Repair commands completed
Sep 22 23:15:08.229 INFO Pop front: ReconcileIO { id: ReconciliationId(237), op: ExtentClose { repair_id: ReconciliationId(237), extent_id: 79 }, state: ClientData([New, New, New]) }
Sep 22 23:15:08.229 INFO Sent repair work, now wait for resp
Sep 22 23:15:08.229 INFO [0] received reconcile message
Sep 22 23:15:08.229 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(237), op: ExtentClose { repair_id: ReconciliationId(237), extent_id: 79 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:08.229 INFO [0] client ExtentClose { repair_id: ReconciliationId(237), extent_id: 79 }
Sep 22 23:15:08.229 INFO [1] received reconcile message
Sep 22 23:15:08.229 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(237), op: ExtentClose { repair_id: ReconciliationId(237), extent_id: 79 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:08.229 INFO [1] client ExtentClose { repair_id: ReconciliationId(237), extent_id: 79 }
Sep 22 23:15:08.229 INFO [2] received reconcile message
Sep 22 23:15:08.229 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(237), op: ExtentClose { repair_id: ReconciliationId(237), extent_id: 79 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:08.229 INFO [2] client ExtentClose { repair_id: ReconciliationId(237), extent_id: 79 }
Sep 22 23:15:08.229 DEBG 237 Close extent 79
Sep 22 23:15:08.229 DEBG 237 Close extent 79
Sep 22 23:15:08.230 DEBG 237 Close extent 79
Sep 22 23:15:08.230 DEBG [2] It's time to notify for 237
Sep 22 23:15:08.230 INFO Completion from [2] id:237 status:true
Sep 22 23:15:08.230 INFO [238/752] Repair commands completed
Sep 22 23:15:08.230 INFO Pop front: ReconcileIO { id: ReconciliationId(238), op: ExtentRepair { repair_id: ReconciliationId(238), extent_id: 79, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:08.230 INFO Sent repair work, now wait for resp
Sep 22 23:15:08.230 INFO [0] received reconcile message
Sep 22 23:15:08.230 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(238), op: ExtentRepair { repair_id: ReconciliationId(238), extent_id: 79, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:08.230 INFO [0] client ExtentRepair { repair_id: ReconciliationId(238), extent_id: 79, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:08.230 INFO [0] Sending repair request ReconciliationId(238)
Sep 22 23:15:08.230 INFO [1] received reconcile message
Sep 22 23:15:08.230 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(238), op: ExtentRepair { repair_id: ReconciliationId(238), extent_id: 79, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:08.230 INFO [1] client ExtentRepair { repair_id: ReconciliationId(238), extent_id: 79, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:08.230 INFO [1] No action required ReconciliationId(238)
Sep 22 23:15:08.230 INFO [2] received reconcile message
Sep 22 23:15:08.230 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(238), op: ExtentRepair { repair_id: ReconciliationId(238), extent_id: 79, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:08.230 INFO [2] client ExtentRepair { repair_id: ReconciliationId(238), extent_id: 79, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:08.230 INFO [2] No action required ReconciliationId(238)
Sep 22 23:15:08.230 DEBG 238 Repair extent 79 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:08.230 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/04F.copy"
Sep 22 23:15:08.294 INFO accepted connection, remote_addr: 127.0.0.1:56211, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.294 TRCE incoming request, uri: /extent/79/files, method: GET, req_id: 1a0ce382-95e0-4b8c-8f36-e28d23b9430e, remote_addr: 127.0.0.1:56211, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.294 INFO request completed, latency_us: 220, response_code: 200, uri: /extent/79/files, method: GET, req_id: 1a0ce382-95e0-4b8c-8f36-e28d23b9430e, remote_addr: 127.0.0.1:56211, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.294 INFO eid:79 Found repair files: ["04F", "04F.db"]
Sep 22 23:15:08.295 TRCE incoming request, uri: /newextent/79/data, method: GET, req_id: da00ece0-ea5a-41fa-9563-12a7df3dccdd, remote_addr: 127.0.0.1:56211, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.295 INFO request completed, latency_us: 318, response_code: 200, uri: /newextent/79/data, method: GET, req_id: da00ece0-ea5a-41fa-9563-12a7df3dccdd, remote_addr: 127.0.0.1:56211, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.300 TRCE incoming request, uri: /newextent/79/db, method: GET, req_id: abd7909e-0c1d-4c69-918a-fab5412e6c8f, remote_addr: 127.0.0.1:56211, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.300 INFO request completed, latency_us: 296, response_code: 200, uri: /newextent/79/db, method: GET, req_id: abd7909e-0c1d-4c69-918a-fab5412e6c8f, remote_addr: 127.0.0.1:56211, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.301 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/04F.copy" to "/tmp/downstairs-vrx8aK6L/00/000/04F.replace"
Sep 22 23:15:08.301 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:08.302 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/04F.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:08.303 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/04F"
Sep 22 23:15:08.303 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/04F.db"
Sep 22 23:15:08.303 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:08.303 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/04F.replace" to "/tmp/downstairs-vrx8aK6L/00/000/04F.completed"
Sep 22 23:15:08.303 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:08.303 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:08.303 DEBG [0] It's time to notify for 238
Sep 22 23:15:08.303 INFO Completion from [0] id:238 status:true
Sep 22 23:15:08.303 INFO [239/752] Repair commands completed
Sep 22 23:15:08.303 INFO Pop front: ReconcileIO { id: ReconciliationId(239), op: ExtentReopen { repair_id: ReconciliationId(239), extent_id: 79 }, state: ClientData([New, New, New]) }
Sep 22 23:15:08.303 INFO Sent repair work, now wait for resp
Sep 22 23:15:08.303 INFO [0] received reconcile message
Sep 22 23:15:08.303 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(239), op: ExtentReopen { repair_id: ReconciliationId(239), extent_id: 79 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:08.303 INFO [0] client ExtentReopen { repair_id: ReconciliationId(239), extent_id: 79 }
Sep 22 23:15:08.303 INFO [1] received reconcile message
Sep 22 23:15:08.303 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(239), op: ExtentReopen { repair_id: ReconciliationId(239), extent_id: 79 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:08.303 INFO [1] client ExtentReopen { repair_id: ReconciliationId(239), extent_id: 79 }
Sep 22 23:15:08.303 INFO [2] received reconcile message
Sep 22 23:15:08.304 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(239), op: ExtentReopen { repair_id: ReconciliationId(239), extent_id: 79 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:08.304 INFO [2] client ExtentReopen { repair_id: ReconciliationId(239), extent_id: 79 }
Sep 22 23:15:08.304 DEBG 239 Reopen extent 79
Sep 22 23:15:08.304 DEBG 239 Reopen extent 79
Sep 22 23:15:08.305 DEBG 239 Reopen extent 79
Sep 22 23:15:08.305 DEBG [2] It's time to notify for 239
Sep 22 23:15:08.305 INFO Completion from [2] id:239 status:true
Sep 22 23:15:08.305 INFO [240/752] Repair commands completed
Sep 22 23:15:08.306 INFO Pop front: ReconcileIO { id: ReconciliationId(240), op: ExtentFlush { repair_id: ReconciliationId(240), extent_id: 97, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:08.306 INFO Sent repair work, now wait for resp
Sep 22 23:15:08.306 INFO [0] received reconcile message
Sep 22 23:15:08.306 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(240), op: ExtentFlush { repair_id: ReconciliationId(240), extent_id: 97, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:08.306 INFO [0] client ExtentFlush { repair_id: ReconciliationId(240), extent_id: 97, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:08.306 INFO [1] received reconcile message
Sep 22 23:15:08.306 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(240), op: ExtentFlush { repair_id: ReconciliationId(240), extent_id: 97, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:08.306 INFO [1] client ExtentFlush { repair_id: ReconciliationId(240), extent_id: 97, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:08.306 INFO [2] received reconcile message
Sep 22 23:15:08.306 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(240), op: ExtentFlush { repair_id: ReconciliationId(240), extent_id: 97, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:08.306 INFO [2] client ExtentFlush { repair_id: ReconciliationId(240), extent_id: 97, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:08.306 DEBG 240 Flush extent 97 with f:2 g:2
Sep 22 23:15:08.306 DEBG Flush just extent 97 with f:2 and g:2
Sep 22 23:15:08.306 DEBG [1] It's time to notify for 240
Sep 22 23:15:08.306 INFO Completion from [1] id:240 status:true
Sep 22 23:15:08.306 INFO [241/752] Repair commands completed
Sep 22 23:15:08.306 INFO Pop front: ReconcileIO { id: ReconciliationId(241), op: ExtentClose { repair_id: ReconciliationId(241), extent_id: 97 }, state: ClientData([New, New, New]) }
Sep 22 23:15:08.306 INFO Sent repair work, now wait for resp
Sep 22 23:15:08.306 INFO [0] received reconcile message
Sep 22 23:15:08.306 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(241), op: ExtentClose { repair_id: ReconciliationId(241), extent_id: 97 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:08.306 INFO [0] client ExtentClose { repair_id: ReconciliationId(241), extent_id: 97 }
Sep 22 23:15:08.306 INFO [1] received reconcile message
Sep 22 23:15:08.306 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(241), op: ExtentClose { repair_id: ReconciliationId(241), extent_id: 97 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:08.306 INFO [1] client ExtentClose { repair_id: ReconciliationId(241), extent_id: 97 }
Sep 22 23:15:08.306 INFO [2] received reconcile message
Sep 22 23:15:08.306 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(241), op: ExtentClose { repair_id: ReconciliationId(241), extent_id: 97 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:08.306 INFO [2] client ExtentClose { repair_id: ReconciliationId(241), extent_id: 97 }
Sep 22 23:15:08.306 DEBG 241 Close extent 97
Sep 22 23:15:08.307 DEBG 241 Close extent 97
Sep 22 23:15:08.307 DEBG 241 Close extent 97
Sep 22 23:15:08.307 DEBG [2] It's time to notify for 241
Sep 22 23:15:08.307 INFO Completion from [2] id:241 status:true
Sep 22 23:15:08.307 INFO [242/752] Repair commands completed
Sep 22 23:15:08.307 INFO Pop front: ReconcileIO { id: ReconciliationId(242), op: ExtentRepair { repair_id: ReconciliationId(242), extent_id: 97, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:08.308 INFO Sent repair work, now wait for resp
Sep 22 23:15:08.308 INFO [0] received reconcile message
Sep 22 23:15:08.308 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(242), op: ExtentRepair { repair_id: ReconciliationId(242), extent_id: 97, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:08.308 INFO [0] client ExtentRepair { repair_id: ReconciliationId(242), extent_id: 97, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:08.308 INFO [0] Sending repair request ReconciliationId(242)
Sep 22 23:15:08.308 INFO [1] received reconcile message
Sep 22 23:15:08.308 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(242), op: ExtentRepair { repair_id: ReconciliationId(242), extent_id: 97, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:08.308 INFO [1] client ExtentRepair { repair_id: ReconciliationId(242), extent_id: 97, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:08.308 INFO [1] No action required ReconciliationId(242)
Sep 22 23:15:08.308 INFO [2] received reconcile message
Sep 22 23:15:08.308 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(242), op: ExtentRepair { repair_id: ReconciliationId(242), extent_id: 97, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:08.308 INFO [2] client ExtentRepair { repair_id: ReconciliationId(242), extent_id: 97, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:08.308 INFO [2] No action required ReconciliationId(242)
Sep 22 23:15:08.308 DEBG 242 Repair extent 97 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:08.308 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/061.copy"
Sep 22 23:15:08.369 INFO accepted connection, remote_addr: 127.0.0.1:56678, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.369 TRCE incoming request, uri: /extent/97/files, method: GET, req_id: 4397840c-4b10-4bee-a068-243448318d36, remote_addr: 127.0.0.1:56678, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.370 INFO request completed, latency_us: 241, response_code: 200, uri: /extent/97/files, method: GET, req_id: 4397840c-4b10-4bee-a068-243448318d36, remote_addr: 127.0.0.1:56678, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.370 INFO eid:97 Found repair files: ["061", "061.db"]
Sep 22 23:15:08.370 TRCE incoming request, uri: /newextent/97/data, method: GET, req_id: 81bf7306-62b8-41af-838b-be81d4466c8d, remote_addr: 127.0.0.1:56678, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.371 INFO request completed, latency_us: 357, response_code: 200, uri: /newextent/97/data, method: GET, req_id: 81bf7306-62b8-41af-838b-be81d4466c8d, remote_addr: 127.0.0.1:56678, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.376 TRCE incoming request, uri: /newextent/97/db, method: GET, req_id: c10fc532-7858-4dc6-9ce1-a6ee83ab8287, remote_addr: 127.0.0.1:56678, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.376 INFO request completed, latency_us: 293, response_code: 200, uri: /newextent/97/db, method: GET, req_id: c10fc532-7858-4dc6-9ce1-a6ee83ab8287, remote_addr: 127.0.0.1:56678, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.377 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/061.copy" to "/tmp/downstairs-vrx8aK6L/00/000/061.replace"
Sep 22 23:15:08.377 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:08.378 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/061.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:08.379 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/061"
Sep 22 23:15:08.379 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/061.db"
Sep 22 23:15:08.379 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:08.379 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/061.replace" to "/tmp/downstairs-vrx8aK6L/00/000/061.completed"
Sep 22 23:15:08.379 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:08.379 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:08.379 DEBG [0] It's time to notify for 242
Sep 22 23:15:08.379 INFO Completion from [0] id:242 status:true
Sep 22 23:15:08.379 INFO [243/752] Repair commands completed
Sep 22 23:15:08.379 INFO Pop front: ReconcileIO { id: ReconciliationId(243), op: ExtentReopen { repair_id: ReconciliationId(243), extent_id: 97 }, state: ClientData([New, New, New]) }
Sep 22 23:15:08.379 INFO Sent repair work, now wait for resp
Sep 22 23:15:08.379 INFO [0] received reconcile message
Sep 22 23:15:08.379 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(243), op: ExtentReopen { repair_id: ReconciliationId(243), extent_id: 97 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:08.379 INFO [0] client ExtentReopen { repair_id: ReconciliationId(243), extent_id: 97 }
Sep 22 23:15:08.379 INFO [1] received reconcile message
Sep 22 23:15:08.379 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(243), op: ExtentReopen { repair_id: ReconciliationId(243), extent_id: 97 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:08.379 INFO [1] client ExtentReopen { repair_id: ReconciliationId(243), extent_id: 97 }
Sep 22 23:15:08.379 INFO [2] received reconcile message
Sep 22 23:15:08.379 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(243), op: ExtentReopen { repair_id: ReconciliationId(243), extent_id: 97 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:08.379 INFO [2] client ExtentReopen { repair_id: ReconciliationId(243), extent_id: 97 }
Sep 22 23:15:08.380 DEBG 243 Reopen extent 97
Sep 22 23:15:08.380 DEBG 243 Reopen extent 97
Sep 22 23:15:08.381 DEBG 243 Reopen extent 97
Sep 22 23:15:08.381 DEBG [2] It's time to notify for 243
Sep 22 23:15:08.381 INFO Completion from [2] id:243 status:true
Sep 22 23:15:08.381 INFO [244/752] Repair commands completed
Sep 22 23:15:08.381 INFO Pop front: ReconcileIO { id: ReconciliationId(244), op: ExtentFlush { repair_id: ReconciliationId(244), extent_id: 112, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:08.382 INFO Sent repair work, now wait for resp
Sep 22 23:15:08.382 INFO [0] received reconcile message
Sep 22 23:15:08.382 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(244), op: ExtentFlush { repair_id: ReconciliationId(244), extent_id: 112, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:08.382 INFO [0] client ExtentFlush { repair_id: ReconciliationId(244), extent_id: 112, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:08.382 INFO [1] received reconcile message
Sep 22 23:15:08.382 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(244), op: ExtentFlush { repair_id: ReconciliationId(244), extent_id: 112, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:08.382 INFO [1] client ExtentFlush { repair_id: ReconciliationId(244), extent_id: 112, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:08.382 INFO [2] received reconcile message
Sep 22 23:15:08.382 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(244), op: ExtentFlush { repair_id: ReconciliationId(244), extent_id: 112, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:08.382 INFO [2] client ExtentFlush { repair_id: ReconciliationId(244), extent_id: 112, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:08.382 DEBG 244 Flush extent 112 with f:2 g:2
Sep 22 23:15:08.382 DEBG Flush just extent 112 with f:2 and g:2
Sep 22 23:15:08.382 DEBG [1] It's time to notify for 244
Sep 22 23:15:08.382 INFO Completion from [1] id:244 status:true
Sep 22 23:15:08.382 INFO [245/752] Repair commands completed
Sep 22 23:15:08.382 INFO Pop front: ReconcileIO { id: ReconciliationId(245), op: ExtentClose { repair_id: ReconciliationId(245), extent_id: 112 }, state: ClientData([New, New, New]) }
Sep 22 23:15:08.382 INFO Sent repair work, now wait for resp
Sep 22 23:15:08.382 INFO [0] received reconcile message
Sep 22 23:15:08.382 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(245), op: ExtentClose { repair_id: ReconciliationId(245), extent_id: 112 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:08.382 INFO [0] client ExtentClose { repair_id: ReconciliationId(245), extent_id: 112 }
Sep 22 23:15:08.382 INFO [1] received reconcile message
Sep 22 23:15:08.382 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(245), op: ExtentClose { repair_id: ReconciliationId(245), extent_id: 112 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:08.382 INFO [1] client ExtentClose { repair_id: ReconciliationId(245), extent_id: 112 }
Sep 22 23:15:08.382 INFO [2] received reconcile message
Sep 22 23:15:08.382 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(245), op: ExtentClose { repair_id: ReconciliationId(245), extent_id: 112 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:08.382 INFO [2] client ExtentClose { repair_id: ReconciliationId(245), extent_id: 112 }
Sep 22 23:15:08.382 DEBG 245 Close extent 112
Sep 22 23:15:08.383 DEBG 245 Close extent 112
Sep 22 23:15:08.383 DEBG 245 Close extent 112
Sep 22 23:15:08.383 DEBG [2] It's time to notify for 245
Sep 22 23:15:08.383 INFO Completion from [2] id:245 status:true
Sep 22 23:15:08.383 INFO [246/752] Repair commands completed
Sep 22 23:15:08.383 INFO Pop front: ReconcileIO { id: ReconciliationId(246), op: ExtentRepair { repair_id: ReconciliationId(246), extent_id: 112, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:08.383 INFO Sent repair work, now wait for resp
Sep 22 23:15:08.384 INFO [0] received reconcile message
Sep 22 23:15:08.384 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(246), op: ExtentRepair { repair_id: ReconciliationId(246), extent_id: 112, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:08.384 INFO [0] client ExtentRepair { repair_id: ReconciliationId(246), extent_id: 112, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:08.384 INFO [0] Sending repair request ReconciliationId(246)
Sep 22 23:15:08.384 INFO [1] received reconcile message
Sep 22 23:15:08.384 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(246), op: ExtentRepair { repair_id: ReconciliationId(246), extent_id: 112, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:08.384 INFO [1] client ExtentRepair { repair_id: ReconciliationId(246), extent_id: 112, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:08.384 INFO [1] No action required ReconciliationId(246)
Sep 22 23:15:08.384 INFO [2] received reconcile message
Sep 22 23:15:08.384 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(246), op: ExtentRepair { repair_id: ReconciliationId(246), extent_id: 112, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:08.384 INFO [2] client ExtentRepair { repair_id: ReconciliationId(246), extent_id: 112, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:08.384 INFO [2] No action required ReconciliationId(246)
Sep 22 23:15:08.384 DEBG 246 Repair extent 112 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:08.384 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/070.copy"
Sep 22 23:15:08.448 INFO accepted connection, remote_addr: 127.0.0.1:51815, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.449 TRCE incoming request, uri: /extent/112/files, method: GET, req_id: b6c274dc-72ff-454c-aa0d-e0058817458c, remote_addr: 127.0.0.1:51815, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.449 INFO request completed, latency_us: 231, response_code: 200, uri: /extent/112/files, method: GET, req_id: b6c274dc-72ff-454c-aa0d-e0058817458c, remote_addr: 127.0.0.1:51815, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.449 INFO eid:112 Found repair files: ["070", "070.db"]
Sep 22 23:15:08.449 TRCE incoming request, uri: /newextent/112/data, method: GET, req_id: d423bf0d-e096-4498-854c-cf34bb546773, remote_addr: 127.0.0.1:51815, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.450 INFO request completed, latency_us: 343, response_code: 200, uri: /newextent/112/data, method: GET, req_id: d423bf0d-e096-4498-854c-cf34bb546773, remote_addr: 127.0.0.1:51815, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.455 TRCE incoming request, uri: /newextent/112/db, method: GET, req_id: 14b047aa-c10f-4f49-9097-4a3c5cdc43e9, remote_addr: 127.0.0.1:51815, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.455 INFO request completed, latency_us: 291, response_code: 200, uri: /newextent/112/db, method: GET, req_id: 14b047aa-c10f-4f49-9097-4a3c5cdc43e9, remote_addr: 127.0.0.1:51815, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.456 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/070.copy" to "/tmp/downstairs-vrx8aK6L/00/000/070.replace"
Sep 22 23:15:08.456 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:08.457 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/070.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:08.457 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/070"
Sep 22 23:15:08.457 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/070.db"
Sep 22 23:15:08.457 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:08.457 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/070.replace" to "/tmp/downstairs-vrx8aK6L/00/000/070.completed"
Sep 22 23:15:08.457 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:08.457 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:08.458 DEBG [0] It's time to notify for 246
Sep 22 23:15:08.458 INFO Completion from [0] id:246 status:true
Sep 22 23:15:08.458 INFO [247/752] Repair commands completed
Sep 22 23:15:08.458 INFO Pop front: ReconcileIO { id: ReconciliationId(247), op: ExtentReopen { repair_id: ReconciliationId(247), extent_id: 112 }, state: ClientData([New, New, New]) }
Sep 22 23:15:08.458 INFO Sent repair work, now wait for resp
Sep 22 23:15:08.458 INFO [0] received reconcile message
Sep 22 23:15:08.458 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(247), op: ExtentReopen { repair_id: ReconciliationId(247), extent_id: 112 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:08.458 INFO [0] client ExtentReopen { repair_id: ReconciliationId(247), extent_id: 112 }
Sep 22 23:15:08.458 INFO [1] received reconcile message
Sep 22 23:15:08.458 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(247), op: ExtentReopen { repair_id: ReconciliationId(247), extent_id: 112 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:08.458 INFO [1] client ExtentReopen { repair_id: ReconciliationId(247), extent_id: 112 }
Sep 22 23:15:08.458 INFO [2] received reconcile message
Sep 22 23:15:08.458 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(247), op: ExtentReopen { repair_id: ReconciliationId(247), extent_id: 112 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:08.458 INFO [2] client ExtentReopen { repair_id: ReconciliationId(247), extent_id: 112 }
Sep 22 23:15:08.458 DEBG 247 Reopen extent 112
Sep 22 23:15:08.459 DEBG 247 Reopen extent 112
Sep 22 23:15:08.459 DEBG 247 Reopen extent 112
Sep 22 23:15:08.460 DEBG [2] It's time to notify for 247
Sep 22 23:15:08.460 INFO Completion from [2] id:247 status:true
Sep 22 23:15:08.460 INFO [248/752] Repair commands completed
Sep 22 23:15:08.460 INFO Pop front: ReconcileIO { id: ReconciliationId(248), op: ExtentFlush { repair_id: ReconciliationId(248), extent_id: 118, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:08.460 INFO Sent repair work, now wait for resp
Sep 22 23:15:08.460 INFO [0] received reconcile message
Sep 22 23:15:08.460 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(248), op: ExtentFlush { repair_id: ReconciliationId(248), extent_id: 118, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:08.460 INFO [0] client ExtentFlush { repair_id: ReconciliationId(248), extent_id: 118, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:08.460 INFO [1] received reconcile message
Sep 22 23:15:08.460 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(248), op: ExtentFlush { repair_id: ReconciliationId(248), extent_id: 118, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:08.460 INFO [1] client ExtentFlush { repair_id: ReconciliationId(248), extent_id: 118, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:08.460 INFO [2] received reconcile message
Sep 22 23:15:08.460 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(248), op: ExtentFlush { repair_id: ReconciliationId(248), extent_id: 118, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:08.460 INFO [2] client ExtentFlush { repair_id: ReconciliationId(248), extent_id: 118, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:08.460 DEBG 248 Flush extent 118 with f:2 g:2
Sep 22 23:15:08.460 DEBG Flush just extent 118 with f:2 and g:2
Sep 22 23:15:08.461 DEBG [1] It's time to notify for 248
Sep 22 23:15:08.461 INFO Completion from [1] id:248 status:true
Sep 22 23:15:08.461 INFO [249/752] Repair commands completed
Sep 22 23:15:08.461 INFO Pop front: ReconcileIO { id: ReconciliationId(249), op: ExtentClose { repair_id: ReconciliationId(249), extent_id: 118 }, state: ClientData([New, New, New]) }
Sep 22 23:15:08.461 INFO Sent repair work, now wait for resp
Sep 22 23:15:08.461 INFO [0] received reconcile message
Sep 22 23:15:08.461 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(249), op: ExtentClose { repair_id: ReconciliationId(249), extent_id: 118 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:08.461 INFO [0] client ExtentClose { repair_id: ReconciliationId(249), extent_id: 118 }
Sep 22 23:15:08.461 INFO [1] received reconcile message
Sep 22 23:15:08.461 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(249), op: ExtentClose { repair_id: ReconciliationId(249), extent_id: 118 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:08.461 INFO [1] client ExtentClose { repair_id: ReconciliationId(249), extent_id: 118 }
Sep 22 23:15:08.461 INFO [2] received reconcile message
Sep 22 23:15:08.461 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(249), op: ExtentClose { repair_id: ReconciliationId(249), extent_id: 118 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:08.461 INFO [2] client ExtentClose { repair_id: ReconciliationId(249), extent_id: 118 }
Sep 22 23:15:08.461 DEBG 249 Close extent 118
Sep 22 23:15:08.461 DEBG 249 Close extent 118
Sep 22 23:15:08.462 DEBG 249 Close extent 118
Sep 22 23:15:08.462 DEBG [2] It's time to notify for 249
Sep 22 23:15:08.462 INFO Completion from [2] id:249 status:true
Sep 22 23:15:08.462 INFO [250/752] Repair commands completed
Sep 22 23:15:08.462 INFO Pop front: ReconcileIO { id: ReconciliationId(250), op: ExtentRepair { repair_id: ReconciliationId(250), extent_id: 118, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:08.462 INFO Sent repair work, now wait for resp
Sep 22 23:15:08.462 INFO [0] received reconcile message
Sep 22 23:15:08.462 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(250), op: ExtentRepair { repair_id: ReconciliationId(250), extent_id: 118, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:08.462 INFO [0] client ExtentRepair { repair_id: ReconciliationId(250), extent_id: 118, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:08.462 INFO [0] Sending repair request ReconciliationId(250)
Sep 22 23:15:08.462 INFO [1] received reconcile message
Sep 22 23:15:08.462 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(250), op: ExtentRepair { repair_id: ReconciliationId(250), extent_id: 118, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:08.462 INFO [1] client ExtentRepair { repair_id: ReconciliationId(250), extent_id: 118, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:08.462 INFO [1] No action required ReconciliationId(250)
Sep 22 23:15:08.462 INFO [2] received reconcile message
Sep 22 23:15:08.462 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(250), op: ExtentRepair { repair_id: ReconciliationId(250), extent_id: 118, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:08.462 INFO [2] client ExtentRepair { repair_id: ReconciliationId(250), extent_id: 118, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:08.462 INFO [2] No action required ReconciliationId(250)
Sep 22 23:15:08.462 DEBG 250 Repair extent 118 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:08.463 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/076.copy"
Sep 22 23:15:08.528 INFO accepted connection, remote_addr: 127.0.0.1:47270, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.528 TRCE incoming request, uri: /extent/118/files, method: GET, req_id: 9411d776-20a5-423f-b71c-f79048f07767, remote_addr: 127.0.0.1:47270, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.528 INFO request completed, latency_us: 248, response_code: 200, uri: /extent/118/files, method: GET, req_id: 9411d776-20a5-423f-b71c-f79048f07767, remote_addr: 127.0.0.1:47270, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.528 INFO eid:118 Found repair files: ["076", "076.db"]
Sep 22 23:15:08.529 TRCE incoming request, uri: /newextent/118/data, method: GET, req_id: 35eeb2c3-5a88-45f0-a70d-04eac0d5cbec, remote_addr: 127.0.0.1:47270, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.529 INFO request completed, latency_us: 351, response_code: 200, uri: /newextent/118/data, method: GET, req_id: 35eeb2c3-5a88-45f0-a70d-04eac0d5cbec, remote_addr: 127.0.0.1:47270, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.534 TRCE incoming request, uri: /newextent/118/db, method: GET, req_id: c3f6d948-0e61-4734-9fe0-5a243de6ff08, remote_addr: 127.0.0.1:47270, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.534 INFO request completed, latency_us: 294, response_code: 200, uri: /newextent/118/db, method: GET, req_id: c3f6d948-0e61-4734-9fe0-5a243de6ff08, remote_addr: 127.0.0.1:47270, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.536 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/076.copy" to "/tmp/downstairs-vrx8aK6L/00/000/076.replace"
Sep 22 23:15:08.536 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:08.537 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/076.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:08.537 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/076"
Sep 22 23:15:08.537 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/076.db"
Sep 22 23:15:08.537 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:08.537 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/076.replace" to "/tmp/downstairs-vrx8aK6L/00/000/076.completed"
Sep 22 23:15:08.537 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:08.537 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:08.537 DEBG [0] It's time to notify for 250
Sep 22 23:15:08.538 INFO Completion from [0] id:250 status:true
Sep 22 23:15:08.538 INFO [251/752] Repair commands completed
Sep 22 23:15:08.538 INFO Pop front: ReconcileIO { id: ReconciliationId(251), op: ExtentReopen { repair_id: ReconciliationId(251), extent_id: 118 }, state: ClientData([New, New, New]) }
Sep 22 23:15:08.538 INFO Sent repair work, now wait for resp
Sep 22 23:15:08.538 INFO [0] received reconcile message
Sep 22 23:15:08.538 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(251), op: ExtentReopen { repair_id: ReconciliationId(251), extent_id: 118 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:08.538 INFO [0] client ExtentReopen { repair_id: ReconciliationId(251), extent_id: 118 }
Sep 22 23:15:08.538 INFO [1] received reconcile message
Sep 22 23:15:08.538 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(251), op: ExtentReopen { repair_id: ReconciliationId(251), extent_id: 118 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:08.538 INFO [1] client ExtentReopen { repair_id: ReconciliationId(251), extent_id: 118 }
Sep 22 23:15:08.538 INFO [2] received reconcile message
Sep 22 23:15:08.538 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(251), op: ExtentReopen { repair_id: ReconciliationId(251), extent_id: 118 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:08.538 INFO [2] client ExtentReopen { repair_id: ReconciliationId(251), extent_id: 118 }
Sep 22 23:15:08.538 DEBG 251 Reopen extent 118
Sep 22 23:15:08.539 DEBG 251 Reopen extent 118
Sep 22 23:15:08.539 DEBG 251 Reopen extent 118
Sep 22 23:15:08.540 DEBG [2] It's time to notify for 251
Sep 22 23:15:08.540 INFO Completion from [2] id:251 status:true
Sep 22 23:15:08.540 INFO [252/752] Repair commands completed
Sep 22 23:15:08.540 INFO Pop front: ReconcileIO { id: ReconciliationId(252), op: ExtentFlush { repair_id: ReconciliationId(252), extent_id: 130, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:08.540 INFO Sent repair work, now wait for resp
Sep 22 23:15:08.540 INFO [0] received reconcile message
Sep 22 23:15:08.540 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(252), op: ExtentFlush { repair_id: ReconciliationId(252), extent_id: 130, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:08.540 INFO [0] client ExtentFlush { repair_id: ReconciliationId(252), extent_id: 130, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:08.540 INFO [1] received reconcile message
Sep 22 23:15:08.540 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(252), op: ExtentFlush { repair_id: ReconciliationId(252), extent_id: 130, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:08.540 INFO [1] client ExtentFlush { repair_id: ReconciliationId(252), extent_id: 130, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:08.540 INFO [2] received reconcile message
Sep 22 23:15:08.540 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(252), op: ExtentFlush { repair_id: ReconciliationId(252), extent_id: 130, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:08.540 INFO [2] client ExtentFlush { repair_id: ReconciliationId(252), extent_id: 130, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:08.540 DEBG 252 Flush extent 130 with f:2 g:2
Sep 22 23:15:08.540 DEBG Flush just extent 130 with f:2 and g:2
Sep 22 23:15:08.541 DEBG [1] It's time to notify for 252
Sep 22 23:15:08.541 INFO Completion from [1] id:252 status:true
Sep 22 23:15:08.541 INFO [253/752] Repair commands completed
Sep 22 23:15:08.541 INFO Pop front: ReconcileIO { id: ReconciliationId(253), op: ExtentClose { repair_id: ReconciliationId(253), extent_id: 130 }, state: ClientData([New, New, New]) }
Sep 22 23:15:08.541 INFO Sent repair work, now wait for resp
Sep 22 23:15:08.541 INFO [0] received reconcile message
Sep 22 23:15:08.541 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(253), op: ExtentClose { repair_id: ReconciliationId(253), extent_id: 130 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:08.541 INFO [0] client ExtentClose { repair_id: ReconciliationId(253), extent_id: 130 }
Sep 22 23:15:08.541 INFO [1] received reconcile message
Sep 22 23:15:08.541 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(253), op: ExtentClose { repair_id: ReconciliationId(253), extent_id: 130 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:08.541 INFO [1] client ExtentClose { repair_id: ReconciliationId(253), extent_id: 130 }
Sep 22 23:15:08.541 INFO [2] received reconcile message
Sep 22 23:15:08.541 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(253), op: ExtentClose { repair_id: ReconciliationId(253), extent_id: 130 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:08.541 INFO [2] client ExtentClose { repair_id: ReconciliationId(253), extent_id: 130 }
Sep 22 23:15:08.541 DEBG 253 Close extent 130
Sep 22 23:15:08.541 DEBG 253 Close extent 130
Sep 22 23:15:08.542 DEBG 253 Close extent 130
Sep 22 23:15:08.542 DEBG [2] It's time to notify for 253
Sep 22 23:15:08.542 INFO Completion from [2] id:253 status:true
Sep 22 23:15:08.542 INFO [254/752] Repair commands completed
Sep 22 23:15:08.542 INFO Pop front: ReconcileIO { id: ReconciliationId(254), op: ExtentRepair { repair_id: ReconciliationId(254), extent_id: 130, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:08.542 INFO Sent repair work, now wait for resp
Sep 22 23:15:08.542 INFO [0] received reconcile message
Sep 22 23:15:08.542 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(254), op: ExtentRepair { repair_id: ReconciliationId(254), extent_id: 130, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:08.542 INFO [0] client ExtentRepair { repair_id: ReconciliationId(254), extent_id: 130, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:08.542 INFO [0] Sending repair request ReconciliationId(254)
Sep 22 23:15:08.542 INFO [1] received reconcile message
Sep 22 23:15:08.542 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(254), op: ExtentRepair { repair_id: ReconciliationId(254), extent_id: 130, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:08.542 INFO [1] client ExtentRepair { repair_id: ReconciliationId(254), extent_id: 130, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:08.542 INFO [1] No action required ReconciliationId(254)
Sep 22 23:15:08.542 INFO [2] received reconcile message
Sep 22 23:15:08.542 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(254), op: ExtentRepair { repair_id: ReconciliationId(254), extent_id: 130, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:08.542 INFO [2] client ExtentRepair { repair_id: ReconciliationId(254), extent_id: 130, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:08.542 INFO [2] No action required ReconciliationId(254)
Sep 22 23:15:08.542 DEBG 254 Repair extent 130 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:08.543 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/082.copy"
Sep 22 23:15:08.550 DEBG up_ds_listen was notified
Sep 22 23:15:08.550 DEBG up_ds_listen process 1079
Sep 22 23:15:08.550 DEBG [A] ack job 1079:80, : downstairs
Sep 22 23:15:08.551 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:08.551 WARN returning error on read!
Sep 22 23:15:08.551 DEBG Read      :1078 deps:[JobId(1077)] res:false
Sep 22 23:15:08.551 INFO [lossy] skipping 1079
Sep 22 23:15:08.551 WARN returning error on read!
Sep 22 23:15:08.551 DEBG Read      :1078 deps:[JobId(1077)] res:false
Sep 22 23:15:08.557 DEBG Read      :1078 deps:[JobId(1077)] res:true
Sep 22 23:15:08.578 DEBG IO Read  1080 has deps [JobId(1079)]
Sep 22 23:15:08.591 ERRO [2] job id 1078 saw error GenericError("test error")
Sep 22 23:15:08.591 ERRO [2] job id 1078 saw error GenericError("test error")
Sep 22 23:15:08.593 DEBG Flush     :1079 extent_limit None deps:[JobId(1078)] res:true f:29 g:1
Sep 22 23:15:08.599 DEBG Read      :1080 deps:[JobId(1079)] res:true
Sep 22 23:15:08.607 INFO accepted connection, remote_addr: 127.0.0.1:46306, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.607 TRCE incoming request, uri: /extent/130/files, method: GET, req_id: 91a25e6a-d277-4ba5-83d7-8195af20ef10, remote_addr: 127.0.0.1:46306, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.608 INFO request completed, latency_us: 251, response_code: 200, uri: /extent/130/files, method: GET, req_id: 91a25e6a-d277-4ba5-83d7-8195af20ef10, remote_addr: 127.0.0.1:46306, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.608 INFO eid:130 Found repair files: ["082", "082.db"]
Sep 22 23:15:08.608 TRCE incoming request, uri: /newextent/130/data, method: GET, req_id: c83db128-2efd-4eec-868a-4a8d7a4432b8, remote_addr: 127.0.0.1:46306, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.609 INFO request completed, latency_us: 353, response_code: 200, uri: /newextent/130/data, method: GET, req_id: c83db128-2efd-4eec-868a-4a8d7a4432b8, remote_addr: 127.0.0.1:46306, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.614 TRCE incoming request, uri: /newextent/130/db, method: GET, req_id: 408c7289-c6fa-4c56-901e-14c0cc0d69f3, remote_addr: 127.0.0.1:46306, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.614 INFO request completed, latency_us: 300, response_code: 200, uri: /newextent/130/db, method: GET, req_id: 408c7289-c6fa-4c56-901e-14c0cc0d69f3, remote_addr: 127.0.0.1:46306, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.615 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/082.copy" to "/tmp/downstairs-vrx8aK6L/00/000/082.replace"
Sep 22 23:15:08.615 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:08.616 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/082.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:08.616 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/082"
Sep 22 23:15:08.617 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/082.db"
Sep 22 23:15:08.617 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:08.617 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/082.replace" to "/tmp/downstairs-vrx8aK6L/00/000/082.completed"
Sep 22 23:15:08.617 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:08.617 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:08.617 DEBG [0] It's time to notify for 254
Sep 22 23:15:08.617 INFO Completion from [0] id:254 status:true
Sep 22 23:15:08.617 INFO [255/752] Repair commands completed
Sep 22 23:15:08.617 INFO Pop front: ReconcileIO { id: ReconciliationId(255), op: ExtentReopen { repair_id: ReconciliationId(255), extent_id: 130 }, state: ClientData([New, New, New]) }
Sep 22 23:15:08.617 INFO Sent repair work, now wait for resp
Sep 22 23:15:08.617 INFO [0] received reconcile message
Sep 22 23:15:08.617 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(255), op: ExtentReopen { repair_id: ReconciliationId(255), extent_id: 130 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:08.617 INFO [0] client ExtentReopen { repair_id: ReconciliationId(255), extent_id: 130 }
Sep 22 23:15:08.617 INFO [1] received reconcile message
Sep 22 23:15:08.617 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(255), op: ExtentReopen { repair_id: ReconciliationId(255), extent_id: 130 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:08.617 INFO [1] client ExtentReopen { repair_id: ReconciliationId(255), extent_id: 130 }
Sep 22 23:15:08.617 INFO [2] received reconcile message
Sep 22 23:15:08.617 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(255), op: ExtentReopen { repair_id: ReconciliationId(255), extent_id: 130 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:08.617 INFO [2] client ExtentReopen { repair_id: ReconciliationId(255), extent_id: 130 }
Sep 22 23:15:08.618 DEBG 255 Reopen extent 130
Sep 22 23:15:08.618 DEBG 255 Reopen extent 130
Sep 22 23:15:08.619 DEBG 255 Reopen extent 130
Sep 22 23:15:08.619 DEBG [2] It's time to notify for 255
Sep 22 23:15:08.620 INFO Completion from [2] id:255 status:true
Sep 22 23:15:08.620 INFO [256/752] Repair commands completed
Sep 22 23:15:08.620 INFO Pop front: ReconcileIO { id: ReconciliationId(256), op: ExtentFlush { repair_id: ReconciliationId(256), extent_id: 25, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:08.620 INFO Sent repair work, now wait for resp
Sep 22 23:15:08.620 INFO [0] received reconcile message
Sep 22 23:15:08.620 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(256), op: ExtentFlush { repair_id: ReconciliationId(256), extent_id: 25, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:08.620 INFO [0] client ExtentFlush { repair_id: ReconciliationId(256), extent_id: 25, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:08.620 INFO [1] received reconcile message
Sep 22 23:15:08.620 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(256), op: ExtentFlush { repair_id: ReconciliationId(256), extent_id: 25, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:08.620 INFO [1] client ExtentFlush { repair_id: ReconciliationId(256), extent_id: 25, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:08.620 INFO [2] received reconcile message
Sep 22 23:15:08.620 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(256), op: ExtentFlush { repair_id: ReconciliationId(256), extent_id: 25, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:08.620 INFO [2] client ExtentFlush { repair_id: ReconciliationId(256), extent_id: 25, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:08.620 DEBG 256 Flush extent 25 with f:2 g:2
Sep 22 23:15:08.620 DEBG Flush just extent 25 with f:2 and g:2
Sep 22 23:15:08.620 DEBG [1] It's time to notify for 256
Sep 22 23:15:08.620 INFO Completion from [1] id:256 status:true
Sep 22 23:15:08.620 INFO [257/752] Repair commands completed
Sep 22 23:15:08.620 INFO Pop front: ReconcileIO { id: ReconciliationId(257), op: ExtentClose { repair_id: ReconciliationId(257), extent_id: 25 }, state: ClientData([New, New, New]) }
Sep 22 23:15:08.620 INFO Sent repair work, now wait for resp
Sep 22 23:15:08.620 INFO [0] received reconcile message
Sep 22 23:15:08.620 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(257), op: ExtentClose { repair_id: ReconciliationId(257), extent_id: 25 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:08.620 INFO [0] client ExtentClose { repair_id: ReconciliationId(257), extent_id: 25 }
Sep 22 23:15:08.620 INFO [1] received reconcile message
Sep 22 23:15:08.620 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(257), op: ExtentClose { repair_id: ReconciliationId(257), extent_id: 25 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:08.620 INFO [1] client ExtentClose { repair_id: ReconciliationId(257), extent_id: 25 }
Sep 22 23:15:08.620 INFO [2] received reconcile message
Sep 22 23:15:08.620 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(257), op: ExtentClose { repair_id: ReconciliationId(257), extent_id: 25 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:08.620 INFO [2] client ExtentClose { repair_id: ReconciliationId(257), extent_id: 25 }
Sep 22 23:15:08.621 DEBG 257 Close extent 25
Sep 22 23:15:08.621 DEBG 257 Close extent 25
Sep 22 23:15:08.621 DEBG 257 Close extent 25
Sep 22 23:15:08.621 DEBG [2] It's time to notify for 257
Sep 22 23:15:08.621 INFO Completion from [2] id:257 status:true
Sep 22 23:15:08.622 INFO [258/752] Repair commands completed
Sep 22 23:15:08.622 INFO Pop front: ReconcileIO { id: ReconciliationId(258), op: ExtentRepair { repair_id: ReconciliationId(258), extent_id: 25, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:08.622 INFO Sent repair work, now wait for resp
Sep 22 23:15:08.622 INFO [0] received reconcile message
Sep 22 23:15:08.622 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(258), op: ExtentRepair { repair_id: ReconciliationId(258), extent_id: 25, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:08.622 INFO [0] client ExtentRepair { repair_id: ReconciliationId(258), extent_id: 25, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:08.622 INFO [0] Sending repair request ReconciliationId(258)
Sep 22 23:15:08.622 INFO [1] received reconcile message
Sep 22 23:15:08.622 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(258), op: ExtentRepair { repair_id: ReconciliationId(258), extent_id: 25, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:08.622 INFO [1] client ExtentRepair { repair_id: ReconciliationId(258), extent_id: 25, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:08.622 INFO [1] No action required ReconciliationId(258)
Sep 22 23:15:08.622 INFO [2] received reconcile message
Sep 22 23:15:08.622 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(258), op: ExtentRepair { repair_id: ReconciliationId(258), extent_id: 25, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:08.622 INFO [2] client ExtentRepair { repair_id: ReconciliationId(258), extent_id: 25, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:08.622 INFO [2] No action required ReconciliationId(258)
Sep 22 23:15:08.622 DEBG 258 Repair extent 25 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:08.622 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/019.copy"
Sep 22 23:15:08.687 INFO accepted connection, remote_addr: 127.0.0.1:65372, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.687 TRCE incoming request, uri: /extent/25/files, method: GET, req_id: b94cbafe-84ee-4da2-8176-11127bfcdf56, remote_addr: 127.0.0.1:65372, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.687 INFO request completed, latency_us: 269, response_code: 200, uri: /extent/25/files, method: GET, req_id: b94cbafe-84ee-4da2-8176-11127bfcdf56, remote_addr: 127.0.0.1:65372, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.688 INFO eid:25 Found repair files: ["019", "019.db"]
Sep 22 23:15:08.688 TRCE incoming request, uri: /newextent/25/data, method: GET, req_id: 48787ab5-a370-470e-a736-bf561e9d7a94, remote_addr: 127.0.0.1:65372, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.688 INFO request completed, latency_us: 401, response_code: 200, uri: /newextent/25/data, method: GET, req_id: 48787ab5-a370-470e-a736-bf561e9d7a94, remote_addr: 127.0.0.1:65372, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.694 TRCE incoming request, uri: /newextent/25/db, method: GET, req_id: 2467fff0-11fc-43ea-8bd5-7b7cee2c28a3, remote_addr: 127.0.0.1:65372, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.694 INFO request completed, latency_us: 300, response_code: 200, uri: /newextent/25/db, method: GET, req_id: 2467fff0-11fc-43ea-8bd5-7b7cee2c28a3, remote_addr: 127.0.0.1:65372, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.695 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/019.copy" to "/tmp/downstairs-vrx8aK6L/00/000/019.replace"
Sep 22 23:15:08.695 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:08.697 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/019.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:08.697 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/019"
Sep 22 23:15:08.697 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/019.db"
Sep 22 23:15:08.697 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:08.697 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/019.replace" to "/tmp/downstairs-vrx8aK6L/00/000/019.completed"
Sep 22 23:15:08.697 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:08.697 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:08.697 DEBG [0] It's time to notify for 258
Sep 22 23:15:08.697 INFO Completion from [0] id:258 status:true
Sep 22 23:15:08.697 INFO [259/752] Repair commands completed
Sep 22 23:15:08.697 INFO Pop front: ReconcileIO { id: ReconciliationId(259), op: ExtentReopen { repair_id: ReconciliationId(259), extent_id: 25 }, state: ClientData([New, New, New]) }
Sep 22 23:15:08.698 INFO Sent repair work, now wait for resp
Sep 22 23:15:08.698 INFO [0] received reconcile message
Sep 22 23:15:08.698 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(259), op: ExtentReopen { repair_id: ReconciliationId(259), extent_id: 25 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:08.698 INFO [0] client ExtentReopen { repair_id: ReconciliationId(259), extent_id: 25 }
Sep 22 23:15:08.698 INFO [1] received reconcile message
Sep 22 23:15:08.698 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(259), op: ExtentReopen { repair_id: ReconciliationId(259), extent_id: 25 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:08.698 INFO [1] client ExtentReopen { repair_id: ReconciliationId(259), extent_id: 25 }
Sep 22 23:15:08.698 INFO [2] received reconcile message
Sep 22 23:15:08.698 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(259), op: ExtentReopen { repair_id: ReconciliationId(259), extent_id: 25 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:08.698 INFO [2] client ExtentReopen { repair_id: ReconciliationId(259), extent_id: 25 }
Sep 22 23:15:08.698 DEBG 259 Reopen extent 25
Sep 22 23:15:08.699 DEBG 259 Reopen extent 25
Sep 22 23:15:08.699 DEBG 259 Reopen extent 25
Sep 22 23:15:08.700 DEBG [2] It's time to notify for 259
Sep 22 23:15:08.700 INFO Completion from [2] id:259 status:true
Sep 22 23:15:08.700 INFO [260/752] Repair commands completed
Sep 22 23:15:08.700 INFO Pop front: ReconcileIO { id: ReconciliationId(260), op: ExtentFlush { repair_id: ReconciliationId(260), extent_id: 47, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:08.700 INFO Sent repair work, now wait for resp
Sep 22 23:15:08.700 INFO [0] received reconcile message
Sep 22 23:15:08.700 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(260), op: ExtentFlush { repair_id: ReconciliationId(260), extent_id: 47, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:08.700 INFO [0] client ExtentFlush { repair_id: ReconciliationId(260), extent_id: 47, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:08.700 INFO [1] received reconcile message
Sep 22 23:15:08.700 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(260), op: ExtentFlush { repair_id: ReconciliationId(260), extent_id: 47, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:08.700 INFO [1] client ExtentFlush { repair_id: ReconciliationId(260), extent_id: 47, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:08.700 INFO [2] received reconcile message
Sep 22 23:15:08.700 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(260), op: ExtentFlush { repair_id: ReconciliationId(260), extent_id: 47, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:08.700 INFO [2] client ExtentFlush { repair_id: ReconciliationId(260), extent_id: 47, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:08.700 DEBG 260 Flush extent 47 with f:2 g:2
Sep 22 23:15:08.700 DEBG Flush just extent 47 with f:2 and g:2
Sep 22 23:15:08.700 DEBG [1] It's time to notify for 260
Sep 22 23:15:08.701 INFO Completion from [1] id:260 status:true
Sep 22 23:15:08.701 INFO [261/752] Repair commands completed
Sep 22 23:15:08.701 INFO Pop front: ReconcileIO { id: ReconciliationId(261), op: ExtentClose { repair_id: ReconciliationId(261), extent_id: 47 }, state: ClientData([New, New, New]) }
Sep 22 23:15:08.701 INFO Sent repair work, now wait for resp
Sep 22 23:15:08.701 INFO [0] received reconcile message
Sep 22 23:15:08.701 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(261), op: ExtentClose { repair_id: ReconciliationId(261), extent_id: 47 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:08.701 INFO [0] client ExtentClose { repair_id: ReconciliationId(261), extent_id: 47 }
Sep 22 23:15:08.701 INFO [1] received reconcile message
Sep 22 23:15:08.701 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(261), op: ExtentClose { repair_id: ReconciliationId(261), extent_id: 47 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:08.701 INFO [1] client ExtentClose { repair_id: ReconciliationId(261), extent_id: 47 }
Sep 22 23:15:08.701 INFO [2] received reconcile message
Sep 22 23:15:08.701 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(261), op: ExtentClose { repair_id: ReconciliationId(261), extent_id: 47 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:08.701 INFO [2] client ExtentClose { repair_id: ReconciliationId(261), extent_id: 47 }
Sep 22 23:15:08.701 DEBG 261 Close extent 47
Sep 22 23:15:08.701 DEBG 261 Close extent 47
Sep 22 23:15:08.702 DEBG 261 Close extent 47
Sep 22 23:15:08.702 DEBG [2] It's time to notify for 261
Sep 22 23:15:08.702 INFO Completion from [2] id:261 status:true
Sep 22 23:15:08.702 INFO [262/752] Repair commands completed
Sep 22 23:15:08.702 INFO Pop front: ReconcileIO { id: ReconciliationId(262), op: ExtentRepair { repair_id: ReconciliationId(262), extent_id: 47, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:08.702 INFO Sent repair work, now wait for resp
Sep 22 23:15:08.702 INFO [0] received reconcile message
Sep 22 23:15:08.702 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(262), op: ExtentRepair { repair_id: ReconciliationId(262), extent_id: 47, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:08.702 INFO [0] client ExtentRepair { repair_id: ReconciliationId(262), extent_id: 47, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:08.702 INFO [0] Sending repair request ReconciliationId(262)
Sep 22 23:15:08.702 INFO [1] received reconcile message
Sep 22 23:15:08.702 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(262), op: ExtentRepair { repair_id: ReconciliationId(262), extent_id: 47, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:08.702 INFO [1] client ExtentRepair { repair_id: ReconciliationId(262), extent_id: 47, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:08.702 INFO [1] No action required ReconciliationId(262)
Sep 22 23:15:08.702 INFO [2] received reconcile message
Sep 22 23:15:08.702 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(262), op: ExtentRepair { repair_id: ReconciliationId(262), extent_id: 47, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:08.702 INFO [2] client ExtentRepair { repair_id: ReconciliationId(262), extent_id: 47, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:08.702 INFO [2] No action required ReconciliationId(262)
Sep 22 23:15:08.702 DEBG 262 Repair extent 47 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:08.702 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/02F.copy"
Sep 22 23:15:08.768 INFO accepted connection, remote_addr: 127.0.0.1:49772, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.768 TRCE incoming request, uri: /extent/47/files, method: GET, req_id: 023b9695-3b92-4745-805d-6fefdfdf4477, remote_addr: 127.0.0.1:49772, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.768 INFO request completed, latency_us: 274, response_code: 200, uri: /extent/47/files, method: GET, req_id: 023b9695-3b92-4745-805d-6fefdfdf4477, remote_addr: 127.0.0.1:49772, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.768 INFO eid:47 Found repair files: ["02F", "02F.db"]
Sep 22 23:15:08.769 TRCE incoming request, uri: /newextent/47/data, method: GET, req_id: 14a9116b-7d54-4529-aec3-f21d7501bde9, remote_addr: 127.0.0.1:49772, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.769 INFO request completed, latency_us: 371, response_code: 200, uri: /newextent/47/data, method: GET, req_id: 14a9116b-7d54-4529-aec3-f21d7501bde9, remote_addr: 127.0.0.1:49772, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.774 TRCE incoming request, uri: /newextent/47/db, method: GET, req_id: 4040ce8a-a082-4191-ae39-1d1c108c24b4, remote_addr: 127.0.0.1:49772, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.775 INFO request completed, latency_us: 295, response_code: 200, uri: /newextent/47/db, method: GET, req_id: 4040ce8a-a082-4191-ae39-1d1c108c24b4, remote_addr: 127.0.0.1:49772, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.776 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/02F.copy" to "/tmp/downstairs-vrx8aK6L/00/000/02F.replace"
Sep 22 23:15:08.776 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:08.777 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/02F.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:08.777 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/02F"
Sep 22 23:15:08.777 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/02F.db"
Sep 22 23:15:08.777 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:08.777 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/02F.replace" to "/tmp/downstairs-vrx8aK6L/00/000/02F.completed"
Sep 22 23:15:08.777 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:08.777 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:08.778 DEBG [0] It's time to notify for 262
Sep 22 23:15:08.778 INFO Completion from [0] id:262 status:true
Sep 22 23:15:08.778 INFO [263/752] Repair commands completed
Sep 22 23:15:08.778 INFO Pop front: ReconcileIO { id: ReconciliationId(263), op: ExtentReopen { repair_id: ReconciliationId(263), extent_id: 47 }, state: ClientData([New, New, New]) }
Sep 22 23:15:08.778 INFO Sent repair work, now wait for resp
Sep 22 23:15:08.778 INFO [0] received reconcile message
Sep 22 23:15:08.778 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(263), op: ExtentReopen { repair_id: ReconciliationId(263), extent_id: 47 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:08.778 INFO [0] client ExtentReopen { repair_id: ReconciliationId(263), extent_id: 47 }
Sep 22 23:15:08.778 INFO [1] received reconcile message
Sep 22 23:15:08.778 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(263), op: ExtentReopen { repair_id: ReconciliationId(263), extent_id: 47 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:08.778 INFO [1] client ExtentReopen { repair_id: ReconciliationId(263), extent_id: 47 }
Sep 22 23:15:08.778 INFO [2] received reconcile message
Sep 22 23:15:08.778 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(263), op: ExtentReopen { repair_id: ReconciliationId(263), extent_id: 47 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:08.778 INFO [2] client ExtentReopen { repair_id: ReconciliationId(263), extent_id: 47 }
Sep 22 23:15:08.778 DEBG 263 Reopen extent 47
Sep 22 23:15:08.779 DEBG 263 Reopen extent 47
Sep 22 23:15:08.780 DEBG 263 Reopen extent 47
Sep 22 23:15:08.780 DEBG [2] It's time to notify for 263
Sep 22 23:15:08.780 INFO Completion from [2] id:263 status:true
Sep 22 23:15:08.780 INFO [264/752] Repair commands completed
Sep 22 23:15:08.780 INFO Pop front: ReconcileIO { id: ReconciliationId(264), op: ExtentFlush { repair_id: ReconciliationId(264), extent_id: 3, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:08.780 INFO Sent repair work, now wait for resp
Sep 22 23:15:08.780 INFO [0] received reconcile message
Sep 22 23:15:08.780 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(264), op: ExtentFlush { repair_id: ReconciliationId(264), extent_id: 3, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:08.780 INFO [0] client ExtentFlush { repair_id: ReconciliationId(264), extent_id: 3, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:08.780 INFO [1] received reconcile message
Sep 22 23:15:08.780 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(264), op: ExtentFlush { repair_id: ReconciliationId(264), extent_id: 3, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:08.780 INFO [1] client ExtentFlush { repair_id: ReconciliationId(264), extent_id: 3, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:08.781 INFO [2] received reconcile message
Sep 22 23:15:08.781 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(264), op: ExtentFlush { repair_id: ReconciliationId(264), extent_id: 3, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:08.781 INFO [2] client ExtentFlush { repair_id: ReconciliationId(264), extent_id: 3, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:08.781 DEBG 264 Flush extent 3 with f:2 g:2
Sep 22 23:15:08.781 DEBG Flush just extent 3 with f:2 and g:2
Sep 22 23:15:08.781 DEBG [1] It's time to notify for 264
Sep 22 23:15:08.781 INFO Completion from [1] id:264 status:true
Sep 22 23:15:08.781 INFO [265/752] Repair commands completed
Sep 22 23:15:08.781 INFO Pop front: ReconcileIO { id: ReconciliationId(265), op: ExtentClose { repair_id: ReconciliationId(265), extent_id: 3 }, state: ClientData([New, New, New]) }
Sep 22 23:15:08.781 INFO Sent repair work, now wait for resp
Sep 22 23:15:08.781 INFO [0] received reconcile message
Sep 22 23:15:08.781 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(265), op: ExtentClose { repair_id: ReconciliationId(265), extent_id: 3 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:08.781 INFO [0] client ExtentClose { repair_id: ReconciliationId(265), extent_id: 3 }
Sep 22 23:15:08.781 INFO [1] received reconcile message
Sep 22 23:15:08.781 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(265), op: ExtentClose { repair_id: ReconciliationId(265), extent_id: 3 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:08.781 INFO [1] client ExtentClose { repair_id: ReconciliationId(265), extent_id: 3 }
Sep 22 23:15:08.781 INFO [2] received reconcile message
Sep 22 23:15:08.781 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(265), op: ExtentClose { repair_id: ReconciliationId(265), extent_id: 3 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:08.781 INFO [2] client ExtentClose { repair_id: ReconciliationId(265), extent_id: 3 }
Sep 22 23:15:08.781 DEBG 265 Close extent 3
Sep 22 23:15:08.782 DEBG 265 Close extent 3
Sep 22 23:15:08.782 DEBG 265 Close extent 3
Sep 22 23:15:08.782 DEBG [2] It's time to notify for 265
Sep 22 23:15:08.782 INFO Completion from [2] id:265 status:true
Sep 22 23:15:08.782 INFO [266/752] Repair commands completed
Sep 22 23:15:08.782 INFO Pop front: ReconcileIO { id: ReconciliationId(266), op: ExtentRepair { repair_id: ReconciliationId(266), extent_id: 3, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:08.782 INFO Sent repair work, now wait for resp
Sep 22 23:15:08.782 INFO [0] received reconcile message
Sep 22 23:15:08.782 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(266), op: ExtentRepair { repair_id: ReconciliationId(266), extent_id: 3, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:08.782 INFO [0] client ExtentRepair { repair_id: ReconciliationId(266), extent_id: 3, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:08.782 INFO [0] Sending repair request ReconciliationId(266)
Sep 22 23:15:08.783 INFO [1] received reconcile message
Sep 22 23:15:08.783 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(266), op: ExtentRepair { repair_id: ReconciliationId(266), extent_id: 3, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:08.783 INFO [1] client ExtentRepair { repair_id: ReconciliationId(266), extent_id: 3, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:08.783 INFO [1] No action required ReconciliationId(266)
Sep 22 23:15:08.783 INFO [2] received reconcile message
Sep 22 23:15:08.783 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(266), op: ExtentRepair { repair_id: ReconciliationId(266), extent_id: 3, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:08.783 INFO [2] client ExtentRepair { repair_id: ReconciliationId(266), extent_id: 3, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:08.783 INFO [2] No action required ReconciliationId(266)
Sep 22 23:15:08.783 DEBG 266 Repair extent 3 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:08.783 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/003.copy"
Sep 22 23:15:08.847 INFO accepted connection, remote_addr: 127.0.0.1:44396, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.847 TRCE incoming request, uri: /extent/3/files, method: GET, req_id: 425a1a92-8976-4cdb-9dd8-fd3fac6fe0d0, remote_addr: 127.0.0.1:44396, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.847 INFO request completed, latency_us: 271, response_code: 200, uri: /extent/3/files, method: GET, req_id: 425a1a92-8976-4cdb-9dd8-fd3fac6fe0d0, remote_addr: 127.0.0.1:44396, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.848 INFO eid:3 Found repair files: ["003", "003.db"]
Sep 22 23:15:08.848 TRCE incoming request, uri: /newextent/3/data, method: GET, req_id: c9fe7f27-3e91-4497-ae33-9a0f28068798, remote_addr: 127.0.0.1:44396, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.848 INFO request completed, latency_us: 317, response_code: 200, uri: /newextent/3/data, method: GET, req_id: c9fe7f27-3e91-4497-ae33-9a0f28068798, remote_addr: 127.0.0.1:44396, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.853 TRCE incoming request, uri: /newextent/3/db, method: GET, req_id: d89aaba0-20b5-49b1-b550-0ec478d8eee1, remote_addr: 127.0.0.1:44396, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.854 INFO request completed, latency_us: 296, response_code: 200, uri: /newextent/3/db, method: GET, req_id: d89aaba0-20b5-49b1-b550-0ec478d8eee1, remote_addr: 127.0.0.1:44396, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.855 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/003.copy" to "/tmp/downstairs-vrx8aK6L/00/000/003.replace"
Sep 22 23:15:08.855 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:08.856 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/003.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:08.857 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/003"
Sep 22 23:15:08.857 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/003.db"
Sep 22 23:15:08.857 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:08.857 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/003.replace" to "/tmp/downstairs-vrx8aK6L/00/000/003.completed"
Sep 22 23:15:08.857 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:08.857 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:08.857 DEBG [0] It's time to notify for 266
Sep 22 23:15:08.857 INFO Completion from [0] id:266 status:true
Sep 22 23:15:08.857 INFO [267/752] Repair commands completed
Sep 22 23:15:08.857 INFO Pop front: ReconcileIO { id: ReconciliationId(267), op: ExtentReopen { repair_id: ReconciliationId(267), extent_id: 3 }, state: ClientData([New, New, New]) }
Sep 22 23:15:08.857 INFO Sent repair work, now wait for resp
Sep 22 23:15:08.857 INFO [0] received reconcile message
Sep 22 23:15:08.857 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(267), op: ExtentReopen { repair_id: ReconciliationId(267), extent_id: 3 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:08.857 INFO [0] client ExtentReopen { repair_id: ReconciliationId(267), extent_id: 3 }
Sep 22 23:15:08.857 INFO [1] received reconcile message
Sep 22 23:15:08.857 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(267), op: ExtentReopen { repair_id: ReconciliationId(267), extent_id: 3 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:08.857 INFO [1] client ExtentReopen { repair_id: ReconciliationId(267), extent_id: 3 }
Sep 22 23:15:08.857 INFO [2] received reconcile message
Sep 22 23:15:08.857 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(267), op: ExtentReopen { repair_id: ReconciliationId(267), extent_id: 3 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:08.858 INFO [2] client ExtentReopen { repair_id: ReconciliationId(267), extent_id: 3 }
Sep 22 23:15:08.858 DEBG 267 Reopen extent 3
Sep 22 23:15:08.858 DEBG 267 Reopen extent 3
Sep 22 23:15:08.859 DEBG 267 Reopen extent 3
Sep 22 23:15:08.860 DEBG [2] It's time to notify for 267
Sep 22 23:15:08.860 INFO Completion from [2] id:267 status:true
Sep 22 23:15:08.860 INFO [268/752] Repair commands completed
Sep 22 23:15:08.860 INFO Pop front: ReconcileIO { id: ReconciliationId(268), op: ExtentFlush { repair_id: ReconciliationId(268), extent_id: 54, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:08.860 INFO Sent repair work, now wait for resp
Sep 22 23:15:08.860 INFO [0] received reconcile message
Sep 22 23:15:08.860 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(268), op: ExtentFlush { repair_id: ReconciliationId(268), extent_id: 54, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:08.860 INFO [0] client ExtentFlush { repair_id: ReconciliationId(268), extent_id: 54, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:08.860 INFO [1] received reconcile message
Sep 22 23:15:08.860 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(268), op: ExtentFlush { repair_id: ReconciliationId(268), extent_id: 54, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:08.860 INFO [1] client ExtentFlush { repair_id: ReconciliationId(268), extent_id: 54, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:08.860 INFO [2] received reconcile message
Sep 22 23:15:08.860 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(268), op: ExtentFlush { repair_id: ReconciliationId(268), extent_id: 54, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:08.860 INFO [2] client ExtentFlush { repair_id: ReconciliationId(268), extent_id: 54, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:08.860 DEBG 268 Flush extent 54 with f:2 g:2
Sep 22 23:15:08.860 DEBG Flush just extent 54 with f:2 and g:2
Sep 22 23:15:08.860 DEBG [1] It's time to notify for 268
Sep 22 23:15:08.860 INFO Completion from [1] id:268 status:true
Sep 22 23:15:08.860 INFO [269/752] Repair commands completed
Sep 22 23:15:08.860 INFO Pop front: ReconcileIO { id: ReconciliationId(269), op: ExtentClose { repair_id: ReconciliationId(269), extent_id: 54 }, state: ClientData([New, New, New]) }
Sep 22 23:15:08.860 INFO Sent repair work, now wait for resp
Sep 22 23:15:08.860 INFO [0] received reconcile message
Sep 22 23:15:08.860 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(269), op: ExtentClose { repair_id: ReconciliationId(269), extent_id: 54 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:08.860 INFO [0] client ExtentClose { repair_id: ReconciliationId(269), extent_id: 54 }
Sep 22 23:15:08.860 INFO [1] received reconcile message
Sep 22 23:15:08.860 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(269), op: ExtentClose { repair_id: ReconciliationId(269), extent_id: 54 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:08.860 INFO [1] client ExtentClose { repair_id: ReconciliationId(269), extent_id: 54 }
Sep 22 23:15:08.860 INFO [2] received reconcile message
Sep 22 23:15:08.860 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(269), op: ExtentClose { repair_id: ReconciliationId(269), extent_id: 54 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:08.861 INFO [2] client ExtentClose { repair_id: ReconciliationId(269), extent_id: 54 }
Sep 22 23:15:08.861 DEBG 269 Close extent 54
Sep 22 23:15:08.861 DEBG 269 Close extent 54
Sep 22 23:15:08.861 DEBG 269 Close extent 54
Sep 22 23:15:08.862 DEBG [2] It's time to notify for 269
Sep 22 23:15:08.862 INFO Completion from [2] id:269 status:true
Sep 22 23:15:08.862 INFO [270/752] Repair commands completed
Sep 22 23:15:08.862 INFO Pop front: ReconcileIO { id: ReconciliationId(270), op: ExtentRepair { repair_id: ReconciliationId(270), extent_id: 54, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:08.862 INFO Sent repair work, now wait for resp
Sep 22 23:15:08.862 INFO [0] received reconcile message
Sep 22 23:15:08.862 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(270), op: ExtentRepair { repair_id: ReconciliationId(270), extent_id: 54, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:08.862 INFO [0] client ExtentRepair { repair_id: ReconciliationId(270), extent_id: 54, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:08.862 INFO [0] Sending repair request ReconciliationId(270)
Sep 22 23:15:08.862 INFO [1] received reconcile message
Sep 22 23:15:08.862 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(270), op: ExtentRepair { repair_id: ReconciliationId(270), extent_id: 54, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:08.862 INFO [1] client ExtentRepair { repair_id: ReconciliationId(270), extent_id: 54, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:08.862 INFO [1] No action required ReconciliationId(270)
Sep 22 23:15:08.862 INFO [2] received reconcile message
Sep 22 23:15:08.862 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(270), op: ExtentRepair { repair_id: ReconciliationId(270), extent_id: 54, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:08.862 INFO [2] client ExtentRepair { repair_id: ReconciliationId(270), extent_id: 54, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:08.862 INFO [2] No action required ReconciliationId(270)
Sep 22 23:15:08.862 DEBG 270 Repair extent 54 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:08.862 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/036.copy"
Sep 22 23:15:08.926 INFO accepted connection, remote_addr: 127.0.0.1:61574, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.926 TRCE incoming request, uri: /extent/54/files, method: GET, req_id: 928d57ed-1600-4df4-9eaa-2ca91051d98c, remote_addr: 127.0.0.1:61574, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.927 INFO request completed, latency_us: 277, response_code: 200, uri: /extent/54/files, method: GET, req_id: 928d57ed-1600-4df4-9eaa-2ca91051d98c, remote_addr: 127.0.0.1:61574, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.927 INFO eid:54 Found repair files: ["036", "036.db"]
Sep 22 23:15:08.927 TRCE incoming request, uri: /newextent/54/data, method: GET, req_id: 5cc2d47e-6f21-495e-9ed3-9fed6fbc16be, remote_addr: 127.0.0.1:61574, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.928 INFO request completed, latency_us: 369, response_code: 200, uri: /newextent/54/data, method: GET, req_id: 5cc2d47e-6f21-495e-9ed3-9fed6fbc16be, remote_addr: 127.0.0.1:61574, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.933 TRCE incoming request, uri: /newextent/54/db, method: GET, req_id: fc21011c-17ac-4efc-a91f-d4faaa8d624f, remote_addr: 127.0.0.1:61574, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.933 INFO request completed, latency_us: 303, response_code: 200, uri: /newextent/54/db, method: GET, req_id: fc21011c-17ac-4efc-a91f-d4faaa8d624f, remote_addr: 127.0.0.1:61574, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:08.934 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/036.copy" to "/tmp/downstairs-vrx8aK6L/00/000/036.replace"
Sep 22 23:15:08.934 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:08.936 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/036.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:08.936 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/036"
Sep 22 23:15:08.936 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/036.db"
Sep 22 23:15:08.936 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:08.936 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/036.replace" to "/tmp/downstairs-vrx8aK6L/00/000/036.completed"
Sep 22 23:15:08.936 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:08.936 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:08.937 DEBG [0] It's time to notify for 270
Sep 22 23:15:08.937 INFO Completion from [0] id:270 status:true
Sep 22 23:15:08.937 INFO [271/752] Repair commands completed
Sep 22 23:15:08.937 INFO Pop front: ReconcileIO { id: ReconciliationId(271), op: ExtentReopen { repair_id: ReconciliationId(271), extent_id: 54 }, state: ClientData([New, New, New]) }
Sep 22 23:15:08.937 INFO Sent repair work, now wait for resp
Sep 22 23:15:08.937 INFO [0] received reconcile message
Sep 22 23:15:08.937 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(271), op: ExtentReopen { repair_id: ReconciliationId(271), extent_id: 54 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:08.937 INFO [0] client ExtentReopen { repair_id: ReconciliationId(271), extent_id: 54 }
Sep 22 23:15:08.937 INFO [1] received reconcile message
Sep 22 23:15:08.937 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(271), op: ExtentReopen { repair_id: ReconciliationId(271), extent_id: 54 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:08.937 INFO [1] client ExtentReopen { repair_id: ReconciliationId(271), extent_id: 54 }
Sep 22 23:15:08.937 INFO [2] received reconcile message
Sep 22 23:15:08.937 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(271), op: ExtentReopen { repair_id: ReconciliationId(271), extent_id: 54 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:08.937 INFO [2] client ExtentReopen { repair_id: ReconciliationId(271), extent_id: 54 }
Sep 22 23:15:08.937 DEBG 271 Reopen extent 54
Sep 22 23:15:08.938 DEBG 271 Reopen extent 54
Sep 22 23:15:08.939 DEBG 271 Reopen extent 54
Sep 22 23:15:08.939 DEBG [2] It's time to notify for 271
Sep 22 23:15:08.939 INFO Completion from [2] id:271 status:true
Sep 22 23:15:08.939 INFO [272/752] Repair commands completed
Sep 22 23:15:08.939 INFO Pop front: ReconcileIO { id: ReconciliationId(272), op: ExtentFlush { repair_id: ReconciliationId(272), extent_id: 58, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:08.939 INFO Sent repair work, now wait for resp
Sep 22 23:15:08.939 INFO [0] received reconcile message
Sep 22 23:15:08.939 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(272), op: ExtentFlush { repair_id: ReconciliationId(272), extent_id: 58, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:08.939 INFO [0] client ExtentFlush { repair_id: ReconciliationId(272), extent_id: 58, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:08.939 INFO [1] received reconcile message
Sep 22 23:15:08.940 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(272), op: ExtentFlush { repair_id: ReconciliationId(272), extent_id: 58, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:08.940 INFO [1] client ExtentFlush { repair_id: ReconciliationId(272), extent_id: 58, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:08.940 INFO [2] received reconcile message
Sep 22 23:15:08.940 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(272), op: ExtentFlush { repair_id: ReconciliationId(272), extent_id: 58, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:08.940 INFO [2] client ExtentFlush { repair_id: ReconciliationId(272), extent_id: 58, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:08.940 DEBG 272 Flush extent 58 with f:2 g:2
Sep 22 23:15:08.940 DEBG Flush just extent 58 with f:2 and g:2
Sep 22 23:15:08.940 DEBG [1] It's time to notify for 272
Sep 22 23:15:08.940 INFO Completion from [1] id:272 status:true
Sep 22 23:15:08.940 INFO [273/752] Repair commands completed
Sep 22 23:15:08.940 INFO Pop front: ReconcileIO { id: ReconciliationId(273), op: ExtentClose { repair_id: ReconciliationId(273), extent_id: 58 }, state: ClientData([New, New, New]) }
Sep 22 23:15:08.940 INFO Sent repair work, now wait for resp
Sep 22 23:15:08.940 INFO [0] received reconcile message
Sep 22 23:15:08.940 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(273), op: ExtentClose { repair_id: ReconciliationId(273), extent_id: 58 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:08.940 INFO [0] client ExtentClose { repair_id: ReconciliationId(273), extent_id: 58 }
Sep 22 23:15:08.940 INFO [1] received reconcile message
Sep 22 23:15:08.940 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(273), op: ExtentClose { repair_id: ReconciliationId(273), extent_id: 58 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:08.940 INFO [1] client ExtentClose { repair_id: ReconciliationId(273), extent_id: 58 }
Sep 22 23:15:08.940 INFO [2] received reconcile message
Sep 22 23:15:08.940 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(273), op: ExtentClose { repair_id: ReconciliationId(273), extent_id: 58 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:08.940 INFO [2] client ExtentClose { repair_id: ReconciliationId(273), extent_id: 58 }
Sep 22 23:15:08.940 DEBG 273 Close extent 58
Sep 22 23:15:08.941 DEBG 273 Close extent 58
Sep 22 23:15:08.941 DEBG 273 Close extent 58
Sep 22 23:15:08.941 DEBG [2] It's time to notify for 273
Sep 22 23:15:08.941 INFO Completion from [2] id:273 status:true
Sep 22 23:15:08.941 INFO [274/752] Repair commands completed
Sep 22 23:15:08.941 INFO Pop front: ReconcileIO { id: ReconciliationId(274), op: ExtentRepair { repair_id: ReconciliationId(274), extent_id: 58, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:08.941 INFO Sent repair work, now wait for resp
Sep 22 23:15:08.941 INFO [0] received reconcile message
Sep 22 23:15:08.941 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(274), op: ExtentRepair { repair_id: ReconciliationId(274), extent_id: 58, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:08.941 INFO [0] client ExtentRepair { repair_id: ReconciliationId(274), extent_id: 58, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:08.942 INFO [0] Sending repair request ReconciliationId(274)
Sep 22 23:15:08.942 INFO [1] received reconcile message
Sep 22 23:15:08.942 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(274), op: ExtentRepair { repair_id: ReconciliationId(274), extent_id: 58, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:08.942 INFO [1] client ExtentRepair { repair_id: ReconciliationId(274), extent_id: 58, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:08.942 INFO [1] No action required ReconciliationId(274)
Sep 22 23:15:08.942 INFO [2] received reconcile message
Sep 22 23:15:08.942 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(274), op: ExtentRepair { repair_id: ReconciliationId(274), extent_id: 58, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:08.942 INFO [2] client ExtentRepair { repair_id: ReconciliationId(274), extent_id: 58, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:08.942 INFO [2] No action required ReconciliationId(274)
Sep 22 23:15:08.942 DEBG 274 Repair extent 58 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:08.942 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/03A.copy"
Sep 22 23:15:09.006 DEBG [rc] retire 1079 clears [JobId(1078), JobId(1079)], : downstairs
Sep 22 23:15:09.007 DEBG IO Flush 1081 has deps [JobId(1080)]
Sep 22 23:15:09.007 INFO [lossy] skipping 1080
Sep 22 23:15:09.007 INFO [lossy] skipping 1080
Sep 22 23:15:09.007 INFO [lossy] skipping 1080
Sep 22 23:15:09.007 INFO accepted connection, remote_addr: 127.0.0.1:59871, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.007 TRCE incoming request, uri: /extent/58/files, method: GET, req_id: 18219db5-28e5-4163-ae6f-aa8ec777c36e, remote_addr: 127.0.0.1:59871, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.008 INFO request completed, latency_us: 260, response_code: 200, uri: /extent/58/files, method: GET, req_id: 18219db5-28e5-4163-ae6f-aa8ec777c36e, remote_addr: 127.0.0.1:59871, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.008 INFO eid:58 Found repair files: ["03A", "03A.db"]
Sep 22 23:15:09.008 TRCE incoming request, uri: /newextent/58/data, method: GET, req_id: ba6329a9-77a3-4181-b2a6-8658cff93bf9, remote_addr: 127.0.0.1:59871, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.009 INFO request completed, latency_us: 360, response_code: 200, uri: /newextent/58/data, method: GET, req_id: ba6329a9-77a3-4181-b2a6-8658cff93bf9, remote_addr: 127.0.0.1:59871, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.013 DEBG Read      :1080 deps:[JobId(1079)] res:true
Sep 22 23:15:09.014 TRCE incoming request, uri: /newextent/58/db, method: GET, req_id: 889bebd9-31e2-4d74-8651-714e3d49581e, remote_addr: 127.0.0.1:59871, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.014 INFO request completed, latency_us: 294, response_code: 200, uri: /newextent/58/db, method: GET, req_id: 889bebd9-31e2-4d74-8651-714e3d49581e, remote_addr: 127.0.0.1:59871, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.015 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/03A.copy" to "/tmp/downstairs-vrx8aK6L/00/000/03A.replace"
Sep 22 23:15:09.015 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:09.016 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/03A.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:09.017 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/03A"
Sep 22 23:15:09.017 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/03A.db"
Sep 22 23:15:09.017 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:09.017 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/03A.replace" to "/tmp/downstairs-vrx8aK6L/00/000/03A.completed"
Sep 22 23:15:09.017 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:09.017 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:09.017 DEBG [0] It's time to notify for 274
Sep 22 23:15:09.017 INFO Completion from [0] id:274 status:true
Sep 22 23:15:09.017 INFO [275/752] Repair commands completed
Sep 22 23:15:09.017 INFO Pop front: ReconcileIO { id: ReconciliationId(275), op: ExtentReopen { repair_id: ReconciliationId(275), extent_id: 58 }, state: ClientData([New, New, New]) }
Sep 22 23:15:09.017 INFO Sent repair work, now wait for resp
Sep 22 23:15:09.017 INFO [0] received reconcile message
Sep 22 23:15:09.017 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(275), op: ExtentReopen { repair_id: ReconciliationId(275), extent_id: 58 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:09.017 INFO [0] client ExtentReopen { repair_id: ReconciliationId(275), extent_id: 58 }
Sep 22 23:15:09.017 INFO [1] received reconcile message
Sep 22 23:15:09.017 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(275), op: ExtentReopen { repair_id: ReconciliationId(275), extent_id: 58 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:09.017 INFO [1] client ExtentReopen { repair_id: ReconciliationId(275), extent_id: 58 }
Sep 22 23:15:09.018 INFO [2] received reconcile message
Sep 22 23:15:09.018 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(275), op: ExtentReopen { repair_id: ReconciliationId(275), extent_id: 58 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:09.018 INFO [2] client ExtentReopen { repair_id: ReconciliationId(275), extent_id: 58 }
Sep 22 23:15:09.018 DEBG 275 Reopen extent 58
Sep 22 23:15:09.018 DEBG 275 Reopen extent 58
Sep 22 23:15:09.019 DEBG 275 Reopen extent 58
Sep 22 23:15:09.020 DEBG [2] It's time to notify for 275
Sep 22 23:15:09.020 INFO Completion from [2] id:275 status:true
Sep 22 23:15:09.020 INFO [276/752] Repair commands completed
Sep 22 23:15:09.020 INFO Pop front: ReconcileIO { id: ReconciliationId(276), op: ExtentFlush { repair_id: ReconciliationId(276), extent_id: 18, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:09.020 INFO Sent repair work, now wait for resp
Sep 22 23:15:09.020 INFO [0] received reconcile message
Sep 22 23:15:09.020 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(276), op: ExtentFlush { repair_id: ReconciliationId(276), extent_id: 18, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:09.020 INFO [0] client ExtentFlush { repair_id: ReconciliationId(276), extent_id: 18, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:09.020 INFO [1] received reconcile message
Sep 22 23:15:09.020 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(276), op: ExtentFlush { repair_id: ReconciliationId(276), extent_id: 18, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:09.020 INFO [1] client ExtentFlush { repair_id: ReconciliationId(276), extent_id: 18, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:09.020 INFO [2] received reconcile message
Sep 22 23:15:09.020 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(276), op: ExtentFlush { repair_id: ReconciliationId(276), extent_id: 18, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:09.020 INFO [2] client ExtentFlush { repair_id: ReconciliationId(276), extent_id: 18, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:09.020 DEBG 276 Flush extent 18 with f:2 g:2
Sep 22 23:15:09.020 DEBG Flush just extent 18 with f:2 and g:2
Sep 22 23:15:09.020 DEBG [1] It's time to notify for 276
Sep 22 23:15:09.020 INFO Completion from [1] id:276 status:true
Sep 22 23:15:09.020 INFO [277/752] Repair commands completed
Sep 22 23:15:09.020 INFO Pop front: ReconcileIO { id: ReconciliationId(277), op: ExtentClose { repair_id: ReconciliationId(277), extent_id: 18 }, state: ClientData([New, New, New]) }
Sep 22 23:15:09.020 INFO Sent repair work, now wait for resp
Sep 22 23:15:09.020 INFO [0] received reconcile message
Sep 22 23:15:09.020 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(277), op: ExtentClose { repair_id: ReconciliationId(277), extent_id: 18 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:09.020 INFO [0] client ExtentClose { repair_id: ReconciliationId(277), extent_id: 18 }
Sep 22 23:15:09.020 INFO [1] received reconcile message
Sep 22 23:15:09.020 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(277), op: ExtentClose { repair_id: ReconciliationId(277), extent_id: 18 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:09.020 INFO [1] client ExtentClose { repair_id: ReconciliationId(277), extent_id: 18 }
Sep 22 23:15:09.020 INFO [2] received reconcile message
Sep 22 23:15:09.020 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(277), op: ExtentClose { repair_id: ReconciliationId(277), extent_id: 18 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:09.020 INFO [2] client ExtentClose { repair_id: ReconciliationId(277), extent_id: 18 }
Sep 22 23:15:09.021 DEBG 277 Close extent 18
Sep 22 23:15:09.021 DEBG 277 Close extent 18
Sep 22 23:15:09.021 DEBG 277 Close extent 18
Sep 22 23:15:09.022 DEBG [2] It's time to notify for 277
Sep 22 23:15:09.022 INFO Completion from [2] id:277 status:true
Sep 22 23:15:09.022 INFO [278/752] Repair commands completed
Sep 22 23:15:09.022 INFO Pop front: ReconcileIO { id: ReconciliationId(278), op: ExtentRepair { repair_id: ReconciliationId(278), extent_id: 18, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:09.022 INFO Sent repair work, now wait for resp
Sep 22 23:15:09.022 INFO [0] received reconcile message
Sep 22 23:15:09.022 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(278), op: ExtentRepair { repair_id: ReconciliationId(278), extent_id: 18, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:09.022 INFO [0] client ExtentRepair { repair_id: ReconciliationId(278), extent_id: 18, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:09.022 INFO [0] Sending repair request ReconciliationId(278)
Sep 22 23:15:09.022 INFO [1] received reconcile message
Sep 22 23:15:09.022 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(278), op: ExtentRepair { repair_id: ReconciliationId(278), extent_id: 18, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:09.022 INFO [1] client ExtentRepair { repair_id: ReconciliationId(278), extent_id: 18, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:09.022 INFO [1] No action required ReconciliationId(278)
Sep 22 23:15:09.022 INFO [2] received reconcile message
Sep 22 23:15:09.022 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(278), op: ExtentRepair { repair_id: ReconciliationId(278), extent_id: 18, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:09.022 INFO [2] client ExtentRepair { repair_id: ReconciliationId(278), extent_id: 18, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:09.022 INFO [2] No action required ReconciliationId(278)
Sep 22 23:15:09.022 DEBG 278 Repair extent 18 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:09.022 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/012.copy"
Sep 22 23:15:09.041 DEBG Read      :1080 deps:[JobId(1079)] res:true
Sep 22 23:15:09.065 DEBG Flush     :1081 extent_limit None deps:[JobId(1080)] res:true f:30 g:1
Sep 22 23:15:09.066 INFO [lossy] sleeping 1 second
Sep 22 23:15:09.086 INFO accepted connection, remote_addr: 127.0.0.1:63513, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.086 TRCE incoming request, uri: /extent/18/files, method: GET, req_id: 35996c0e-b820-4ceb-9a8c-30ee8c71fd40, remote_addr: 127.0.0.1:63513, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.086 INFO request completed, latency_us: 258, response_code: 200, uri: /extent/18/files, method: GET, req_id: 35996c0e-b820-4ceb-9a8c-30ee8c71fd40, remote_addr: 127.0.0.1:63513, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.087 INFO eid:18 Found repair files: ["012", "012.db"]
Sep 22 23:15:09.087 TRCE incoming request, uri: /newextent/18/data, method: GET, req_id: 6d31bf7d-4b15-4e1f-a8c9-1513d1dbc2b2, remote_addr: 127.0.0.1:63513, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.087 INFO request completed, latency_us: 325, response_code: 200, uri: /newextent/18/data, method: GET, req_id: 6d31bf7d-4b15-4e1f-a8c9-1513d1dbc2b2, remote_addr: 127.0.0.1:63513, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.092 TRCE incoming request, uri: /newextent/18/db, method: GET, req_id: 6351641e-f6ab-4547-838b-56b8e37a6d52, remote_addr: 127.0.0.1:63513, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.093 INFO request completed, latency_us: 296, response_code: 200, uri: /newextent/18/db, method: GET, req_id: 6351641e-f6ab-4547-838b-56b8e37a6d52, remote_addr: 127.0.0.1:63513, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.094 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/012.copy" to "/tmp/downstairs-vrx8aK6L/00/000/012.replace"
Sep 22 23:15:09.094 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:09.095 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/012.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:09.095 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/012"
Sep 22 23:15:09.095 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/012.db"
Sep 22 23:15:09.095 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:09.095 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/012.replace" to "/tmp/downstairs-vrx8aK6L/00/000/012.completed"
Sep 22 23:15:09.095 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:09.095 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:09.096 DEBG [0] It's time to notify for 278
Sep 22 23:15:09.096 INFO Completion from [0] id:278 status:true
Sep 22 23:15:09.096 INFO [279/752] Repair commands completed
Sep 22 23:15:09.096 INFO Pop front: ReconcileIO { id: ReconciliationId(279), op: ExtentReopen { repair_id: ReconciliationId(279), extent_id: 18 }, state: ClientData([New, New, New]) }
Sep 22 23:15:09.096 INFO Sent repair work, now wait for resp
Sep 22 23:15:09.096 INFO [0] received reconcile message
Sep 22 23:15:09.096 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(279), op: ExtentReopen { repair_id: ReconciliationId(279), extent_id: 18 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:09.096 INFO [0] client ExtentReopen { repair_id: ReconciliationId(279), extent_id: 18 }
Sep 22 23:15:09.096 INFO [1] received reconcile message
Sep 22 23:15:09.096 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(279), op: ExtentReopen { repair_id: ReconciliationId(279), extent_id: 18 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:09.096 INFO [1] client ExtentReopen { repair_id: ReconciliationId(279), extent_id: 18 }
Sep 22 23:15:09.096 INFO [2] received reconcile message
Sep 22 23:15:09.096 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(279), op: ExtentReopen { repair_id: ReconciliationId(279), extent_id: 18 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:09.096 INFO [2] client ExtentReopen { repair_id: ReconciliationId(279), extent_id: 18 }
Sep 22 23:15:09.096 DEBG 279 Reopen extent 18
Sep 22 23:15:09.097 DEBG 279 Reopen extent 18
Sep 22 23:15:09.097 DEBG 279 Reopen extent 18
Sep 22 23:15:09.098 DEBG [2] It's time to notify for 279
Sep 22 23:15:09.098 INFO Completion from [2] id:279 status:true
Sep 22 23:15:09.098 INFO [280/752] Repair commands completed
Sep 22 23:15:09.098 INFO Pop front: ReconcileIO { id: ReconciliationId(280), op: ExtentFlush { repair_id: ReconciliationId(280), extent_id: 138, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:09.098 INFO Sent repair work, now wait for resp
Sep 22 23:15:09.098 INFO [0] received reconcile message
Sep 22 23:15:09.098 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(280), op: ExtentFlush { repair_id: ReconciliationId(280), extent_id: 138, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:09.098 INFO [0] client ExtentFlush { repair_id: ReconciliationId(280), extent_id: 138, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:09.098 INFO [1] received reconcile message
Sep 22 23:15:09.098 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(280), op: ExtentFlush { repair_id: ReconciliationId(280), extent_id: 138, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:09.098 INFO [1] client ExtentFlush { repair_id: ReconciliationId(280), extent_id: 138, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:09.098 INFO [2] received reconcile message
Sep 22 23:15:09.098 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(280), op: ExtentFlush { repair_id: ReconciliationId(280), extent_id: 138, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:09.098 INFO [2] client ExtentFlush { repair_id: ReconciliationId(280), extent_id: 138, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:09.098 DEBG 280 Flush extent 138 with f:2 g:2
Sep 22 23:15:09.098 DEBG Flush just extent 138 with f:2 and g:2
Sep 22 23:15:09.099 DEBG [1] It's time to notify for 280
Sep 22 23:15:09.099 INFO Completion from [1] id:280 status:true
Sep 22 23:15:09.099 INFO [281/752] Repair commands completed
Sep 22 23:15:09.099 INFO Pop front: ReconcileIO { id: ReconciliationId(281), op: ExtentClose { repair_id: ReconciliationId(281), extent_id: 138 }, state: ClientData([New, New, New]) }
Sep 22 23:15:09.099 INFO Sent repair work, now wait for resp
Sep 22 23:15:09.099 INFO [0] received reconcile message
Sep 22 23:15:09.099 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(281), op: ExtentClose { repair_id: ReconciliationId(281), extent_id: 138 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:09.099 INFO [0] client ExtentClose { repair_id: ReconciliationId(281), extent_id: 138 }
Sep 22 23:15:09.099 INFO [1] received reconcile message
Sep 22 23:15:09.099 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(281), op: ExtentClose { repair_id: ReconciliationId(281), extent_id: 138 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:09.099 INFO [1] client ExtentClose { repair_id: ReconciliationId(281), extent_id: 138 }
Sep 22 23:15:09.099 INFO [2] received reconcile message
Sep 22 23:15:09.099 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(281), op: ExtentClose { repair_id: ReconciliationId(281), extent_id: 138 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:09.099 INFO [2] client ExtentClose { repair_id: ReconciliationId(281), extent_id: 138 }
Sep 22 23:15:09.099 DEBG 281 Close extent 138
Sep 22 23:15:09.099 DEBG 281 Close extent 138
Sep 22 23:15:09.100 DEBG 281 Close extent 138
Sep 22 23:15:09.100 DEBG [2] It's time to notify for 281
Sep 22 23:15:09.100 INFO Completion from [2] id:281 status:true
Sep 22 23:15:09.100 INFO [282/752] Repair commands completed
Sep 22 23:15:09.100 INFO Pop front: ReconcileIO { id: ReconciliationId(282), op: ExtentRepair { repair_id: ReconciliationId(282), extent_id: 138, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:09.100 INFO Sent repair work, now wait for resp
Sep 22 23:15:09.100 INFO [0] received reconcile message
Sep 22 23:15:09.100 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(282), op: ExtentRepair { repair_id: ReconciliationId(282), extent_id: 138, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:09.100 INFO [0] client ExtentRepair { repair_id: ReconciliationId(282), extent_id: 138, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:09.100 INFO [0] Sending repair request ReconciliationId(282)
Sep 22 23:15:09.100 INFO [1] received reconcile message
Sep 22 23:15:09.100 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(282), op: ExtentRepair { repair_id: ReconciliationId(282), extent_id: 138, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:09.100 INFO [1] client ExtentRepair { repair_id: ReconciliationId(282), extent_id: 138, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:09.100 INFO [1] No action required ReconciliationId(282)
Sep 22 23:15:09.100 INFO [2] received reconcile message
Sep 22 23:15:09.100 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(282), op: ExtentRepair { repair_id: ReconciliationId(282), extent_id: 138, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:09.100 INFO [2] client ExtentRepair { repair_id: ReconciliationId(282), extent_id: 138, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:09.100 INFO [2] No action required ReconciliationId(282)
Sep 22 23:15:09.100 DEBG 282 Repair extent 138 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:09.101 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/08A.copy"
Sep 22 23:15:09.162 INFO accepted connection, remote_addr: 127.0.0.1:39632, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.163 TRCE incoming request, uri: /extent/138/files, method: GET, req_id: 78543559-ffd9-4cf2-9a57-38d01d59d88a, remote_addr: 127.0.0.1:39632, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.163 INFO request completed, latency_us: 243, response_code: 200, uri: /extent/138/files, method: GET, req_id: 78543559-ffd9-4cf2-9a57-38d01d59d88a, remote_addr: 127.0.0.1:39632, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.163 INFO eid:138 Found repair files: ["08A", "08A.db"]
Sep 22 23:15:09.164 TRCE incoming request, uri: /newextent/138/data, method: GET, req_id: fcc43585-6ab5-403c-b2ae-0fa763745671, remote_addr: 127.0.0.1:39632, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.164 INFO request completed, latency_us: 329, response_code: 200, uri: /newextent/138/data, method: GET, req_id: fcc43585-6ab5-403c-b2ae-0fa763745671, remote_addr: 127.0.0.1:39632, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.169 TRCE incoming request, uri: /newextent/138/db, method: GET, req_id: 0e7e9c0f-6b76-423f-84f4-bb8939bf22b9, remote_addr: 127.0.0.1:39632, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.169 INFO request completed, latency_us: 290, response_code: 200, uri: /newextent/138/db, method: GET, req_id: 0e7e9c0f-6b76-423f-84f4-bb8939bf22b9, remote_addr: 127.0.0.1:39632, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.171 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/08A.copy" to "/tmp/downstairs-vrx8aK6L/00/000/08A.replace"
Sep 22 23:15:09.171 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:09.171 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/08A.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:09.172 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/08A"
Sep 22 23:15:09.172 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/08A.db"
Sep 22 23:15:09.172 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:09.172 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/08A.replace" to "/tmp/downstairs-vrx8aK6L/00/000/08A.completed"
Sep 22 23:15:09.172 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:09.172 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:09.172 DEBG [0] It's time to notify for 282
Sep 22 23:15:09.172 INFO Completion from [0] id:282 status:true
Sep 22 23:15:09.172 INFO [283/752] Repair commands completed
Sep 22 23:15:09.172 INFO Pop front: ReconcileIO { id: ReconciliationId(283), op: ExtentReopen { repair_id: ReconciliationId(283), extent_id: 138 }, state: ClientData([New, New, New]) }
Sep 22 23:15:09.172 INFO Sent repair work, now wait for resp
Sep 22 23:15:09.172 INFO [0] received reconcile message
Sep 22 23:15:09.172 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(283), op: ExtentReopen { repair_id: ReconciliationId(283), extent_id: 138 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:09.172 INFO [0] client ExtentReopen { repair_id: ReconciliationId(283), extent_id: 138 }
Sep 22 23:15:09.172 INFO [1] received reconcile message
Sep 22 23:15:09.172 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(283), op: ExtentReopen { repair_id: ReconciliationId(283), extent_id: 138 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:09.172 INFO [1] client ExtentReopen { repair_id: ReconciliationId(283), extent_id: 138 }
Sep 22 23:15:09.173 INFO [2] received reconcile message
Sep 22 23:15:09.173 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(283), op: ExtentReopen { repair_id: ReconciliationId(283), extent_id: 138 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:09.173 INFO [2] client ExtentReopen { repair_id: ReconciliationId(283), extent_id: 138 }
Sep 22 23:15:09.173 DEBG 283 Reopen extent 138
Sep 22 23:15:09.173 DEBG 283 Reopen extent 138
Sep 22 23:15:09.174 DEBG 283 Reopen extent 138
Sep 22 23:15:09.175 DEBG [2] It's time to notify for 283
Sep 22 23:15:09.175 INFO Completion from [2] id:283 status:true
Sep 22 23:15:09.175 INFO [284/752] Repair commands completed
Sep 22 23:15:09.175 INFO Pop front: ReconcileIO { id: ReconciliationId(284), op: ExtentFlush { repair_id: ReconciliationId(284), extent_id: 100, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:09.175 INFO Sent repair work, now wait for resp
Sep 22 23:15:09.175 INFO [0] received reconcile message
Sep 22 23:15:09.175 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(284), op: ExtentFlush { repair_id: ReconciliationId(284), extent_id: 100, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:09.175 INFO [0] client ExtentFlush { repair_id: ReconciliationId(284), extent_id: 100, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:09.175 INFO [1] received reconcile message
Sep 22 23:15:09.175 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(284), op: ExtentFlush { repair_id: ReconciliationId(284), extent_id: 100, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:09.175 INFO [1] client ExtentFlush { repair_id: ReconciliationId(284), extent_id: 100, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:09.175 INFO [2] received reconcile message
Sep 22 23:15:09.175 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(284), op: ExtentFlush { repair_id: ReconciliationId(284), extent_id: 100, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:09.175 INFO [2] client ExtentFlush { repair_id: ReconciliationId(284), extent_id: 100, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:09.175 DEBG 284 Flush extent 100 with f:2 g:2
Sep 22 23:15:09.175 DEBG Flush just extent 100 with f:2 and g:2
Sep 22 23:15:09.175 DEBG [1] It's time to notify for 284
Sep 22 23:15:09.175 INFO Completion from [1] id:284 status:true
Sep 22 23:15:09.175 INFO [285/752] Repair commands completed
Sep 22 23:15:09.175 INFO Pop front: ReconcileIO { id: ReconciliationId(285), op: ExtentClose { repair_id: ReconciliationId(285), extent_id: 100 }, state: ClientData([New, New, New]) }
Sep 22 23:15:09.175 INFO Sent repair work, now wait for resp
Sep 22 23:15:09.175 INFO [0] received reconcile message
Sep 22 23:15:09.175 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(285), op: ExtentClose { repair_id: ReconciliationId(285), extent_id: 100 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:09.175 INFO [0] client ExtentClose { repair_id: ReconciliationId(285), extent_id: 100 }
Sep 22 23:15:09.175 INFO [1] received reconcile message
Sep 22 23:15:09.175 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(285), op: ExtentClose { repair_id: ReconciliationId(285), extent_id: 100 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:09.175 INFO [1] client ExtentClose { repair_id: ReconciliationId(285), extent_id: 100 }
Sep 22 23:15:09.175 INFO [2] received reconcile message
Sep 22 23:15:09.175 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(285), op: ExtentClose { repair_id: ReconciliationId(285), extent_id: 100 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:09.175 INFO [2] client ExtentClose { repair_id: ReconciliationId(285), extent_id: 100 }
Sep 22 23:15:09.176 DEBG 285 Close extent 100
Sep 22 23:15:09.176 DEBG 285 Close extent 100
Sep 22 23:15:09.176 DEBG 285 Close extent 100
Sep 22 23:15:09.177 DEBG [2] It's time to notify for 285
Sep 22 23:15:09.177 INFO Completion from [2] id:285 status:true
Sep 22 23:15:09.177 INFO [286/752] Repair commands completed
Sep 22 23:15:09.177 INFO Pop front: ReconcileIO { id: ReconciliationId(286), op: ExtentRepair { repair_id: ReconciliationId(286), extent_id: 100, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:09.177 INFO Sent repair work, now wait for resp
Sep 22 23:15:09.177 INFO [0] received reconcile message
Sep 22 23:15:09.177 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(286), op: ExtentRepair { repair_id: ReconciliationId(286), extent_id: 100, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:09.177 INFO [0] client ExtentRepair { repair_id: ReconciliationId(286), extent_id: 100, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:09.177 INFO [0] Sending repair request ReconciliationId(286)
Sep 22 23:15:09.177 INFO [1] received reconcile message
Sep 22 23:15:09.177 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(286), op: ExtentRepair { repair_id: ReconciliationId(286), extent_id: 100, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:09.177 INFO [1] client ExtentRepair { repair_id: ReconciliationId(286), extent_id: 100, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:09.177 INFO [1] No action required ReconciliationId(286)
Sep 22 23:15:09.177 INFO [2] received reconcile message
Sep 22 23:15:09.177 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(286), op: ExtentRepair { repair_id: ReconciliationId(286), extent_id: 100, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:09.177 INFO [2] client ExtentRepair { repair_id: ReconciliationId(286), extent_id: 100, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:09.177 INFO [2] No action required ReconciliationId(286)
Sep 22 23:15:09.177 DEBG 286 Repair extent 100 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:09.177 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/064.copy"
Sep 22 23:15:09.241 INFO accepted connection, remote_addr: 127.0.0.1:43266, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.241 TRCE incoming request, uri: /extent/100/files, method: GET, req_id: 40a0c71e-9925-402b-b559-c95aef17a84e, remote_addr: 127.0.0.1:43266, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.241 INFO request completed, latency_us: 225, response_code: 200, uri: /extent/100/files, method: GET, req_id: 40a0c71e-9925-402b-b559-c95aef17a84e, remote_addr: 127.0.0.1:43266, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.241 INFO eid:100 Found repair files: ["064", "064.db"]
Sep 22 23:15:09.242 TRCE incoming request, uri: /newextent/100/data, method: GET, req_id: 624548a2-bdae-4018-b3d0-846abe0e89ec, remote_addr: 127.0.0.1:43266, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.242 INFO request completed, latency_us: 342, response_code: 200, uri: /newextent/100/data, method: GET, req_id: 624548a2-bdae-4018-b3d0-846abe0e89ec, remote_addr: 127.0.0.1:43266, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.247 TRCE incoming request, uri: /newextent/100/db, method: GET, req_id: 1e277602-6ad6-49af-9002-c37040960090, remote_addr: 127.0.0.1:43266, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.248 INFO request completed, latency_us: 296, response_code: 200, uri: /newextent/100/db, method: GET, req_id: 1e277602-6ad6-49af-9002-c37040960090, remote_addr: 127.0.0.1:43266, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.249 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/064.copy" to "/tmp/downstairs-vrx8aK6L/00/000/064.replace"
Sep 22 23:15:09.249 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:09.250 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/064.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:09.250 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/064"
Sep 22 23:15:09.250 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/064.db"
Sep 22 23:15:09.250 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:09.250 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/064.replace" to "/tmp/downstairs-vrx8aK6L/00/000/064.completed"
Sep 22 23:15:09.250 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:09.250 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:09.250 DEBG [0] It's time to notify for 286
Sep 22 23:15:09.250 INFO Completion from [0] id:286 status:true
Sep 22 23:15:09.250 INFO [287/752] Repair commands completed
Sep 22 23:15:09.250 INFO Pop front: ReconcileIO { id: ReconciliationId(287), op: ExtentReopen { repair_id: ReconciliationId(287), extent_id: 100 }, state: ClientData([New, New, New]) }
Sep 22 23:15:09.251 INFO Sent repair work, now wait for resp
Sep 22 23:15:09.251 INFO [0] received reconcile message
Sep 22 23:15:09.251 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(287), op: ExtentReopen { repair_id: ReconciliationId(287), extent_id: 100 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:09.251 INFO [0] client ExtentReopen { repair_id: ReconciliationId(287), extent_id: 100 }
Sep 22 23:15:09.251 INFO [1] received reconcile message
Sep 22 23:15:09.251 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(287), op: ExtentReopen { repair_id: ReconciliationId(287), extent_id: 100 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:09.251 INFO [1] client ExtentReopen { repair_id: ReconciliationId(287), extent_id: 100 }
Sep 22 23:15:09.251 INFO [2] received reconcile message
Sep 22 23:15:09.251 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(287), op: ExtentReopen { repair_id: ReconciliationId(287), extent_id: 100 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:09.251 INFO [2] client ExtentReopen { repair_id: ReconciliationId(287), extent_id: 100 }
Sep 22 23:15:09.251 DEBG 287 Reopen extent 100
Sep 22 23:15:09.252 DEBG 287 Reopen extent 100
Sep 22 23:15:09.252 DEBG 287 Reopen extent 100
Sep 22 23:15:09.253 DEBG [2] It's time to notify for 287
Sep 22 23:15:09.253 INFO Completion from [2] id:287 status:true
Sep 22 23:15:09.253 INFO [288/752] Repair commands completed
Sep 22 23:15:09.253 INFO Pop front: ReconcileIO { id: ReconciliationId(288), op: ExtentFlush { repair_id: ReconciliationId(288), extent_id: 123, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:09.253 INFO Sent repair work, now wait for resp
Sep 22 23:15:09.253 INFO [0] received reconcile message
Sep 22 23:15:09.253 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(288), op: ExtentFlush { repair_id: ReconciliationId(288), extent_id: 123, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:09.253 INFO [0] client ExtentFlush { repair_id: ReconciliationId(288), extent_id: 123, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:09.253 INFO [1] received reconcile message
Sep 22 23:15:09.253 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(288), op: ExtentFlush { repair_id: ReconciliationId(288), extent_id: 123, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:09.253 INFO [1] client ExtentFlush { repair_id: ReconciliationId(288), extent_id: 123, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:09.253 INFO [2] received reconcile message
Sep 22 23:15:09.253 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(288), op: ExtentFlush { repair_id: ReconciliationId(288), extent_id: 123, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:09.253 INFO [2] client ExtentFlush { repair_id: ReconciliationId(288), extent_id: 123, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:09.253 DEBG 288 Flush extent 123 with f:2 g:2
Sep 22 23:15:09.253 DEBG Flush just extent 123 with f:2 and g:2
Sep 22 23:15:09.253 DEBG [1] It's time to notify for 288
Sep 22 23:15:09.253 INFO Completion from [1] id:288 status:true
Sep 22 23:15:09.253 INFO [289/752] Repair commands completed
Sep 22 23:15:09.253 INFO Pop front: ReconcileIO { id: ReconciliationId(289), op: ExtentClose { repair_id: ReconciliationId(289), extent_id: 123 }, state: ClientData([New, New, New]) }
Sep 22 23:15:09.253 INFO Sent repair work, now wait for resp
Sep 22 23:15:09.253 INFO [0] received reconcile message
Sep 22 23:15:09.253 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(289), op: ExtentClose { repair_id: ReconciliationId(289), extent_id: 123 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:09.253 INFO [0] client ExtentClose { repair_id: ReconciliationId(289), extent_id: 123 }
Sep 22 23:15:09.253 INFO [1] received reconcile message
Sep 22 23:15:09.254 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(289), op: ExtentClose { repair_id: ReconciliationId(289), extent_id: 123 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:09.254 INFO [1] client ExtentClose { repair_id: ReconciliationId(289), extent_id: 123 }
Sep 22 23:15:09.254 INFO [2] received reconcile message
Sep 22 23:15:09.254 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(289), op: ExtentClose { repair_id: ReconciliationId(289), extent_id: 123 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:09.254 INFO [2] client ExtentClose { repair_id: ReconciliationId(289), extent_id: 123 }
Sep 22 23:15:09.254 DEBG 289 Close extent 123
Sep 22 23:15:09.254 DEBG 289 Close extent 123
Sep 22 23:15:09.254 DEBG 289 Close extent 123
Sep 22 23:15:09.255 DEBG [2] It's time to notify for 289
Sep 22 23:15:09.255 INFO Completion from [2] id:289 status:true
Sep 22 23:15:09.255 INFO [290/752] Repair commands completed
Sep 22 23:15:09.255 INFO Pop front: ReconcileIO { id: ReconciliationId(290), op: ExtentRepair { repair_id: ReconciliationId(290), extent_id: 123, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:09.255 INFO Sent repair work, now wait for resp
Sep 22 23:15:09.255 INFO [0] received reconcile message
Sep 22 23:15:09.255 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(290), op: ExtentRepair { repair_id: ReconciliationId(290), extent_id: 123, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:09.255 INFO [0] client ExtentRepair { repair_id: ReconciliationId(290), extent_id: 123, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:09.255 INFO [0] Sending repair request ReconciliationId(290)
Sep 22 23:15:09.255 INFO [1] received reconcile message
Sep 22 23:15:09.255 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(290), op: ExtentRepair { repair_id: ReconciliationId(290), extent_id: 123, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:09.255 INFO [1] client ExtentRepair { repair_id: ReconciliationId(290), extent_id: 123, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:09.255 INFO [1] No action required ReconciliationId(290)
Sep 22 23:15:09.255 INFO [2] received reconcile message
Sep 22 23:15:09.255 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(290), op: ExtentRepair { repair_id: ReconciliationId(290), extent_id: 123, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:09.255 INFO [2] client ExtentRepair { repair_id: ReconciliationId(290), extent_id: 123, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:09.255 INFO [2] No action required ReconciliationId(290)
Sep 22 23:15:09.255 DEBG 290 Repair extent 123 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:09.255 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/07B.copy"
Sep 22 23:15:09.319 INFO accepted connection, remote_addr: 127.0.0.1:57416, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.319 TRCE incoming request, uri: /extent/123/files, method: GET, req_id: 2ca770b5-e4b4-44f4-a8f4-0935e3cca897, remote_addr: 127.0.0.1:57416, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.319 INFO request completed, latency_us: 204, response_code: 200, uri: /extent/123/files, method: GET, req_id: 2ca770b5-e4b4-44f4-a8f4-0935e3cca897, remote_addr: 127.0.0.1:57416, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.319 INFO eid:123 Found repair files: ["07B", "07B.db"]
Sep 22 23:15:09.320 TRCE incoming request, uri: /newextent/123/data, method: GET, req_id: 219edfbc-7f84-4c0f-b1ec-9607a7ae1d20, remote_addr: 127.0.0.1:57416, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.320 INFO request completed, latency_us: 306, response_code: 200, uri: /newextent/123/data, method: GET, req_id: 219edfbc-7f84-4c0f-b1ec-9607a7ae1d20, remote_addr: 127.0.0.1:57416, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.325 TRCE incoming request, uri: /newextent/123/db, method: GET, req_id: 1478200b-e0ae-486b-ace8-26204d330287, remote_addr: 127.0.0.1:57416, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.325 INFO request completed, latency_us: 314, response_code: 200, uri: /newextent/123/db, method: GET, req_id: 1478200b-e0ae-486b-ace8-26204d330287, remote_addr: 127.0.0.1:57416, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.327 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/07B.copy" to "/tmp/downstairs-vrx8aK6L/00/000/07B.replace"
Sep 22 23:15:09.327 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:09.327 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/07B.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:09.328 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/07B"
Sep 22 23:15:09.328 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/07B.db"
Sep 22 23:15:09.328 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:09.328 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/07B.replace" to "/tmp/downstairs-vrx8aK6L/00/000/07B.completed"
Sep 22 23:15:09.328 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:09.328 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:09.328 DEBG [0] It's time to notify for 290
Sep 22 23:15:09.328 INFO Completion from [0] id:290 status:true
Sep 22 23:15:09.328 INFO [291/752] Repair commands completed
Sep 22 23:15:09.328 INFO Pop front: ReconcileIO { id: ReconciliationId(291), op: ExtentReopen { repair_id: ReconciliationId(291), extent_id: 123 }, state: ClientData([New, New, New]) }
Sep 22 23:15:09.328 INFO Sent repair work, now wait for resp
Sep 22 23:15:09.328 INFO [0] received reconcile message
Sep 22 23:15:09.328 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(291), op: ExtentReopen { repair_id: ReconciliationId(291), extent_id: 123 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:09.328 INFO [0] client ExtentReopen { repair_id: ReconciliationId(291), extent_id: 123 }
Sep 22 23:15:09.328 INFO [1] received reconcile message
Sep 22 23:15:09.328 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(291), op: ExtentReopen { repair_id: ReconciliationId(291), extent_id: 123 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:09.328 INFO [1] client ExtentReopen { repair_id: ReconciliationId(291), extent_id: 123 }
Sep 22 23:15:09.328 INFO [2] received reconcile message
Sep 22 23:15:09.329 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(291), op: ExtentReopen { repair_id: ReconciliationId(291), extent_id: 123 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:09.329 INFO [2] client ExtentReopen { repair_id: ReconciliationId(291), extent_id: 123 }
Sep 22 23:15:09.329 DEBG 291 Reopen extent 123
Sep 22 23:15:09.329 DEBG 291 Reopen extent 123
Sep 22 23:15:09.330 DEBG 291 Reopen extent 123
Sep 22 23:15:09.330 DEBG [2] It's time to notify for 291
Sep 22 23:15:09.330 INFO Completion from [2] id:291 status:true
Sep 22 23:15:09.330 INFO [292/752] Repair commands completed
Sep 22 23:15:09.330 INFO Pop front: ReconcileIO { id: ReconciliationId(292), op: ExtentFlush { repair_id: ReconciliationId(292), extent_id: 96, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:09.330 INFO Sent repair work, now wait for resp
Sep 22 23:15:09.330 INFO [0] received reconcile message
Sep 22 23:15:09.330 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(292), op: ExtentFlush { repair_id: ReconciliationId(292), extent_id: 96, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:09.331 INFO [0] client ExtentFlush { repair_id: ReconciliationId(292), extent_id: 96, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:09.331 INFO [1] received reconcile message
Sep 22 23:15:09.331 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(292), op: ExtentFlush { repair_id: ReconciliationId(292), extent_id: 96, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:09.331 INFO [1] client ExtentFlush { repair_id: ReconciliationId(292), extent_id: 96, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:09.331 INFO [2] received reconcile message
Sep 22 23:15:09.331 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(292), op: ExtentFlush { repair_id: ReconciliationId(292), extent_id: 96, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:09.331 INFO [2] client ExtentFlush { repair_id: ReconciliationId(292), extent_id: 96, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:09.331 DEBG 292 Flush extent 96 with f:2 g:2
Sep 22 23:15:09.331 DEBG Flush just extent 96 with f:2 and g:2
Sep 22 23:15:09.331 DEBG [1] It's time to notify for 292
Sep 22 23:15:09.331 INFO Completion from [1] id:292 status:true
Sep 22 23:15:09.331 INFO [293/752] Repair commands completed
Sep 22 23:15:09.331 INFO Pop front: ReconcileIO { id: ReconciliationId(293), op: ExtentClose { repair_id: ReconciliationId(293), extent_id: 96 }, state: ClientData([New, New, New]) }
Sep 22 23:15:09.331 INFO Sent repair work, now wait for resp
Sep 22 23:15:09.331 INFO [0] received reconcile message
Sep 22 23:15:09.331 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(293), op: ExtentClose { repair_id: ReconciliationId(293), extent_id: 96 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:09.331 INFO [0] client ExtentClose { repair_id: ReconciliationId(293), extent_id: 96 }
Sep 22 23:15:09.331 INFO [1] received reconcile message
Sep 22 23:15:09.331 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(293), op: ExtentClose { repair_id: ReconciliationId(293), extent_id: 96 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:09.331 INFO [1] client ExtentClose { repair_id: ReconciliationId(293), extent_id: 96 }
Sep 22 23:15:09.331 INFO [2] received reconcile message
Sep 22 23:15:09.331 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(293), op: ExtentClose { repair_id: ReconciliationId(293), extent_id: 96 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:09.331 INFO [2] client ExtentClose { repair_id: ReconciliationId(293), extent_id: 96 }
Sep 22 23:15:09.331 DEBG 293 Close extent 96
Sep 22 23:15:09.332 DEBG 293 Close extent 96
Sep 22 23:15:09.332 DEBG 293 Close extent 96
Sep 22 23:15:09.332 DEBG [2] It's time to notify for 293
Sep 22 23:15:09.332 INFO Completion from [2] id:293 status:true
Sep 22 23:15:09.332 INFO [294/752] Repair commands completed
Sep 22 23:15:09.332 INFO Pop front: ReconcileIO { id: ReconciliationId(294), op: ExtentRepair { repair_id: ReconciliationId(294), extent_id: 96, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:09.332 INFO Sent repair work, now wait for resp
Sep 22 23:15:09.332 INFO [0] received reconcile message
Sep 22 23:15:09.332 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(294), op: ExtentRepair { repair_id: ReconciliationId(294), extent_id: 96, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:09.332 INFO [0] client ExtentRepair { repair_id: ReconciliationId(294), extent_id: 96, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:09.332 INFO [0] Sending repair request ReconciliationId(294)
Sep 22 23:15:09.333 INFO [1] received reconcile message
Sep 22 23:15:09.333 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(294), op: ExtentRepair { repair_id: ReconciliationId(294), extent_id: 96, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:09.333 INFO [1] client ExtentRepair { repair_id: ReconciliationId(294), extent_id: 96, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:09.333 INFO [1] No action required ReconciliationId(294)
Sep 22 23:15:09.333 INFO [2] received reconcile message
Sep 22 23:15:09.333 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(294), op: ExtentRepair { repair_id: ReconciliationId(294), extent_id: 96, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:09.333 INFO [2] client ExtentRepair { repair_id: ReconciliationId(294), extent_id: 96, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:09.333 INFO [2] No action required ReconciliationId(294)
Sep 22 23:15:09.333 DEBG 294 Repair extent 96 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:09.333 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/060.copy"
Sep 22 23:15:09.397 INFO accepted connection, remote_addr: 127.0.0.1:51861, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.397 TRCE incoming request, uri: /extent/96/files, method: GET, req_id: 3df62914-d664-4563-aebc-d0143a732ff2, remote_addr: 127.0.0.1:51861, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.397 INFO request completed, latency_us: 224, response_code: 200, uri: /extent/96/files, method: GET, req_id: 3df62914-d664-4563-aebc-d0143a732ff2, remote_addr: 127.0.0.1:51861, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.398 INFO eid:96 Found repair files: ["060", "060.db"]
Sep 22 23:15:09.398 TRCE incoming request, uri: /newextent/96/data, method: GET, req_id: 813b681e-29bf-42eb-a2f2-cc8f654f415f, remote_addr: 127.0.0.1:51861, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.398 INFO request completed, latency_us: 335, response_code: 200, uri: /newextent/96/data, method: GET, req_id: 813b681e-29bf-42eb-a2f2-cc8f654f415f, remote_addr: 127.0.0.1:51861, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.403 TRCE incoming request, uri: /newextent/96/db, method: GET, req_id: 5d13749e-d6c4-4a53-a5e7-46209e88de2c, remote_addr: 127.0.0.1:51861, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.404 INFO request completed, latency_us: 292, response_code: 200, uri: /newextent/96/db, method: GET, req_id: 5d13749e-d6c4-4a53-a5e7-46209e88de2c, remote_addr: 127.0.0.1:51861, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.405 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/060.copy" to "/tmp/downstairs-vrx8aK6L/00/000/060.replace"
Sep 22 23:15:09.405 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:09.406 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/060.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:09.406 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/060"
Sep 22 23:15:09.406 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/060.db"
Sep 22 23:15:09.406 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:09.406 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/060.replace" to "/tmp/downstairs-vrx8aK6L/00/000/060.completed"
Sep 22 23:15:09.406 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:09.406 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:09.406 DEBG [0] It's time to notify for 294
Sep 22 23:15:09.407 INFO Completion from [0] id:294 status:true
Sep 22 23:15:09.407 INFO [295/752] Repair commands completed
Sep 22 23:15:09.407 INFO Pop front: ReconcileIO { id: ReconciliationId(295), op: ExtentReopen { repair_id: ReconciliationId(295), extent_id: 96 }, state: ClientData([New, New, New]) }
Sep 22 23:15:09.407 INFO Sent repair work, now wait for resp
Sep 22 23:15:09.407 INFO [0] received reconcile message
Sep 22 23:15:09.407 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(295), op: ExtentReopen { repair_id: ReconciliationId(295), extent_id: 96 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:09.407 INFO [0] client ExtentReopen { repair_id: ReconciliationId(295), extent_id: 96 }
Sep 22 23:15:09.407 INFO [1] received reconcile message
Sep 22 23:15:09.407 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(295), op: ExtentReopen { repair_id: ReconciliationId(295), extent_id: 96 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:09.407 INFO [1] client ExtentReopen { repair_id: ReconciliationId(295), extent_id: 96 }
Sep 22 23:15:09.407 INFO [2] received reconcile message
Sep 22 23:15:09.407 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(295), op: ExtentReopen { repair_id: ReconciliationId(295), extent_id: 96 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:09.407 INFO [2] client ExtentReopen { repair_id: ReconciliationId(295), extent_id: 96 }
Sep 22 23:15:09.407 DEBG 295 Reopen extent 96
Sep 22 23:15:09.408 DEBG 295 Reopen extent 96
Sep 22 23:15:09.408 DEBG 295 Reopen extent 96
Sep 22 23:15:09.409 DEBG [2] It's time to notify for 295
Sep 22 23:15:09.409 INFO Completion from [2] id:295 status:true
Sep 22 23:15:09.409 INFO [296/752] Repair commands completed
Sep 22 23:15:09.409 INFO Pop front: ReconcileIO { id: ReconciliationId(296), op: ExtentFlush { repair_id: ReconciliationId(296), extent_id: 28, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:09.409 INFO Sent repair work, now wait for resp
Sep 22 23:15:09.409 INFO [0] received reconcile message
Sep 22 23:15:09.409 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(296), op: ExtentFlush { repair_id: ReconciliationId(296), extent_id: 28, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:09.409 INFO [0] client ExtentFlush { repair_id: ReconciliationId(296), extent_id: 28, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:09.409 INFO [1] received reconcile message
Sep 22 23:15:09.409 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(296), op: ExtentFlush { repair_id: ReconciliationId(296), extent_id: 28, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:09.409 INFO [1] client ExtentFlush { repair_id: ReconciliationId(296), extent_id: 28, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:09.409 INFO [2] received reconcile message
Sep 22 23:15:09.409 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(296), op: ExtentFlush { repair_id: ReconciliationId(296), extent_id: 28, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:09.409 INFO [2] client ExtentFlush { repair_id: ReconciliationId(296), extent_id: 28, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:09.409 DEBG 296 Flush extent 28 with f:2 g:2
Sep 22 23:15:09.409 DEBG Flush just extent 28 with f:2 and g:2
Sep 22 23:15:09.409 DEBG [1] It's time to notify for 296
Sep 22 23:15:09.409 INFO Completion from [1] id:296 status:true
Sep 22 23:15:09.409 INFO [297/752] Repair commands completed
Sep 22 23:15:09.409 INFO Pop front: ReconcileIO { id: ReconciliationId(297), op: ExtentClose { repair_id: ReconciliationId(297), extent_id: 28 }, state: ClientData([New, New, New]) }
Sep 22 23:15:09.409 INFO Sent repair work, now wait for resp
Sep 22 23:15:09.409 INFO [0] received reconcile message
Sep 22 23:15:09.409 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(297), op: ExtentClose { repair_id: ReconciliationId(297), extent_id: 28 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:09.409 INFO [0] client ExtentClose { repair_id: ReconciliationId(297), extent_id: 28 }
Sep 22 23:15:09.410 INFO [1] received reconcile message
Sep 22 23:15:09.410 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(297), op: ExtentClose { repair_id: ReconciliationId(297), extent_id: 28 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:09.410 INFO [1] client ExtentClose { repair_id: ReconciliationId(297), extent_id: 28 }
Sep 22 23:15:09.410 INFO [2] received reconcile message
Sep 22 23:15:09.410 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(297), op: ExtentClose { repair_id: ReconciliationId(297), extent_id: 28 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:09.410 INFO [2] client ExtentClose { repair_id: ReconciliationId(297), extent_id: 28 }
Sep 22 23:15:09.410 DEBG 297 Close extent 28
Sep 22 23:15:09.410 DEBG 297 Close extent 28
Sep 22 23:15:09.410 DEBG 297 Close extent 28
Sep 22 23:15:09.411 DEBG [2] It's time to notify for 297
Sep 22 23:15:09.411 INFO Completion from [2] id:297 status:true
Sep 22 23:15:09.411 INFO [298/752] Repair commands completed
Sep 22 23:15:09.411 INFO Pop front: ReconcileIO { id: ReconciliationId(298), op: ExtentRepair { repair_id: ReconciliationId(298), extent_id: 28, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:09.411 INFO Sent repair work, now wait for resp
Sep 22 23:15:09.411 INFO [0] received reconcile message
Sep 22 23:15:09.411 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(298), op: ExtentRepair { repair_id: ReconciliationId(298), extent_id: 28, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:09.411 INFO [0] client ExtentRepair { repair_id: ReconciliationId(298), extent_id: 28, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:09.411 INFO [0] Sending repair request ReconciliationId(298)
Sep 22 23:15:09.411 INFO [1] received reconcile message
Sep 22 23:15:09.411 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(298), op: ExtentRepair { repair_id: ReconciliationId(298), extent_id: 28, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:09.411 INFO [1] client ExtentRepair { repair_id: ReconciliationId(298), extent_id: 28, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:09.411 INFO [1] No action required ReconciliationId(298)
Sep 22 23:15:09.411 INFO [2] received reconcile message
Sep 22 23:15:09.411 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(298), op: ExtentRepair { repair_id: ReconciliationId(298), extent_id: 28, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:09.411 INFO [2] client ExtentRepair { repair_id: ReconciliationId(298), extent_id: 28, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:09.411 INFO [2] No action required ReconciliationId(298)
Sep 22 23:15:09.411 DEBG 298 Repair extent 28 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:09.411 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/01C.copy"
Sep 22 23:15:09.444 DEBG [2] Read AckReady 1080, : downstairs
Sep 22 23:15:09.445 DEBG up_ds_listen was notified
Sep 22 23:15:09.445 DEBG up_ds_listen process 1080
Sep 22 23:15:09.446 DEBG [A] ack job 1080:81, : downstairs
Sep 22 23:15:09.472 INFO accepted connection, remote_addr: 127.0.0.1:42693, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.472 TRCE incoming request, uri: /extent/28/files, method: GET, req_id: f0428ef2-b9de-4ea5-b101-407f4782d0bf, remote_addr: 127.0.0.1:42693, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.472 INFO request completed, latency_us: 203, response_code: 200, uri: /extent/28/files, method: GET, req_id: f0428ef2-b9de-4ea5-b101-407f4782d0bf, remote_addr: 127.0.0.1:42693, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.473 INFO eid:28 Found repair files: ["01C", "01C.db"]
Sep 22 23:15:09.473 TRCE incoming request, uri: /newextent/28/data, method: GET, req_id: 25382c69-63cb-4f78-a41a-0770f9aec17b, remote_addr: 127.0.0.1:42693, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.473 INFO request completed, latency_us: 350, response_code: 200, uri: /newextent/28/data, method: GET, req_id: 25382c69-63cb-4f78-a41a-0770f9aec17b, remote_addr: 127.0.0.1:42693, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.479 TRCE incoming request, uri: /newextent/28/db, method: GET, req_id: 99d5edb5-a9ba-4c7b-b3c1-5cddc9077c00, remote_addr: 127.0.0.1:42693, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.479 INFO request completed, latency_us: 294, response_code: 200, uri: /newextent/28/db, method: GET, req_id: 99d5edb5-a9ba-4c7b-b3c1-5cddc9077c00, remote_addr: 127.0.0.1:42693, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.480 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/01C.copy" to "/tmp/downstairs-vrx8aK6L/00/000/01C.replace"
Sep 22 23:15:09.480 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:09.481 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/01C.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:09.481 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/01C"
Sep 22 23:15:09.481 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/01C.db"
Sep 22 23:15:09.481 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:09.481 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/01C.replace" to "/tmp/downstairs-vrx8aK6L/00/000/01C.completed"
Sep 22 23:15:09.481 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:09.481 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:09.481 DEBG [0] It's time to notify for 298
Sep 22 23:15:09.482 INFO Completion from [0] id:298 status:true
Sep 22 23:15:09.482 INFO [299/752] Repair commands completed
Sep 22 23:15:09.482 INFO Pop front: ReconcileIO { id: ReconciliationId(299), op: ExtentReopen { repair_id: ReconciliationId(299), extent_id: 28 }, state: ClientData([New, New, New]) }
Sep 22 23:15:09.482 INFO Sent repair work, now wait for resp
Sep 22 23:15:09.482 INFO [0] received reconcile message
Sep 22 23:15:09.482 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(299), op: ExtentReopen { repair_id: ReconciliationId(299), extent_id: 28 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:09.482 INFO [0] client ExtentReopen { repair_id: ReconciliationId(299), extent_id: 28 }
Sep 22 23:15:09.482 INFO [1] received reconcile message
Sep 22 23:15:09.482 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(299), op: ExtentReopen { repair_id: ReconciliationId(299), extent_id: 28 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:09.482 INFO [1] client ExtentReopen { repair_id: ReconciliationId(299), extent_id: 28 }
Sep 22 23:15:09.482 INFO [2] received reconcile message
Sep 22 23:15:09.482 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(299), op: ExtentReopen { repair_id: ReconciliationId(299), extent_id: 28 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:09.482 INFO [2] client ExtentReopen { repair_id: ReconciliationId(299), extent_id: 28 }
Sep 22 23:15:09.482 DEBG 299 Reopen extent 28
Sep 22 23:15:09.483 DEBG 299 Reopen extent 28
Sep 22 23:15:09.483 DEBG 299 Reopen extent 28
Sep 22 23:15:09.484 DEBG [2] It's time to notify for 299
Sep 22 23:15:09.484 INFO Completion from [2] id:299 status:true
Sep 22 23:15:09.484 INFO [300/752] Repair commands completed
Sep 22 23:15:09.484 INFO Pop front: ReconcileIO { id: ReconciliationId(300), op: ExtentFlush { repair_id: ReconciliationId(300), extent_id: 178, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:09.484 INFO Sent repair work, now wait for resp
Sep 22 23:15:09.484 INFO [0] received reconcile message
Sep 22 23:15:09.484 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(300), op: ExtentFlush { repair_id: ReconciliationId(300), extent_id: 178, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:09.484 INFO [0] client ExtentFlush { repair_id: ReconciliationId(300), extent_id: 178, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:09.484 INFO [1] received reconcile message
Sep 22 23:15:09.484 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(300), op: ExtentFlush { repair_id: ReconciliationId(300), extent_id: 178, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:09.484 INFO [1] client ExtentFlush { repair_id: ReconciliationId(300), extent_id: 178, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:09.484 INFO [2] received reconcile message
Sep 22 23:15:09.484 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(300), op: ExtentFlush { repair_id: ReconciliationId(300), extent_id: 178, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:09.484 INFO [2] client ExtentFlush { repair_id: ReconciliationId(300), extent_id: 178, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:09.484 DEBG 300 Flush extent 178 with f:2 g:2
Sep 22 23:15:09.484 DEBG Flush just extent 178 with f:2 and g:2
Sep 22 23:15:09.484 DEBG [1] It's time to notify for 300
Sep 22 23:15:09.484 INFO Completion from [1] id:300 status:true
Sep 22 23:15:09.484 INFO [301/752] Repair commands completed
Sep 22 23:15:09.484 INFO Pop front: ReconcileIO { id: ReconciliationId(301), op: ExtentClose { repair_id: ReconciliationId(301), extent_id: 178 }, state: ClientData([New, New, New]) }
Sep 22 23:15:09.484 INFO Sent repair work, now wait for resp
Sep 22 23:15:09.484 INFO [0] received reconcile message
Sep 22 23:15:09.484 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(301), op: ExtentClose { repair_id: ReconciliationId(301), extent_id: 178 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:09.484 INFO [0] client ExtentClose { repair_id: ReconciliationId(301), extent_id: 178 }
Sep 22 23:15:09.485 INFO [1] received reconcile message
Sep 22 23:15:09.485 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(301), op: ExtentClose { repair_id: ReconciliationId(301), extent_id: 178 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:09.485 INFO [1] client ExtentClose { repair_id: ReconciliationId(301), extent_id: 178 }
Sep 22 23:15:09.485 INFO [2] received reconcile message
Sep 22 23:15:09.485 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(301), op: ExtentClose { repair_id: ReconciliationId(301), extent_id: 178 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:09.485 INFO [2] client ExtentClose { repair_id: ReconciliationId(301), extent_id: 178 }
Sep 22 23:15:09.485 DEBG 301 Close extent 178
Sep 22 23:15:09.485 DEBG 301 Close extent 178
Sep 22 23:15:09.485 DEBG 301 Close extent 178
Sep 22 23:15:09.486 DEBG [2] It's time to notify for 301
Sep 22 23:15:09.486 INFO Completion from [2] id:301 status:true
Sep 22 23:15:09.486 INFO [302/752] Repair commands completed
Sep 22 23:15:09.486 INFO Pop front: ReconcileIO { id: ReconciliationId(302), op: ExtentRepair { repair_id: ReconciliationId(302), extent_id: 178, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:09.486 INFO Sent repair work, now wait for resp
Sep 22 23:15:09.486 INFO [0] received reconcile message
Sep 22 23:15:09.486 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(302), op: ExtentRepair { repair_id: ReconciliationId(302), extent_id: 178, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:09.486 INFO [0] client ExtentRepair { repair_id: ReconciliationId(302), extent_id: 178, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:09.486 INFO [0] Sending repair request ReconciliationId(302)
Sep 22 23:15:09.486 INFO [1] received reconcile message
Sep 22 23:15:09.486 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(302), op: ExtentRepair { repair_id: ReconciliationId(302), extent_id: 178, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:09.486 INFO [1] client ExtentRepair { repair_id: ReconciliationId(302), extent_id: 178, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:09.486 INFO [1] No action required ReconciliationId(302)
Sep 22 23:15:09.486 INFO [2] received reconcile message
Sep 22 23:15:09.486 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(302), op: ExtentRepair { repair_id: ReconciliationId(302), extent_id: 178, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:09.486 INFO [2] client ExtentRepair { repair_id: ReconciliationId(302), extent_id: 178, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:09.486 INFO [2] No action required ReconciliationId(302)
Sep 22 23:15:09.486 DEBG 302 Repair extent 178 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:09.486 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/0B2.copy"
Sep 22 23:15:09.499 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:09.501 INFO [lossy] skipping 1081
Sep 22 23:15:09.501 DEBG Flush     :1081 extent_limit None deps:[JobId(1080)] res:true f:30 g:1
Sep 22 23:15:09.501 INFO [lossy] sleeping 1 second
Sep 22 23:15:09.503 INFO [lossy] sleeping 1 second
Sep 22 23:15:09.551 INFO accepted connection, remote_addr: 127.0.0.1:35760, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.551 TRCE incoming request, uri: /extent/178/files, method: GET, req_id: 3a56285e-9698-4cd7-8e9a-3b380d3b2b2d, remote_addr: 127.0.0.1:35760, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.552 INFO request completed, latency_us: 266, response_code: 200, uri: /extent/178/files, method: GET, req_id: 3a56285e-9698-4cd7-8e9a-3b380d3b2b2d, remote_addr: 127.0.0.1:35760, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.552 INFO eid:178 Found repair files: ["0B2", "0B2.db"]
Sep 22 23:15:09.552 TRCE incoming request, uri: /newextent/178/data, method: GET, req_id: ed27d62b-a2db-4ae2-bb0a-5ebc27f272ea, remote_addr: 127.0.0.1:35760, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.553 INFO request completed, latency_us: 343, response_code: 200, uri: /newextent/178/data, method: GET, req_id: ed27d62b-a2db-4ae2-bb0a-5ebc27f272ea, remote_addr: 127.0.0.1:35760, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.558 TRCE incoming request, uri: /newextent/178/db, method: GET, req_id: 141be047-48c7-40b0-9e94-3edb96ee702a, remote_addr: 127.0.0.1:35760, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.558 INFO request completed, latency_us: 294, response_code: 200, uri: /newextent/178/db, method: GET, req_id: 141be047-48c7-40b0-9e94-3edb96ee702a, remote_addr: 127.0.0.1:35760, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.559 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/0B2.copy" to "/tmp/downstairs-vrx8aK6L/00/000/0B2.replace"
Sep 22 23:15:09.559 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:09.560 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/0B2.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:09.561 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/0B2"
Sep 22 23:15:09.561 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/0B2.db"
Sep 22 23:15:09.561 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:09.561 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/0B2.replace" to "/tmp/downstairs-vrx8aK6L/00/000/0B2.completed"
Sep 22 23:15:09.561 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:09.561 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:09.561 DEBG [0] It's time to notify for 302
Sep 22 23:15:09.561 INFO Completion from [0] id:302 status:true
Sep 22 23:15:09.561 INFO [303/752] Repair commands completed
Sep 22 23:15:09.561 INFO Pop front: ReconcileIO { id: ReconciliationId(303), op: ExtentReopen { repair_id: ReconciliationId(303), extent_id: 178 }, state: ClientData([New, New, New]) }
Sep 22 23:15:09.561 INFO Sent repair work, now wait for resp
Sep 22 23:15:09.561 INFO [0] received reconcile message
Sep 22 23:15:09.561 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(303), op: ExtentReopen { repair_id: ReconciliationId(303), extent_id: 178 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:09.561 INFO [0] client ExtentReopen { repair_id: ReconciliationId(303), extent_id: 178 }
Sep 22 23:15:09.561 INFO [1] received reconcile message
Sep 22 23:15:09.561 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(303), op: ExtentReopen { repair_id: ReconciliationId(303), extent_id: 178 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:09.562 INFO [1] client ExtentReopen { repair_id: ReconciliationId(303), extent_id: 178 }
Sep 22 23:15:09.562 INFO [2] received reconcile message
Sep 22 23:15:09.562 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(303), op: ExtentReopen { repair_id: ReconciliationId(303), extent_id: 178 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:09.562 INFO [2] client ExtentReopen { repair_id: ReconciliationId(303), extent_id: 178 }
Sep 22 23:15:09.562 DEBG 303 Reopen extent 178
Sep 22 23:15:09.563 DEBG 303 Reopen extent 178
Sep 22 23:15:09.563 DEBG 303 Reopen extent 178
Sep 22 23:15:09.564 DEBG [2] It's time to notify for 303
Sep 22 23:15:09.564 INFO Completion from [2] id:303 status:true
Sep 22 23:15:09.564 INFO [304/752] Repair commands completed
Sep 22 23:15:09.564 INFO Pop front: ReconcileIO { id: ReconciliationId(304), op: ExtentFlush { repair_id: ReconciliationId(304), extent_id: 36, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:09.564 INFO Sent repair work, now wait for resp
Sep 22 23:15:09.564 INFO [0] received reconcile message
Sep 22 23:15:09.564 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(304), op: ExtentFlush { repair_id: ReconciliationId(304), extent_id: 36, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:09.564 INFO [0] client ExtentFlush { repair_id: ReconciliationId(304), extent_id: 36, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:09.564 INFO [1] received reconcile message
Sep 22 23:15:09.564 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(304), op: ExtentFlush { repair_id: ReconciliationId(304), extent_id: 36, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:09.564 INFO [1] client ExtentFlush { repair_id: ReconciliationId(304), extent_id: 36, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:09.564 INFO [2] received reconcile message
Sep 22 23:15:09.564 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(304), op: ExtentFlush { repair_id: ReconciliationId(304), extent_id: 36, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:09.564 INFO [2] client ExtentFlush { repair_id: ReconciliationId(304), extent_id: 36, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:09.564 DEBG 304 Flush extent 36 with f:2 g:2
Sep 22 23:15:09.564 DEBG Flush just extent 36 with f:2 and g:2
Sep 22 23:15:09.564 DEBG [1] It's time to notify for 304
Sep 22 23:15:09.564 INFO Completion from [1] id:304 status:true
Sep 22 23:15:09.564 INFO [305/752] Repair commands completed
Sep 22 23:15:09.564 INFO Pop front: ReconcileIO { id: ReconciliationId(305), op: ExtentClose { repair_id: ReconciliationId(305), extent_id: 36 }, state: ClientData([New, New, New]) }
Sep 22 23:15:09.564 INFO Sent repair work, now wait for resp
Sep 22 23:15:09.564 INFO [0] received reconcile message
Sep 22 23:15:09.564 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(305), op: ExtentClose { repair_id: ReconciliationId(305), extent_id: 36 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:09.564 INFO [0] client ExtentClose { repair_id: ReconciliationId(305), extent_id: 36 }
Sep 22 23:15:09.564 INFO [1] received reconcile message
Sep 22 23:15:09.564 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(305), op: ExtentClose { repair_id: ReconciliationId(305), extent_id: 36 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:09.564 INFO [1] client ExtentClose { repair_id: ReconciliationId(305), extent_id: 36 }
Sep 22 23:15:09.564 INFO [2] received reconcile message
Sep 22 23:15:09.565 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(305), op: ExtentClose { repair_id: ReconciliationId(305), extent_id: 36 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:09.565 INFO [2] client ExtentClose { repair_id: ReconciliationId(305), extent_id: 36 }
Sep 22 23:15:09.565 DEBG 305 Close extent 36
Sep 22 23:15:09.565 DEBG 305 Close extent 36
Sep 22 23:15:09.565 DEBG 305 Close extent 36
Sep 22 23:15:09.566 DEBG [2] It's time to notify for 305
Sep 22 23:15:09.566 INFO Completion from [2] id:305 status:true
Sep 22 23:15:09.566 INFO [306/752] Repair commands completed
Sep 22 23:15:09.566 INFO Pop front: ReconcileIO { id: ReconciliationId(306), op: ExtentRepair { repair_id: ReconciliationId(306), extent_id: 36, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:09.566 INFO Sent repair work, now wait for resp
Sep 22 23:15:09.566 INFO [0] received reconcile message
Sep 22 23:15:09.566 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(306), op: ExtentRepair { repair_id: ReconciliationId(306), extent_id: 36, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:09.566 INFO [0] client ExtentRepair { repair_id: ReconciliationId(306), extent_id: 36, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:09.566 INFO [0] Sending repair request ReconciliationId(306)
Sep 22 23:15:09.566 INFO [1] received reconcile message
Sep 22 23:15:09.566 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(306), op: ExtentRepair { repair_id: ReconciliationId(306), extent_id: 36, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:09.566 INFO [1] client ExtentRepair { repair_id: ReconciliationId(306), extent_id: 36, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:09.566 INFO [1] No action required ReconciliationId(306)
Sep 22 23:15:09.566 INFO [2] received reconcile message
Sep 22 23:15:09.566 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(306), op: ExtentRepair { repair_id: ReconciliationId(306), extent_id: 36, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:09.566 INFO [2] client ExtentRepair { repair_id: ReconciliationId(306), extent_id: 36, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:09.566 INFO [2] No action required ReconciliationId(306)
Sep 22 23:15:09.566 DEBG 306 Repair extent 36 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:09.566 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/024.copy"
Sep 22 23:15:09.600 DEBG IO Read  1082 has deps [JobId(1081)]
Sep 22 23:15:09.629 INFO accepted connection, remote_addr: 127.0.0.1:64703, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.629 TRCE incoming request, uri: /extent/36/files, method: GET, req_id: 0836efef-0c30-49a6-8f85-dca2ff77050a, remote_addr: 127.0.0.1:64703, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.629 INFO request completed, latency_us: 256, response_code: 200, uri: /extent/36/files, method: GET, req_id: 0836efef-0c30-49a6-8f85-dca2ff77050a, remote_addr: 127.0.0.1:64703, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.629 INFO eid:36 Found repair files: ["024", "024.db"]
Sep 22 23:15:09.630 TRCE incoming request, uri: /newextent/36/data, method: GET, req_id: fd1a5aea-80c3-49e6-9429-bfc5a777ffa4, remote_addr: 127.0.0.1:64703, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.630 INFO request completed, latency_us: 363, response_code: 200, uri: /newextent/36/data, method: GET, req_id: fd1a5aea-80c3-49e6-9429-bfc5a777ffa4, remote_addr: 127.0.0.1:64703, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.635 TRCE incoming request, uri: /newextent/36/db, method: GET, req_id: 95e13cbb-984c-4297-9708-690b4bcdf696, remote_addr: 127.0.0.1:64703, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.636 INFO request completed, latency_us: 299, response_code: 200, uri: /newextent/36/db, method: GET, req_id: 95e13cbb-984c-4297-9708-690b4bcdf696, remote_addr: 127.0.0.1:64703, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.637 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/024.copy" to "/tmp/downstairs-vrx8aK6L/00/000/024.replace"
Sep 22 23:15:09.637 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:09.638 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/024.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:09.638 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/024"
Sep 22 23:15:09.638 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/024.db"
Sep 22 23:15:09.638 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:09.638 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/024.replace" to "/tmp/downstairs-vrx8aK6L/00/000/024.completed"
Sep 22 23:15:09.638 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:09.638 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:09.638 DEBG [0] It's time to notify for 306
Sep 22 23:15:09.639 INFO Completion from [0] id:306 status:true
Sep 22 23:15:09.639 INFO [307/752] Repair commands completed
Sep 22 23:15:09.639 INFO Pop front: ReconcileIO { id: ReconciliationId(307), op: ExtentReopen { repair_id: ReconciliationId(307), extent_id: 36 }, state: ClientData([New, New, New]) }
Sep 22 23:15:09.639 INFO Sent repair work, now wait for resp
Sep 22 23:15:09.639 INFO [0] received reconcile message
Sep 22 23:15:09.639 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(307), op: ExtentReopen { repair_id: ReconciliationId(307), extent_id: 36 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:09.639 INFO [0] client ExtentReopen { repair_id: ReconciliationId(307), extent_id: 36 }
Sep 22 23:15:09.639 INFO [1] received reconcile message
Sep 22 23:15:09.639 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(307), op: ExtentReopen { repair_id: ReconciliationId(307), extent_id: 36 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:09.639 INFO [1] client ExtentReopen { repair_id: ReconciliationId(307), extent_id: 36 }
Sep 22 23:15:09.639 INFO [2] received reconcile message
Sep 22 23:15:09.639 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(307), op: ExtentReopen { repair_id: ReconciliationId(307), extent_id: 36 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:09.639 INFO [2] client ExtentReopen { repair_id: ReconciliationId(307), extent_id: 36 }
Sep 22 23:15:09.639 DEBG 307 Reopen extent 36
Sep 22 23:15:09.640 DEBG 307 Reopen extent 36
Sep 22 23:15:09.640 DEBG 307 Reopen extent 36
Sep 22 23:15:09.641 DEBG [2] It's time to notify for 307
Sep 22 23:15:09.641 INFO Completion from [2] id:307 status:true
Sep 22 23:15:09.641 INFO [308/752] Repair commands completed
Sep 22 23:15:09.641 INFO Pop front: ReconcileIO { id: ReconciliationId(308), op: ExtentFlush { repair_id: ReconciliationId(308), extent_id: 92, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:09.641 INFO Sent repair work, now wait for resp
Sep 22 23:15:09.641 INFO [0] received reconcile message
Sep 22 23:15:09.641 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(308), op: ExtentFlush { repair_id: ReconciliationId(308), extent_id: 92, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:09.641 INFO [0] client ExtentFlush { repair_id: ReconciliationId(308), extent_id: 92, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:09.641 INFO [1] received reconcile message
Sep 22 23:15:09.641 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(308), op: ExtentFlush { repair_id: ReconciliationId(308), extent_id: 92, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:09.641 INFO [1] client ExtentFlush { repair_id: ReconciliationId(308), extent_id: 92, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:09.641 INFO [2] received reconcile message
Sep 22 23:15:09.641 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(308), op: ExtentFlush { repair_id: ReconciliationId(308), extent_id: 92, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:09.641 INFO [2] client ExtentFlush { repair_id: ReconciliationId(308), extent_id: 92, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:09.641 DEBG 308 Flush extent 92 with f:2 g:2
Sep 22 23:15:09.641 DEBG Flush just extent 92 with f:2 and g:2
Sep 22 23:15:09.641 DEBG [1] It's time to notify for 308
Sep 22 23:15:09.642 INFO Completion from [1] id:308 status:true
Sep 22 23:15:09.642 INFO [309/752] Repair commands completed
Sep 22 23:15:09.642 INFO Pop front: ReconcileIO { id: ReconciliationId(309), op: ExtentClose { repair_id: ReconciliationId(309), extent_id: 92 }, state: ClientData([New, New, New]) }
Sep 22 23:15:09.642 INFO Sent repair work, now wait for resp
Sep 22 23:15:09.642 INFO [0] received reconcile message
Sep 22 23:15:09.642 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(309), op: ExtentClose { repair_id: ReconciliationId(309), extent_id: 92 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:09.642 INFO [0] client ExtentClose { repair_id: ReconciliationId(309), extent_id: 92 }
Sep 22 23:15:09.642 INFO [1] received reconcile message
Sep 22 23:15:09.642 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(309), op: ExtentClose { repair_id: ReconciliationId(309), extent_id: 92 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:09.642 INFO [1] client ExtentClose { repair_id: ReconciliationId(309), extent_id: 92 }
Sep 22 23:15:09.642 INFO [2] received reconcile message
Sep 22 23:15:09.642 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(309), op: ExtentClose { repair_id: ReconciliationId(309), extent_id: 92 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:09.642 INFO [2] client ExtentClose { repair_id: ReconciliationId(309), extent_id: 92 }
Sep 22 23:15:09.642 DEBG 309 Close extent 92
Sep 22 23:15:09.642 DEBG 309 Close extent 92
Sep 22 23:15:09.643 DEBG 309 Close extent 92
Sep 22 23:15:09.643 DEBG [2] It's time to notify for 309
Sep 22 23:15:09.643 INFO Completion from [2] id:309 status:true
Sep 22 23:15:09.643 INFO [310/752] Repair commands completed
Sep 22 23:15:09.643 INFO Pop front: ReconcileIO { id: ReconciliationId(310), op: ExtentRepair { repair_id: ReconciliationId(310), extent_id: 92, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:09.643 INFO Sent repair work, now wait for resp
Sep 22 23:15:09.643 INFO [0] received reconcile message
Sep 22 23:15:09.643 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(310), op: ExtentRepair { repair_id: ReconciliationId(310), extent_id: 92, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:09.643 INFO [0] client ExtentRepair { repair_id: ReconciliationId(310), extent_id: 92, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:09.643 INFO [0] Sending repair request ReconciliationId(310)
Sep 22 23:15:09.643 INFO [1] received reconcile message
Sep 22 23:15:09.643 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(310), op: ExtentRepair { repair_id: ReconciliationId(310), extent_id: 92, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:09.643 INFO [1] client ExtentRepair { repair_id: ReconciliationId(310), extent_id: 92, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:09.643 INFO [1] No action required ReconciliationId(310)
Sep 22 23:15:09.643 INFO [2] received reconcile message
Sep 22 23:15:09.643 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(310), op: ExtentRepair { repair_id: ReconciliationId(310), extent_id: 92, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:09.643 INFO [2] client ExtentRepair { repair_id: ReconciliationId(310), extent_id: 92, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:09.643 INFO [2] No action required ReconciliationId(310)
Sep 22 23:15:09.643 DEBG 310 Repair extent 92 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:09.643 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/05C.copy"
Sep 22 23:15:09.707 INFO accepted connection, remote_addr: 127.0.0.1:44147, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.707 TRCE incoming request, uri: /extent/92/files, method: GET, req_id: 86f01bff-af3d-43d7-a1ae-6c24fdeb8ab3, remote_addr: 127.0.0.1:44147, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.707 INFO request completed, latency_us: 222, response_code: 200, uri: /extent/92/files, method: GET, req_id: 86f01bff-af3d-43d7-a1ae-6c24fdeb8ab3, remote_addr: 127.0.0.1:44147, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.708 INFO eid:92 Found repair files: ["05C", "05C.db"]
Sep 22 23:15:09.708 TRCE incoming request, uri: /newextent/92/data, method: GET, req_id: d8c93b2e-41df-4c3a-b695-a37205ae6a77, remote_addr: 127.0.0.1:44147, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.708 INFO request completed, latency_us: 337, response_code: 200, uri: /newextent/92/data, method: GET, req_id: d8c93b2e-41df-4c3a-b695-a37205ae6a77, remote_addr: 127.0.0.1:44147, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.713 TRCE incoming request, uri: /newextent/92/db, method: GET, req_id: 9cebed4a-123d-4e3b-85f0-9f92a55b28aa, remote_addr: 127.0.0.1:44147, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.714 INFO request completed, latency_us: 294, response_code: 200, uri: /newextent/92/db, method: GET, req_id: 9cebed4a-123d-4e3b-85f0-9f92a55b28aa, remote_addr: 127.0.0.1:44147, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.715 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/05C.copy" to "/tmp/downstairs-vrx8aK6L/00/000/05C.replace"
Sep 22 23:15:09.715 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:09.716 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/05C.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:09.716 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/05C"
Sep 22 23:15:09.716 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/05C.db"
Sep 22 23:15:09.716 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:09.716 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/05C.replace" to "/tmp/downstairs-vrx8aK6L/00/000/05C.completed"
Sep 22 23:15:09.716 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:09.716 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:09.716 DEBG [0] It's time to notify for 310
Sep 22 23:15:09.717 INFO Completion from [0] id:310 status:true
Sep 22 23:15:09.717 INFO [311/752] Repair commands completed
Sep 22 23:15:09.717 INFO Pop front: ReconcileIO { id: ReconciliationId(311), op: ExtentReopen { repair_id: ReconciliationId(311), extent_id: 92 }, state: ClientData([New, New, New]) }
Sep 22 23:15:09.717 INFO Sent repair work, now wait for resp
Sep 22 23:15:09.717 INFO [0] received reconcile message
Sep 22 23:15:09.717 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(311), op: ExtentReopen { repair_id: ReconciliationId(311), extent_id: 92 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:09.717 INFO [0] client ExtentReopen { repair_id: ReconciliationId(311), extent_id: 92 }
Sep 22 23:15:09.717 INFO [1] received reconcile message
Sep 22 23:15:09.717 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(311), op: ExtentReopen { repair_id: ReconciliationId(311), extent_id: 92 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:09.717 INFO [1] client ExtentReopen { repair_id: ReconciliationId(311), extent_id: 92 }
Sep 22 23:15:09.717 INFO [2] received reconcile message
Sep 22 23:15:09.717 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(311), op: ExtentReopen { repair_id: ReconciliationId(311), extent_id: 92 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:09.717 INFO [2] client ExtentReopen { repair_id: ReconciliationId(311), extent_id: 92 }
Sep 22 23:15:09.717 DEBG 311 Reopen extent 92
Sep 22 23:15:09.718 DEBG 311 Reopen extent 92
Sep 22 23:15:09.718 DEBG 311 Reopen extent 92
Sep 22 23:15:09.719 DEBG [2] It's time to notify for 311
Sep 22 23:15:09.719 INFO Completion from [2] id:311 status:true
Sep 22 23:15:09.719 INFO [312/752] Repair commands completed
Sep 22 23:15:09.719 INFO Pop front: ReconcileIO { id: ReconciliationId(312), op: ExtentFlush { repair_id: ReconciliationId(312), extent_id: 147, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:09.719 INFO Sent repair work, now wait for resp
Sep 22 23:15:09.719 INFO [0] received reconcile message
Sep 22 23:15:09.719 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(312), op: ExtentFlush { repair_id: ReconciliationId(312), extent_id: 147, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:09.719 INFO [0] client ExtentFlush { repair_id: ReconciliationId(312), extent_id: 147, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:09.719 INFO [1] received reconcile message
Sep 22 23:15:09.719 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(312), op: ExtentFlush { repair_id: ReconciliationId(312), extent_id: 147, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:09.719 INFO [1] client ExtentFlush { repair_id: ReconciliationId(312), extent_id: 147, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:09.719 INFO [2] received reconcile message
Sep 22 23:15:09.719 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(312), op: ExtentFlush { repair_id: ReconciliationId(312), extent_id: 147, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:09.719 INFO [2] client ExtentFlush { repair_id: ReconciliationId(312), extent_id: 147, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:09.719 DEBG 312 Flush extent 147 with f:2 g:2
Sep 22 23:15:09.719 DEBG Flush just extent 147 with f:2 and g:2
Sep 22 23:15:09.719 DEBG [1] It's time to notify for 312
Sep 22 23:15:09.719 INFO Completion from [1] id:312 status:true
Sep 22 23:15:09.719 INFO [313/752] Repair commands completed
Sep 22 23:15:09.719 INFO Pop front: ReconcileIO { id: ReconciliationId(313), op: ExtentClose { repair_id: ReconciliationId(313), extent_id: 147 }, state: ClientData([New, New, New]) }
Sep 22 23:15:09.719 INFO Sent repair work, now wait for resp
Sep 22 23:15:09.719 INFO [0] received reconcile message
Sep 22 23:15:09.720 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(313), op: ExtentClose { repair_id: ReconciliationId(313), extent_id: 147 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:09.720 INFO [0] client ExtentClose { repair_id: ReconciliationId(313), extent_id: 147 }
Sep 22 23:15:09.720 INFO [1] received reconcile message
Sep 22 23:15:09.720 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(313), op: ExtentClose { repair_id: ReconciliationId(313), extent_id: 147 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:09.720 INFO [1] client ExtentClose { repair_id: ReconciliationId(313), extent_id: 147 }
Sep 22 23:15:09.720 INFO [2] received reconcile message
Sep 22 23:15:09.720 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(313), op: ExtentClose { repair_id: ReconciliationId(313), extent_id: 147 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:09.720 INFO [2] client ExtentClose { repair_id: ReconciliationId(313), extent_id: 147 }
Sep 22 23:15:09.720 DEBG 313 Close extent 147
Sep 22 23:15:09.720 DEBG 313 Close extent 147
Sep 22 23:15:09.720 DEBG 313 Close extent 147
Sep 22 23:15:09.721 DEBG [2] It's time to notify for 313
Sep 22 23:15:09.721 INFO Completion from [2] id:313 status:true
Sep 22 23:15:09.721 INFO [314/752] Repair commands completed
Sep 22 23:15:09.721 INFO Pop front: ReconcileIO { id: ReconciliationId(314), op: ExtentRepair { repair_id: ReconciliationId(314), extent_id: 147, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:09.721 INFO Sent repair work, now wait for resp
Sep 22 23:15:09.721 INFO [0] received reconcile message
Sep 22 23:15:09.721 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(314), op: ExtentRepair { repair_id: ReconciliationId(314), extent_id: 147, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:09.721 INFO [0] client ExtentRepair { repair_id: ReconciliationId(314), extent_id: 147, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:09.721 INFO [0] Sending repair request ReconciliationId(314)
Sep 22 23:15:09.721 INFO [1] received reconcile message
Sep 22 23:15:09.721 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(314), op: ExtentRepair { repair_id: ReconciliationId(314), extent_id: 147, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:09.721 INFO [1] client ExtentRepair { repair_id: ReconciliationId(314), extent_id: 147, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:09.721 INFO [1] No action required ReconciliationId(314)
Sep 22 23:15:09.721 INFO [2] received reconcile message
Sep 22 23:15:09.721 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(314), op: ExtentRepair { repair_id: ReconciliationId(314), extent_id: 147, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:09.721 INFO [2] client ExtentRepair { repair_id: ReconciliationId(314), extent_id: 147, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:09.721 INFO [2] No action required ReconciliationId(314)
Sep 22 23:15:09.721 DEBG 314 Repair extent 147 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:09.721 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/093.copy"
Sep 22 23:15:09.782 INFO accepted connection, remote_addr: 127.0.0.1:46870, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.782 TRCE incoming request, uri: /extent/147/files, method: GET, req_id: 20031d70-0c35-4397-b021-e141acdd59c4, remote_addr: 127.0.0.1:46870, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.783 INFO request completed, latency_us: 212, response_code: 200, uri: /extent/147/files, method: GET, req_id: 20031d70-0c35-4397-b021-e141acdd59c4, remote_addr: 127.0.0.1:46870, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.783 INFO eid:147 Found repair files: ["093", "093.db"]
Sep 22 23:15:09.783 TRCE incoming request, uri: /newextent/147/data, method: GET, req_id: 1c62ead1-9800-4f65-8a6b-dbab3be9147b, remote_addr: 127.0.0.1:46870, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.784 INFO request completed, latency_us: 330, response_code: 200, uri: /newextent/147/data, method: GET, req_id: 1c62ead1-9800-4f65-8a6b-dbab3be9147b, remote_addr: 127.0.0.1:46870, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.789 TRCE incoming request, uri: /newextent/147/db, method: GET, req_id: f3924cc3-7bb4-4008-af40-91ce74de18aa, remote_addr: 127.0.0.1:46870, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.789 INFO request completed, latency_us: 293, response_code: 200, uri: /newextent/147/db, method: GET, req_id: f3924cc3-7bb4-4008-af40-91ce74de18aa, remote_addr: 127.0.0.1:46870, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.790 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/093.copy" to "/tmp/downstairs-vrx8aK6L/00/000/093.replace"
Sep 22 23:15:09.790 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:09.791 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/093.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:09.791 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/093"
Sep 22 23:15:09.791 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/093.db"
Sep 22 23:15:09.791 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:09.791 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/093.replace" to "/tmp/downstairs-vrx8aK6L/00/000/093.completed"
Sep 22 23:15:09.791 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:09.791 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:09.792 DEBG [0] It's time to notify for 314
Sep 22 23:15:09.792 INFO Completion from [0] id:314 status:true
Sep 22 23:15:09.792 INFO [315/752] Repair commands completed
Sep 22 23:15:09.792 INFO Pop front: ReconcileIO { id: ReconciliationId(315), op: ExtentReopen { repair_id: ReconciliationId(315), extent_id: 147 }, state: ClientData([New, New, New]) }
Sep 22 23:15:09.792 INFO Sent repair work, now wait for resp
Sep 22 23:15:09.792 INFO [0] received reconcile message
Sep 22 23:15:09.792 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(315), op: ExtentReopen { repair_id: ReconciliationId(315), extent_id: 147 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:09.792 INFO [0] client ExtentReopen { repair_id: ReconciliationId(315), extent_id: 147 }
Sep 22 23:15:09.792 INFO [1] received reconcile message
Sep 22 23:15:09.792 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(315), op: ExtentReopen { repair_id: ReconciliationId(315), extent_id: 147 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:09.792 INFO [1] client ExtentReopen { repair_id: ReconciliationId(315), extent_id: 147 }
Sep 22 23:15:09.792 INFO [2] received reconcile message
Sep 22 23:15:09.792 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(315), op: ExtentReopen { repair_id: ReconciliationId(315), extent_id: 147 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:09.792 INFO [2] client ExtentReopen { repair_id: ReconciliationId(315), extent_id: 147 }
Sep 22 23:15:09.792 DEBG 315 Reopen extent 147
Sep 22 23:15:09.793 DEBG 315 Reopen extent 147
Sep 22 23:15:09.793 DEBG 315 Reopen extent 147
Sep 22 23:15:09.794 DEBG [2] It's time to notify for 315
Sep 22 23:15:09.794 INFO Completion from [2] id:315 status:true
Sep 22 23:15:09.794 INFO [316/752] Repair commands completed
Sep 22 23:15:09.794 INFO Pop front: ReconcileIO { id: ReconciliationId(316), op: ExtentFlush { repair_id: ReconciliationId(316), extent_id: 111, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:09.794 INFO Sent repair work, now wait for resp
Sep 22 23:15:09.794 INFO [0] received reconcile message
Sep 22 23:15:09.794 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(316), op: ExtentFlush { repair_id: ReconciliationId(316), extent_id: 111, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:09.794 INFO [0] client ExtentFlush { repair_id: ReconciliationId(316), extent_id: 111, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:09.794 INFO [1] received reconcile message
Sep 22 23:15:09.794 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(316), op: ExtentFlush { repair_id: ReconciliationId(316), extent_id: 111, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:09.794 INFO [1] client ExtentFlush { repair_id: ReconciliationId(316), extent_id: 111, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:09.794 INFO [2] received reconcile message
Sep 22 23:15:09.794 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(316), op: ExtentFlush { repair_id: ReconciliationId(316), extent_id: 111, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:09.794 INFO [2] client ExtentFlush { repair_id: ReconciliationId(316), extent_id: 111, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:09.794 DEBG 316 Flush extent 111 with f:2 g:2
Sep 22 23:15:09.794 DEBG Flush just extent 111 with f:2 and g:2
Sep 22 23:15:09.795 DEBG [1] It's time to notify for 316
Sep 22 23:15:09.795 INFO Completion from [1] id:316 status:true
Sep 22 23:15:09.795 INFO [317/752] Repair commands completed
Sep 22 23:15:09.795 INFO Pop front: ReconcileIO { id: ReconciliationId(317), op: ExtentClose { repair_id: ReconciliationId(317), extent_id: 111 }, state: ClientData([New, New, New]) }
Sep 22 23:15:09.795 INFO Sent repair work, now wait for resp
Sep 22 23:15:09.795 INFO [0] received reconcile message
Sep 22 23:15:09.795 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(317), op: ExtentClose { repair_id: ReconciliationId(317), extent_id: 111 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:09.795 INFO [0] client ExtentClose { repair_id: ReconciliationId(317), extent_id: 111 }
Sep 22 23:15:09.795 INFO [1] received reconcile message
Sep 22 23:15:09.795 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(317), op: ExtentClose { repair_id: ReconciliationId(317), extent_id: 111 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:09.795 INFO [1] client ExtentClose { repair_id: ReconciliationId(317), extent_id: 111 }
Sep 22 23:15:09.795 INFO [2] received reconcile message
Sep 22 23:15:09.795 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(317), op: ExtentClose { repair_id: ReconciliationId(317), extent_id: 111 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:09.795 INFO [2] client ExtentClose { repair_id: ReconciliationId(317), extent_id: 111 }
Sep 22 23:15:09.795 DEBG 317 Close extent 111
Sep 22 23:15:09.795 DEBG 317 Close extent 111
Sep 22 23:15:09.796 DEBG 317 Close extent 111
Sep 22 23:15:09.796 DEBG [2] It's time to notify for 317
Sep 22 23:15:09.796 INFO Completion from [2] id:317 status:true
Sep 22 23:15:09.796 INFO [318/752] Repair commands completed
Sep 22 23:15:09.796 INFO Pop front: ReconcileIO { id: ReconciliationId(318), op: ExtentRepair { repair_id: ReconciliationId(318), extent_id: 111, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:09.796 INFO Sent repair work, now wait for resp
Sep 22 23:15:09.796 INFO [0] received reconcile message
Sep 22 23:15:09.796 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(318), op: ExtentRepair { repair_id: ReconciliationId(318), extent_id: 111, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:09.796 INFO [0] client ExtentRepair { repair_id: ReconciliationId(318), extent_id: 111, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:09.796 INFO [0] Sending repair request ReconciliationId(318)
Sep 22 23:15:09.796 INFO [1] received reconcile message
Sep 22 23:15:09.796 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(318), op: ExtentRepair { repair_id: ReconciliationId(318), extent_id: 111, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:09.796 INFO [1] client ExtentRepair { repair_id: ReconciliationId(318), extent_id: 111, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:09.796 INFO [1] No action required ReconciliationId(318)
Sep 22 23:15:09.796 INFO [2] received reconcile message
Sep 22 23:15:09.796 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(318), op: ExtentRepair { repair_id: ReconciliationId(318), extent_id: 111, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:09.796 INFO [2] client ExtentRepair { repair_id: ReconciliationId(318), extent_id: 111, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:09.796 INFO [2] No action required ReconciliationId(318)
Sep 22 23:15:09.796 DEBG 318 Repair extent 111 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:09.796 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/06F.copy"
Sep 22 23:15:09.861 INFO accepted connection, remote_addr: 127.0.0.1:44048, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.861 TRCE incoming request, uri: /extent/111/files, method: GET, req_id: a8e9f900-2f5b-4ca5-a68e-2e73d8b87642, remote_addr: 127.0.0.1:44048, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.861 INFO request completed, latency_us: 216, response_code: 200, uri: /extent/111/files, method: GET, req_id: a8e9f900-2f5b-4ca5-a68e-2e73d8b87642, remote_addr: 127.0.0.1:44048, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.862 INFO eid:111 Found repair files: ["06F", "06F.db"]
Sep 22 23:15:09.862 TRCE incoming request, uri: /newextent/111/data, method: GET, req_id: ca891a44-b9b6-44e7-86a7-03ba10eb6159, remote_addr: 127.0.0.1:44048, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.862 INFO request completed, latency_us: 337, response_code: 200, uri: /newextent/111/data, method: GET, req_id: ca891a44-b9b6-44e7-86a7-03ba10eb6159, remote_addr: 127.0.0.1:44048, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.867 TRCE incoming request, uri: /newextent/111/db, method: GET, req_id: a59cab81-144e-41ea-92fc-6822d4d1344c, remote_addr: 127.0.0.1:44048, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.868 INFO request completed, latency_us: 294, response_code: 200, uri: /newextent/111/db, method: GET, req_id: a59cab81-144e-41ea-92fc-6822d4d1344c, remote_addr: 127.0.0.1:44048, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.869 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/06F.copy" to "/tmp/downstairs-vrx8aK6L/00/000/06F.replace"
Sep 22 23:15:09.869 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:09.870 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/06F.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:09.870 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/06F"
Sep 22 23:15:09.870 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/06F.db"
Sep 22 23:15:09.870 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:09.870 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/06F.replace" to "/tmp/downstairs-vrx8aK6L/00/000/06F.completed"
Sep 22 23:15:09.870 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:09.870 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:09.870 DEBG [0] It's time to notify for 318
Sep 22 23:15:09.871 INFO Completion from [0] id:318 status:true
Sep 22 23:15:09.871 INFO [319/752] Repair commands completed
Sep 22 23:15:09.871 INFO Pop front: ReconcileIO { id: ReconciliationId(319), op: ExtentReopen { repair_id: ReconciliationId(319), extent_id: 111 }, state: ClientData([New, New, New]) }
Sep 22 23:15:09.871 INFO Sent repair work, now wait for resp
Sep 22 23:15:09.871 INFO [0] received reconcile message
Sep 22 23:15:09.871 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(319), op: ExtentReopen { repair_id: ReconciliationId(319), extent_id: 111 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:09.871 INFO [0] client ExtentReopen { repair_id: ReconciliationId(319), extent_id: 111 }
Sep 22 23:15:09.871 INFO [1] received reconcile message
Sep 22 23:15:09.871 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(319), op: ExtentReopen { repair_id: ReconciliationId(319), extent_id: 111 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:09.871 INFO [1] client ExtentReopen { repair_id: ReconciliationId(319), extent_id: 111 }
Sep 22 23:15:09.871 INFO [2] received reconcile message
Sep 22 23:15:09.871 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(319), op: ExtentReopen { repair_id: ReconciliationId(319), extent_id: 111 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:09.871 INFO [2] client ExtentReopen { repair_id: ReconciliationId(319), extent_id: 111 }
Sep 22 23:15:09.871 DEBG 319 Reopen extent 111
Sep 22 23:15:09.872 DEBG 319 Reopen extent 111
Sep 22 23:15:09.872 DEBG 319 Reopen extent 111
Sep 22 23:15:09.873 DEBG [2] It's time to notify for 319
Sep 22 23:15:09.873 INFO Completion from [2] id:319 status:true
Sep 22 23:15:09.873 INFO [320/752] Repair commands completed
Sep 22 23:15:09.873 INFO Pop front: ReconcileIO { id: ReconciliationId(320), op: ExtentFlush { repair_id: ReconciliationId(320), extent_id: 187, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:09.873 INFO Sent repair work, now wait for resp
Sep 22 23:15:09.873 INFO [0] received reconcile message
Sep 22 23:15:09.873 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(320), op: ExtentFlush { repair_id: ReconciliationId(320), extent_id: 187, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:09.873 INFO [0] client ExtentFlush { repair_id: ReconciliationId(320), extent_id: 187, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:09.873 INFO [1] received reconcile message
Sep 22 23:15:09.873 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(320), op: ExtentFlush { repair_id: ReconciliationId(320), extent_id: 187, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:09.873 INFO [1] client ExtentFlush { repair_id: ReconciliationId(320), extent_id: 187, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:09.873 INFO [2] received reconcile message
Sep 22 23:15:09.873 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(320), op: ExtentFlush { repair_id: ReconciliationId(320), extent_id: 187, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:09.873 INFO [2] client ExtentFlush { repair_id: ReconciliationId(320), extent_id: 187, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:09.873 DEBG 320 Flush extent 187 with f:2 g:2
Sep 22 23:15:09.873 DEBG Flush just extent 187 with f:2 and g:2
Sep 22 23:15:09.873 DEBG [1] It's time to notify for 320
Sep 22 23:15:09.873 INFO Completion from [1] id:320 status:true
Sep 22 23:15:09.873 INFO [321/752] Repair commands completed
Sep 22 23:15:09.873 INFO Pop front: ReconcileIO { id: ReconciliationId(321), op: ExtentClose { repair_id: ReconciliationId(321), extent_id: 187 }, state: ClientData([New, New, New]) }
Sep 22 23:15:09.873 INFO Sent repair work, now wait for resp
Sep 22 23:15:09.873 INFO [0] received reconcile message
Sep 22 23:15:09.873 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(321), op: ExtentClose { repair_id: ReconciliationId(321), extent_id: 187 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:09.873 INFO [0] client ExtentClose { repair_id: ReconciliationId(321), extent_id: 187 }
Sep 22 23:15:09.874 INFO [1] received reconcile message
Sep 22 23:15:09.874 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(321), op: ExtentClose { repair_id: ReconciliationId(321), extent_id: 187 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:09.874 INFO [1] client ExtentClose { repair_id: ReconciliationId(321), extent_id: 187 }
Sep 22 23:15:09.874 INFO [2] received reconcile message
Sep 22 23:15:09.874 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(321), op: ExtentClose { repair_id: ReconciliationId(321), extent_id: 187 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:09.874 INFO [2] client ExtentClose { repair_id: ReconciliationId(321), extent_id: 187 }
Sep 22 23:15:09.874 DEBG 321 Close extent 187
Sep 22 23:15:09.874 DEBG 321 Close extent 187
Sep 22 23:15:09.874 DEBG 321 Close extent 187
Sep 22 23:15:09.875 DEBG [2] It's time to notify for 321
Sep 22 23:15:09.875 INFO Completion from [2] id:321 status:true
Sep 22 23:15:09.875 INFO [322/752] Repair commands completed
Sep 22 23:15:09.875 INFO Pop front: ReconcileIO { id: ReconciliationId(322), op: ExtentRepair { repair_id: ReconciliationId(322), extent_id: 187, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:09.875 INFO Sent repair work, now wait for resp
Sep 22 23:15:09.875 INFO [0] received reconcile message
Sep 22 23:15:09.875 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(322), op: ExtentRepair { repair_id: ReconciliationId(322), extent_id: 187, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:09.875 INFO [0] client ExtentRepair { repair_id: ReconciliationId(322), extent_id: 187, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:09.875 INFO [0] Sending repair request ReconciliationId(322)
Sep 22 23:15:09.875 INFO [1] received reconcile message
Sep 22 23:15:09.875 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(322), op: ExtentRepair { repair_id: ReconciliationId(322), extent_id: 187, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:09.875 INFO [1] client ExtentRepair { repair_id: ReconciliationId(322), extent_id: 187, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:09.875 INFO [1] No action required ReconciliationId(322)
Sep 22 23:15:09.875 INFO [2] received reconcile message
Sep 22 23:15:09.875 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(322), op: ExtentRepair { repair_id: ReconciliationId(322), extent_id: 187, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:09.875 INFO [2] client ExtentRepair { repair_id: ReconciliationId(322), extent_id: 187, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:09.875 INFO [2] No action required ReconciliationId(322)
Sep 22 23:15:09.875 DEBG 322 Repair extent 187 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:09.875 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/0BB.copy"
Sep 22 23:15:09.937 INFO accepted connection, remote_addr: 127.0.0.1:59897, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.937 TRCE incoming request, uri: /extent/187/files, method: GET, req_id: 42d4932b-2b56-4667-8bf3-1f4ad76e6381, remote_addr: 127.0.0.1:59897, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.937 INFO request completed, latency_us: 217, response_code: 200, uri: /extent/187/files, method: GET, req_id: 42d4932b-2b56-4667-8bf3-1f4ad76e6381, remote_addr: 127.0.0.1:59897, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.937 INFO eid:187 Found repair files: ["0BB", "0BB.db"]
Sep 22 23:15:09.938 TRCE incoming request, uri: /newextent/187/data, method: GET, req_id: be66fccc-1647-415e-9e28-e6bcb1564b8e, remote_addr: 127.0.0.1:59897, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.938 INFO request completed, latency_us: 332, response_code: 200, uri: /newextent/187/data, method: GET, req_id: be66fccc-1647-415e-9e28-e6bcb1564b8e, remote_addr: 127.0.0.1:59897, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.943 TRCE incoming request, uri: /newextent/187/db, method: GET, req_id: e4a9f6d7-2b16-4f2d-a725-e6b57bab09c6, remote_addr: 127.0.0.1:59897, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.943 INFO request completed, latency_us: 292, response_code: 200, uri: /newextent/187/db, method: GET, req_id: e4a9f6d7-2b16-4f2d-a725-e6b57bab09c6, remote_addr: 127.0.0.1:59897, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:09.945 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/0BB.copy" to "/tmp/downstairs-vrx8aK6L/00/000/0BB.replace"
Sep 22 23:15:09.945 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:09.946 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/0BB.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:09.946 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/0BB"
Sep 22 23:15:09.946 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/0BB.db"
Sep 22 23:15:09.946 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:09.946 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/0BB.replace" to "/tmp/downstairs-vrx8aK6L/00/000/0BB.completed"
Sep 22 23:15:09.946 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:09.946 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:09.946 DEBG [0] It's time to notify for 322
Sep 22 23:15:09.946 INFO Completion from [0] id:322 status:true
Sep 22 23:15:09.946 INFO [323/752] Repair commands completed
Sep 22 23:15:09.946 INFO Pop front: ReconcileIO { id: ReconciliationId(323), op: ExtentReopen { repair_id: ReconciliationId(323), extent_id: 187 }, state: ClientData([New, New, New]) }
Sep 22 23:15:09.946 INFO Sent repair work, now wait for resp
Sep 22 23:15:09.946 INFO [0] received reconcile message
Sep 22 23:15:09.946 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(323), op: ExtentReopen { repair_id: ReconciliationId(323), extent_id: 187 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:09.946 INFO [0] client ExtentReopen { repair_id: ReconciliationId(323), extent_id: 187 }
Sep 22 23:15:09.947 INFO [1] received reconcile message
Sep 22 23:15:09.947 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(323), op: ExtentReopen { repair_id: ReconciliationId(323), extent_id: 187 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:09.947 INFO [1] client ExtentReopen { repair_id: ReconciliationId(323), extent_id: 187 }
Sep 22 23:15:09.947 INFO [2] received reconcile message
Sep 22 23:15:09.947 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(323), op: ExtentReopen { repair_id: ReconciliationId(323), extent_id: 187 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:09.947 INFO [2] client ExtentReopen { repair_id: ReconciliationId(323), extent_id: 187 }
Sep 22 23:15:09.947 DEBG 323 Reopen extent 187
Sep 22 23:15:09.947 DEBG 323 Reopen extent 187
Sep 22 23:15:09.948 DEBG 323 Reopen extent 187
Sep 22 23:15:09.948 DEBG [2] It's time to notify for 323
Sep 22 23:15:09.949 INFO Completion from [2] id:323 status:true
Sep 22 23:15:09.949 INFO [324/752] Repair commands completed
Sep 22 23:15:09.949 INFO Pop front: ReconcileIO { id: ReconciliationId(324), op: ExtentFlush { repair_id: ReconciliationId(324), extent_id: 149, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:09.949 INFO Sent repair work, now wait for resp
Sep 22 23:15:09.949 INFO [0] received reconcile message
Sep 22 23:15:09.949 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(324), op: ExtentFlush { repair_id: ReconciliationId(324), extent_id: 149, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:09.949 INFO [0] client ExtentFlush { repair_id: ReconciliationId(324), extent_id: 149, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:09.949 INFO [1] received reconcile message
Sep 22 23:15:09.949 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(324), op: ExtentFlush { repair_id: ReconciliationId(324), extent_id: 149, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:09.949 INFO [1] client ExtentFlush { repair_id: ReconciliationId(324), extent_id: 149, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:09.949 INFO [2] received reconcile message
Sep 22 23:15:09.949 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(324), op: ExtentFlush { repair_id: ReconciliationId(324), extent_id: 149, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:09.949 INFO [2] client ExtentFlush { repair_id: ReconciliationId(324), extent_id: 149, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:09.949 DEBG 324 Flush extent 149 with f:2 g:2
Sep 22 23:15:09.949 DEBG Flush just extent 149 with f:2 and g:2
Sep 22 23:15:09.949 DEBG [1] It's time to notify for 324
Sep 22 23:15:09.949 INFO Completion from [1] id:324 status:true
Sep 22 23:15:09.949 INFO [325/752] Repair commands completed
Sep 22 23:15:09.949 INFO Pop front: ReconcileIO { id: ReconciliationId(325), op: ExtentClose { repair_id: ReconciliationId(325), extent_id: 149 }, state: ClientData([New, New, New]) }
Sep 22 23:15:09.949 INFO Sent repair work, now wait for resp
Sep 22 23:15:09.949 INFO [0] received reconcile message
Sep 22 23:15:09.949 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(325), op: ExtentClose { repair_id: ReconciliationId(325), extent_id: 149 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:09.949 INFO [0] client ExtentClose { repair_id: ReconciliationId(325), extent_id: 149 }
Sep 22 23:15:09.949 INFO [1] received reconcile message
Sep 22 23:15:09.949 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(325), op: ExtentClose { repair_id: ReconciliationId(325), extent_id: 149 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:09.949 INFO [1] client ExtentClose { repair_id: ReconciliationId(325), extent_id: 149 }
Sep 22 23:15:09.949 INFO [2] received reconcile message
Sep 22 23:15:09.949 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(325), op: ExtentClose { repair_id: ReconciliationId(325), extent_id: 149 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:09.949 INFO [2] client ExtentClose { repair_id: ReconciliationId(325), extent_id: 149 }
Sep 22 23:15:09.950 DEBG 325 Close extent 149
Sep 22 23:15:09.950 DEBG 325 Close extent 149
Sep 22 23:15:09.950 DEBG 325 Close extent 149
Sep 22 23:15:09.950 DEBG [2] It's time to notify for 325
Sep 22 23:15:09.951 INFO Completion from [2] id:325 status:true
Sep 22 23:15:09.951 INFO [326/752] Repair commands completed
Sep 22 23:15:09.951 INFO Pop front: ReconcileIO { id: ReconciliationId(326), op: ExtentRepair { repair_id: ReconciliationId(326), extent_id: 149, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:09.951 INFO Sent repair work, now wait for resp
Sep 22 23:15:09.951 INFO [0] received reconcile message
Sep 22 23:15:09.951 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(326), op: ExtentRepair { repair_id: ReconciliationId(326), extent_id: 149, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:09.951 INFO [0] client ExtentRepair { repair_id: ReconciliationId(326), extent_id: 149, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:09.951 INFO [0] Sending repair request ReconciliationId(326)
Sep 22 23:15:09.951 INFO [1] received reconcile message
Sep 22 23:15:09.951 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(326), op: ExtentRepair { repair_id: ReconciliationId(326), extent_id: 149, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:09.951 INFO [1] client ExtentRepair { repair_id: ReconciliationId(326), extent_id: 149, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:09.951 INFO [1] No action required ReconciliationId(326)
Sep 22 23:15:09.951 INFO [2] received reconcile message
Sep 22 23:15:09.951 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(326), op: ExtentRepair { repair_id: ReconciliationId(326), extent_id: 149, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:09.951 INFO [2] client ExtentRepair { repair_id: ReconciliationId(326), extent_id: 149, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:09.951 INFO [2] No action required ReconciliationId(326)
Sep 22 23:15:09.951 DEBG 326 Repair extent 149 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:09.951 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/095.copy"
Sep 22 23:15:10.015 INFO accepted connection, remote_addr: 127.0.0.1:56724, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.015 TRCE incoming request, uri: /extent/149/files, method: GET, req_id: 03e49669-379c-4c59-8037-a8f792dfd807, remote_addr: 127.0.0.1:56724, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.016 INFO request completed, latency_us: 269, response_code: 200, uri: /extent/149/files, method: GET, req_id: 03e49669-379c-4c59-8037-a8f792dfd807, remote_addr: 127.0.0.1:56724, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.016 INFO eid:149 Found repair files: ["095", "095.db"]
Sep 22 23:15:10.016 TRCE incoming request, uri: /newextent/149/data, method: GET, req_id: f67b9d05-450b-4499-b8c9-4f93af7250bf, remote_addr: 127.0.0.1:56724, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.017 INFO request completed, latency_us: 379, response_code: 200, uri: /newextent/149/data, method: GET, req_id: f67b9d05-450b-4499-b8c9-4f93af7250bf, remote_addr: 127.0.0.1:56724, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.022 TRCE incoming request, uri: /newextent/149/db, method: GET, req_id: 2fd5f5b0-3d85-40a2-80eb-6bdd487517f3, remote_addr: 127.0.0.1:56724, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.022 INFO request completed, latency_us: 292, response_code: 200, uri: /newextent/149/db, method: GET, req_id: 2fd5f5b0-3d85-40a2-80eb-6bdd487517f3, remote_addr: 127.0.0.1:56724, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.023 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/095.copy" to "/tmp/downstairs-vrx8aK6L/00/000/095.replace"
Sep 22 23:15:10.023 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:10.025 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/095.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:10.025 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/095"
Sep 22 23:15:10.025 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/095.db"
Sep 22 23:15:10.025 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:10.025 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/095.replace" to "/tmp/downstairs-vrx8aK6L/00/000/095.completed"
Sep 22 23:15:10.025 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:10.025 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:10.025 DEBG [0] It's time to notify for 326
Sep 22 23:15:10.025 INFO Completion from [0] id:326 status:true
Sep 22 23:15:10.025 INFO [327/752] Repair commands completed
Sep 22 23:15:10.025 INFO Pop front: ReconcileIO { id: ReconciliationId(327), op: ExtentReopen { repair_id: ReconciliationId(327), extent_id: 149 }, state: ClientData([New, New, New]) }
Sep 22 23:15:10.025 INFO Sent repair work, now wait for resp
Sep 22 23:15:10.025 INFO [0] received reconcile message
Sep 22 23:15:10.025 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(327), op: ExtentReopen { repair_id: ReconciliationId(327), extent_id: 149 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:10.026 INFO [0] client ExtentReopen { repair_id: ReconciliationId(327), extent_id: 149 }
Sep 22 23:15:10.026 INFO [1] received reconcile message
Sep 22 23:15:10.026 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(327), op: ExtentReopen { repair_id: ReconciliationId(327), extent_id: 149 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:10.026 INFO [1] client ExtentReopen { repair_id: ReconciliationId(327), extent_id: 149 }
Sep 22 23:15:10.026 INFO [2] received reconcile message
Sep 22 23:15:10.026 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(327), op: ExtentReopen { repair_id: ReconciliationId(327), extent_id: 149 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:10.026 INFO [2] client ExtentReopen { repair_id: ReconciliationId(327), extent_id: 149 }
Sep 22 23:15:10.026 DEBG 327 Reopen extent 149
Sep 22 23:15:10.027 DEBG 327 Reopen extent 149
Sep 22 23:15:10.027 DEBG 327 Reopen extent 149
Sep 22 23:15:10.028 DEBG [2] It's time to notify for 327
Sep 22 23:15:10.028 INFO Completion from [2] id:327 status:true
Sep 22 23:15:10.028 INFO [328/752] Repair commands completed
Sep 22 23:15:10.028 INFO Pop front: ReconcileIO { id: ReconciliationId(328), op: ExtentFlush { repair_id: ReconciliationId(328), extent_id: 160, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:10.028 INFO Sent repair work, now wait for resp
Sep 22 23:15:10.028 INFO [0] received reconcile message
Sep 22 23:15:10.028 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(328), op: ExtentFlush { repair_id: ReconciliationId(328), extent_id: 160, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:10.028 INFO [0] client ExtentFlush { repair_id: ReconciliationId(328), extent_id: 160, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:10.028 INFO [1] received reconcile message
Sep 22 23:15:10.028 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(328), op: ExtentFlush { repair_id: ReconciliationId(328), extent_id: 160, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:10.028 INFO [1] client ExtentFlush { repair_id: ReconciliationId(328), extent_id: 160, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:10.028 INFO [2] received reconcile message
Sep 22 23:15:10.028 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(328), op: ExtentFlush { repair_id: ReconciliationId(328), extent_id: 160, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:10.028 INFO [2] client ExtentFlush { repair_id: ReconciliationId(328), extent_id: 160, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:10.028 DEBG 328 Flush extent 160 with f:2 g:2
Sep 22 23:15:10.028 DEBG Flush just extent 160 with f:2 and g:2
Sep 22 23:15:10.028 DEBG [1] It's time to notify for 328
Sep 22 23:15:10.028 INFO Completion from [1] id:328 status:true
Sep 22 23:15:10.028 INFO [329/752] Repair commands completed
Sep 22 23:15:10.028 INFO Pop front: ReconcileIO { id: ReconciliationId(329), op: ExtentClose { repair_id: ReconciliationId(329), extent_id: 160 }, state: ClientData([New, New, New]) }
Sep 22 23:15:10.028 INFO Sent repair work, now wait for resp
Sep 22 23:15:10.028 INFO [0] received reconcile message
Sep 22 23:15:10.028 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(329), op: ExtentClose { repair_id: ReconciliationId(329), extent_id: 160 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:10.028 INFO [0] client ExtentClose { repair_id: ReconciliationId(329), extent_id: 160 }
Sep 22 23:15:10.029 INFO [1] received reconcile message
Sep 22 23:15:10.029 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(329), op: ExtentClose { repair_id: ReconciliationId(329), extent_id: 160 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:10.029 INFO [1] client ExtentClose { repair_id: ReconciliationId(329), extent_id: 160 }
Sep 22 23:15:10.029 INFO [2] received reconcile message
Sep 22 23:15:10.029 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(329), op: ExtentClose { repair_id: ReconciliationId(329), extent_id: 160 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:10.029 INFO [2] client ExtentClose { repair_id: ReconciliationId(329), extent_id: 160 }
Sep 22 23:15:10.029 DEBG 329 Close extent 160
Sep 22 23:15:10.029 DEBG 329 Close extent 160
Sep 22 23:15:10.029 DEBG 329 Close extent 160
Sep 22 23:15:10.030 DEBG [2] It's time to notify for 329
Sep 22 23:15:10.030 INFO Completion from [2] id:329 status:true
Sep 22 23:15:10.030 INFO [330/752] Repair commands completed
Sep 22 23:15:10.030 INFO Pop front: ReconcileIO { id: ReconciliationId(330), op: ExtentRepair { repair_id: ReconciliationId(330), extent_id: 160, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:10.030 INFO Sent repair work, now wait for resp
Sep 22 23:15:10.030 INFO [0] received reconcile message
Sep 22 23:15:10.030 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(330), op: ExtentRepair { repair_id: ReconciliationId(330), extent_id: 160, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:10.030 INFO [0] client ExtentRepair { repair_id: ReconciliationId(330), extent_id: 160, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:10.030 INFO [0] Sending repair request ReconciliationId(330)
Sep 22 23:15:10.030 INFO [1] received reconcile message
Sep 22 23:15:10.030 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(330), op: ExtentRepair { repair_id: ReconciliationId(330), extent_id: 160, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:10.030 INFO [1] client ExtentRepair { repair_id: ReconciliationId(330), extent_id: 160, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:10.030 INFO [1] No action required ReconciliationId(330)
Sep 22 23:15:10.030 INFO [2] received reconcile message
Sep 22 23:15:10.030 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(330), op: ExtentRepair { repair_id: ReconciliationId(330), extent_id: 160, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:10.030 INFO [2] client ExtentRepair { repair_id: ReconciliationId(330), extent_id: 160, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:10.030 INFO [2] No action required ReconciliationId(330)
Sep 22 23:15:10.030 DEBG 330 Repair extent 160 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:10.030 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/0A0.copy"
Sep 22 23:15:10.094 INFO accepted connection, remote_addr: 127.0.0.1:46651, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.094 TRCE incoming request, uri: /extent/160/files, method: GET, req_id: 9e57b6ea-ac05-4977-989a-a3caf010aaaf, remote_addr: 127.0.0.1:46651, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.095 INFO request completed, latency_us: 241, response_code: 200, uri: /extent/160/files, method: GET, req_id: 9e57b6ea-ac05-4977-989a-a3caf010aaaf, remote_addr: 127.0.0.1:46651, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.095 INFO eid:160 Found repair files: ["0A0", "0A0.db"]
Sep 22 23:15:10.095 TRCE incoming request, uri: /newextent/160/data, method: GET, req_id: 6783773b-67e2-4387-b959-004d777537e6, remote_addr: 127.0.0.1:46651, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.096 INFO request completed, latency_us: 350, response_code: 200, uri: /newextent/160/data, method: GET, req_id: 6783773b-67e2-4387-b959-004d777537e6, remote_addr: 127.0.0.1:46651, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.101 TRCE incoming request, uri: /newextent/160/db, method: GET, req_id: 97e1a3fa-86c3-47d7-8408-936e57550e2f, remote_addr: 127.0.0.1:46651, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.101 INFO request completed, latency_us: 292, response_code: 200, uri: /newextent/160/db, method: GET, req_id: 97e1a3fa-86c3-47d7-8408-936e57550e2f, remote_addr: 127.0.0.1:46651, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.102 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/0A0.copy" to "/tmp/downstairs-vrx8aK6L/00/000/0A0.replace"
Sep 22 23:15:10.102 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:10.103 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/0A0.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:10.103 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/0A0"
Sep 22 23:15:10.103 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/0A0.db"
Sep 22 23:15:10.103 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:10.103 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/0A0.replace" to "/tmp/downstairs-vrx8aK6L/00/000/0A0.completed"
Sep 22 23:15:10.103 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:10.103 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:10.104 DEBG [0] It's time to notify for 330
Sep 22 23:15:10.104 INFO Completion from [0] id:330 status:true
Sep 22 23:15:10.104 INFO [331/752] Repair commands completed
Sep 22 23:15:10.104 INFO Pop front: ReconcileIO { id: ReconciliationId(331), op: ExtentReopen { repair_id: ReconciliationId(331), extent_id: 160 }, state: ClientData([New, New, New]) }
Sep 22 23:15:10.104 INFO Sent repair work, now wait for resp
Sep 22 23:15:10.104 INFO [0] received reconcile message
Sep 22 23:15:10.104 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(331), op: ExtentReopen { repair_id: ReconciliationId(331), extent_id: 160 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:10.104 INFO [0] client ExtentReopen { repair_id: ReconciliationId(331), extent_id: 160 }
Sep 22 23:15:10.104 INFO [1] received reconcile message
Sep 22 23:15:10.104 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(331), op: ExtentReopen { repair_id: ReconciliationId(331), extent_id: 160 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:10.104 INFO [1] client ExtentReopen { repair_id: ReconciliationId(331), extent_id: 160 }
Sep 22 23:15:10.104 INFO [2] received reconcile message
Sep 22 23:15:10.104 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(331), op: ExtentReopen { repair_id: ReconciliationId(331), extent_id: 160 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:10.104 INFO [2] client ExtentReopen { repair_id: ReconciliationId(331), extent_id: 160 }
Sep 22 23:15:10.104 DEBG 331 Reopen extent 160
Sep 22 23:15:10.105 DEBG 331 Reopen extent 160
Sep 22 23:15:10.106 DEBG 331 Reopen extent 160
Sep 22 23:15:10.106 DEBG [2] It's time to notify for 331
Sep 22 23:15:10.106 INFO Completion from [2] id:331 status:true
Sep 22 23:15:10.106 INFO [332/752] Repair commands completed
Sep 22 23:15:10.106 INFO Pop front: ReconcileIO { id: ReconciliationId(332), op: ExtentFlush { repair_id: ReconciliationId(332), extent_id: 122, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:10.106 INFO Sent repair work, now wait for resp
Sep 22 23:15:10.106 INFO [0] received reconcile message
Sep 22 23:15:10.106 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(332), op: ExtentFlush { repair_id: ReconciliationId(332), extent_id: 122, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:10.106 INFO [0] client ExtentFlush { repair_id: ReconciliationId(332), extent_id: 122, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:10.106 INFO [1] received reconcile message
Sep 22 23:15:10.106 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(332), op: ExtentFlush { repair_id: ReconciliationId(332), extent_id: 122, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:10.106 INFO [1] client ExtentFlush { repair_id: ReconciliationId(332), extent_id: 122, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:10.106 INFO [2] received reconcile message
Sep 22 23:15:10.106 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(332), op: ExtentFlush { repair_id: ReconciliationId(332), extent_id: 122, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:10.106 INFO [2] client ExtentFlush { repair_id: ReconciliationId(332), extent_id: 122, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:10.107 DEBG 332 Flush extent 122 with f:2 g:2
Sep 22 23:15:10.107 DEBG Flush just extent 122 with f:2 and g:2
Sep 22 23:15:10.107 DEBG [1] It's time to notify for 332
Sep 22 23:15:10.107 INFO Completion from [1] id:332 status:true
Sep 22 23:15:10.107 INFO [333/752] Repair commands completed
Sep 22 23:15:10.107 INFO Pop front: ReconcileIO { id: ReconciliationId(333), op: ExtentClose { repair_id: ReconciliationId(333), extent_id: 122 }, state: ClientData([New, New, New]) }
Sep 22 23:15:10.107 INFO Sent repair work, now wait for resp
Sep 22 23:15:10.107 INFO [0] received reconcile message
Sep 22 23:15:10.107 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(333), op: ExtentClose { repair_id: ReconciliationId(333), extent_id: 122 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:10.107 INFO [0] client ExtentClose { repair_id: ReconciliationId(333), extent_id: 122 }
Sep 22 23:15:10.107 INFO [1] received reconcile message
Sep 22 23:15:10.107 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(333), op: ExtentClose { repair_id: ReconciliationId(333), extent_id: 122 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:10.107 INFO [1] client ExtentClose { repair_id: ReconciliationId(333), extent_id: 122 }
Sep 22 23:15:10.107 INFO [2] received reconcile message
Sep 22 23:15:10.107 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(333), op: ExtentClose { repair_id: ReconciliationId(333), extent_id: 122 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:10.107 INFO [2] client ExtentClose { repair_id: ReconciliationId(333), extent_id: 122 }
Sep 22 23:15:10.107 DEBG 333 Close extent 122
Sep 22 23:15:10.107 DEBG 333 Close extent 122
Sep 22 23:15:10.108 DEBG 333 Close extent 122
Sep 22 23:15:10.108 DEBG [2] It's time to notify for 333
Sep 22 23:15:10.108 INFO Completion from [2] id:333 status:true
Sep 22 23:15:10.108 INFO [334/752] Repair commands completed
Sep 22 23:15:10.108 INFO Pop front: ReconcileIO { id: ReconciliationId(334), op: ExtentRepair { repair_id: ReconciliationId(334), extent_id: 122, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:10.108 INFO Sent repair work, now wait for resp
Sep 22 23:15:10.108 INFO [0] received reconcile message
Sep 22 23:15:10.108 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(334), op: ExtentRepair { repair_id: ReconciliationId(334), extent_id: 122, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:10.108 INFO [0] client ExtentRepair { repair_id: ReconciliationId(334), extent_id: 122, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:10.108 INFO [0] Sending repair request ReconciliationId(334)
Sep 22 23:15:10.108 INFO [1] received reconcile message
Sep 22 23:15:10.108 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(334), op: ExtentRepair { repair_id: ReconciliationId(334), extent_id: 122, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:10.108 INFO [1] client ExtentRepair { repair_id: ReconciliationId(334), extent_id: 122, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:10.108 INFO [1] No action required ReconciliationId(334)
Sep 22 23:15:10.108 INFO [2] received reconcile message
Sep 22 23:15:10.108 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(334), op: ExtentRepair { repair_id: ReconciliationId(334), extent_id: 122, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:10.108 INFO [2] client ExtentRepair { repair_id: ReconciliationId(334), extent_id: 122, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:10.108 INFO [2] No action required ReconciliationId(334)
Sep 22 23:15:10.109 DEBG 334 Repair extent 122 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:10.109 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/07A.copy"
Sep 22 23:15:10.172 INFO accepted connection, remote_addr: 127.0.0.1:52236, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.173 TRCE incoming request, uri: /extent/122/files, method: GET, req_id: 49cdad64-0775-4fd3-93a7-1a1d5bc005af, remote_addr: 127.0.0.1:52236, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.173 INFO request completed, latency_us: 272, response_code: 200, uri: /extent/122/files, method: GET, req_id: 49cdad64-0775-4fd3-93a7-1a1d5bc005af, remote_addr: 127.0.0.1:52236, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.173 INFO eid:122 Found repair files: ["07A", "07A.db"]
Sep 22 23:15:10.174 TRCE incoming request, uri: /newextent/122/data, method: GET, req_id: a9062b19-25a5-401e-9ec1-c3363bfc7428, remote_addr: 127.0.0.1:52236, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.174 INFO request completed, latency_us: 350, response_code: 200, uri: /newextent/122/data, method: GET, req_id: a9062b19-25a5-401e-9ec1-c3363bfc7428, remote_addr: 127.0.0.1:52236, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.179 TRCE incoming request, uri: /newextent/122/db, method: GET, req_id: ef91452f-f889-4537-9240-2686e7dc3f9a, remote_addr: 127.0.0.1:52236, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.179 INFO request completed, latency_us: 295, response_code: 200, uri: /newextent/122/db, method: GET, req_id: ef91452f-f889-4537-9240-2686e7dc3f9a, remote_addr: 127.0.0.1:52236, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.181 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/07A.copy" to "/tmp/downstairs-vrx8aK6L/00/000/07A.replace"
Sep 22 23:15:10.181 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:10.182 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/07A.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:10.182 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/07A"
Sep 22 23:15:10.182 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/07A.db"
Sep 22 23:15:10.182 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:10.182 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/07A.replace" to "/tmp/downstairs-vrx8aK6L/00/000/07A.completed"
Sep 22 23:15:10.182 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:10.183 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:10.183 DEBG [0] It's time to notify for 334
Sep 22 23:15:10.183 INFO Completion from [0] id:334 status:true
Sep 22 23:15:10.183 INFO [335/752] Repair commands completed
Sep 22 23:15:10.183 INFO Pop front: ReconcileIO { id: ReconciliationId(335), op: ExtentReopen { repair_id: ReconciliationId(335), extent_id: 122 }, state: ClientData([New, New, New]) }
Sep 22 23:15:10.183 INFO Sent repair work, now wait for resp
Sep 22 23:15:10.183 INFO [0] received reconcile message
Sep 22 23:15:10.183 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(335), op: ExtentReopen { repair_id: ReconciliationId(335), extent_id: 122 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:10.183 INFO [0] client ExtentReopen { repair_id: ReconciliationId(335), extent_id: 122 }
Sep 22 23:15:10.183 INFO [1] received reconcile message
Sep 22 23:15:10.183 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(335), op: ExtentReopen { repair_id: ReconciliationId(335), extent_id: 122 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:10.183 INFO [1] client ExtentReopen { repair_id: ReconciliationId(335), extent_id: 122 }
Sep 22 23:15:10.183 INFO [2] received reconcile message
Sep 22 23:15:10.183 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(335), op: ExtentReopen { repair_id: ReconciliationId(335), extent_id: 122 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:10.183 INFO [2] client ExtentReopen { repair_id: ReconciliationId(335), extent_id: 122 }
Sep 22 23:15:10.183 DEBG 335 Reopen extent 122
Sep 22 23:15:10.184 DEBG 335 Reopen extent 122
Sep 22 23:15:10.185 DEBG 335 Reopen extent 122
Sep 22 23:15:10.185 DEBG [2] It's time to notify for 335
Sep 22 23:15:10.185 INFO Completion from [2] id:335 status:true
Sep 22 23:15:10.185 INFO [336/752] Repair commands completed
Sep 22 23:15:10.185 INFO Pop front: ReconcileIO { id: ReconciliationId(336), op: ExtentFlush { repair_id: ReconciliationId(336), extent_id: 77, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:10.185 INFO Sent repair work, now wait for resp
Sep 22 23:15:10.185 INFO [0] received reconcile message
Sep 22 23:15:10.185 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(336), op: ExtentFlush { repair_id: ReconciliationId(336), extent_id: 77, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:10.185 INFO [0] client ExtentFlush { repair_id: ReconciliationId(336), extent_id: 77, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:10.186 INFO [1] received reconcile message
Sep 22 23:15:10.186 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(336), op: ExtentFlush { repair_id: ReconciliationId(336), extent_id: 77, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:10.186 INFO [1] client ExtentFlush { repair_id: ReconciliationId(336), extent_id: 77, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:10.186 INFO [2] received reconcile message
Sep 22 23:15:10.186 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(336), op: ExtentFlush { repair_id: ReconciliationId(336), extent_id: 77, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:10.186 INFO [2] client ExtentFlush { repair_id: ReconciliationId(336), extent_id: 77, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:10.186 DEBG 336 Flush extent 77 with f:2 g:2
Sep 22 23:15:10.186 DEBG Flush just extent 77 with f:2 and g:2
Sep 22 23:15:10.186 DEBG [1] It's time to notify for 336
Sep 22 23:15:10.186 INFO Completion from [1] id:336 status:true
Sep 22 23:15:10.186 INFO [337/752] Repair commands completed
Sep 22 23:15:10.186 INFO Pop front: ReconcileIO { id: ReconciliationId(337), op: ExtentClose { repair_id: ReconciliationId(337), extent_id: 77 }, state: ClientData([New, New, New]) }
Sep 22 23:15:10.186 INFO Sent repair work, now wait for resp
Sep 22 23:15:10.186 INFO [0] received reconcile message
Sep 22 23:15:10.186 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(337), op: ExtentClose { repair_id: ReconciliationId(337), extent_id: 77 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:10.186 INFO [0] client ExtentClose { repair_id: ReconciliationId(337), extent_id: 77 }
Sep 22 23:15:10.186 INFO [1] received reconcile message
Sep 22 23:15:10.186 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(337), op: ExtentClose { repair_id: ReconciliationId(337), extent_id: 77 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:10.186 INFO [1] client ExtentClose { repair_id: ReconciliationId(337), extent_id: 77 }
Sep 22 23:15:10.186 INFO [2] received reconcile message
Sep 22 23:15:10.186 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(337), op: ExtentClose { repair_id: ReconciliationId(337), extent_id: 77 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:10.186 INFO [2] client ExtentClose { repair_id: ReconciliationId(337), extent_id: 77 }
Sep 22 23:15:10.186 DEBG 337 Close extent 77
Sep 22 23:15:10.187 DEBG 337 Close extent 77
Sep 22 23:15:10.187 DEBG 337 Close extent 77
Sep 22 23:15:10.187 DEBG [2] It's time to notify for 337
Sep 22 23:15:10.187 INFO Completion from [2] id:337 status:true
Sep 22 23:15:10.187 INFO [338/752] Repair commands completed
Sep 22 23:15:10.187 INFO Pop front: ReconcileIO { id: ReconciliationId(338), op: ExtentRepair { repair_id: ReconciliationId(338), extent_id: 77, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:10.187 INFO Sent repair work, now wait for resp
Sep 22 23:15:10.187 INFO [0] received reconcile message
Sep 22 23:15:10.187 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(338), op: ExtentRepair { repair_id: ReconciliationId(338), extent_id: 77, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:10.187 INFO [0] client ExtentRepair { repair_id: ReconciliationId(338), extent_id: 77, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:10.187 INFO [0] Sending repair request ReconciliationId(338)
Sep 22 23:15:10.188 INFO [1] received reconcile message
Sep 22 23:15:10.188 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(338), op: ExtentRepair { repair_id: ReconciliationId(338), extent_id: 77, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:10.188 INFO [1] client ExtentRepair { repair_id: ReconciliationId(338), extent_id: 77, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:10.188 INFO [1] No action required ReconciliationId(338)
Sep 22 23:15:10.188 INFO [2] received reconcile message
Sep 22 23:15:10.188 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(338), op: ExtentRepair { repair_id: ReconciliationId(338), extent_id: 77, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:10.188 INFO [2] client ExtentRepair { repair_id: ReconciliationId(338), extent_id: 77, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:10.188 INFO [2] No action required ReconciliationId(338)
Sep 22 23:15:10.188 DEBG 338 Repair extent 77 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:10.188 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/04D.copy"
Sep 22 23:15:10.253 INFO accepted connection, remote_addr: 127.0.0.1:44518, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.254 TRCE incoming request, uri: /extent/77/files, method: GET, req_id: 42b1156e-e643-4cae-abf1-4fc4498bcc15, remote_addr: 127.0.0.1:44518, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.254 INFO request completed, latency_us: 285, response_code: 200, uri: /extent/77/files, method: GET, req_id: 42b1156e-e643-4cae-abf1-4fc4498bcc15, remote_addr: 127.0.0.1:44518, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.254 INFO eid:77 Found repair files: ["04D", "04D.db"]
Sep 22 23:15:10.254 TRCE incoming request, uri: /newextent/77/data, method: GET, req_id: e33f287b-c026-4d75-bb86-9ccdc0c6bf95, remote_addr: 127.0.0.1:44518, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.255 INFO request completed, latency_us: 350, response_code: 200, uri: /newextent/77/data, method: GET, req_id: e33f287b-c026-4d75-bb86-9ccdc0c6bf95, remote_addr: 127.0.0.1:44518, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.260 TRCE incoming request, uri: /newextent/77/db, method: GET, req_id: 3359a5d9-b79a-4f36-a126-6becc0f33873, remote_addr: 127.0.0.1:44518, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.260 INFO request completed, latency_us: 293, response_code: 200, uri: /newextent/77/db, method: GET, req_id: 3359a5d9-b79a-4f36-a126-6becc0f33873, remote_addr: 127.0.0.1:44518, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.261 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/04D.copy" to "/tmp/downstairs-vrx8aK6L/00/000/04D.replace"
Sep 22 23:15:10.261 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:10.262 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/04D.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:10.263 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/04D"
Sep 22 23:15:10.263 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/04D.db"
Sep 22 23:15:10.263 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:10.263 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/04D.replace" to "/tmp/downstairs-vrx8aK6L/00/000/04D.completed"
Sep 22 23:15:10.263 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:10.263 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:10.263 DEBG [0] It's time to notify for 338
Sep 22 23:15:10.263 INFO Completion from [0] id:338 status:true
Sep 22 23:15:10.263 INFO [339/752] Repair commands completed
Sep 22 23:15:10.263 INFO Pop front: ReconcileIO { id: ReconciliationId(339), op: ExtentReopen { repair_id: ReconciliationId(339), extent_id: 77 }, state: ClientData([New, New, New]) }
Sep 22 23:15:10.263 INFO Sent repair work, now wait for resp
Sep 22 23:15:10.263 INFO [0] received reconcile message
Sep 22 23:15:10.263 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(339), op: ExtentReopen { repair_id: ReconciliationId(339), extent_id: 77 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:10.263 INFO [0] client ExtentReopen { repair_id: ReconciliationId(339), extent_id: 77 }
Sep 22 23:15:10.264 INFO [1] received reconcile message
Sep 22 23:15:10.264 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(339), op: ExtentReopen { repair_id: ReconciliationId(339), extent_id: 77 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:10.264 INFO [1] client ExtentReopen { repair_id: ReconciliationId(339), extent_id: 77 }
Sep 22 23:15:10.264 INFO [2] received reconcile message
Sep 22 23:15:10.264 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(339), op: ExtentReopen { repair_id: ReconciliationId(339), extent_id: 77 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:10.264 INFO [2] client ExtentReopen { repair_id: ReconciliationId(339), extent_id: 77 }
Sep 22 23:15:10.264 DEBG 339 Reopen extent 77
Sep 22 23:15:10.265 DEBG 339 Reopen extent 77
Sep 22 23:15:10.265 DEBG 339 Reopen extent 77
Sep 22 23:15:10.266 DEBG [2] It's time to notify for 339
Sep 22 23:15:10.266 INFO Completion from [2] id:339 status:true
Sep 22 23:15:10.266 INFO [340/752] Repair commands completed
Sep 22 23:15:10.266 INFO Pop front: ReconcileIO { id: ReconciliationId(340), op: ExtentFlush { repair_id: ReconciliationId(340), extent_id: 136, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:10.266 INFO Sent repair work, now wait for resp
Sep 22 23:15:10.266 INFO [0] received reconcile message
Sep 22 23:15:10.266 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(340), op: ExtentFlush { repair_id: ReconciliationId(340), extent_id: 136, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:10.266 INFO [0] client ExtentFlush { repair_id: ReconciliationId(340), extent_id: 136, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:10.266 INFO [1] received reconcile message
Sep 22 23:15:10.266 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(340), op: ExtentFlush { repair_id: ReconciliationId(340), extent_id: 136, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:10.266 INFO [1] client ExtentFlush { repair_id: ReconciliationId(340), extent_id: 136, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:10.266 INFO [2] received reconcile message
Sep 22 23:15:10.266 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(340), op: ExtentFlush { repair_id: ReconciliationId(340), extent_id: 136, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:10.266 INFO [2] client ExtentFlush { repair_id: ReconciliationId(340), extent_id: 136, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:10.266 DEBG 340 Flush extent 136 with f:2 g:2
Sep 22 23:15:10.266 DEBG Flush just extent 136 with f:2 and g:2
Sep 22 23:15:10.266 DEBG [1] It's time to notify for 340
Sep 22 23:15:10.266 INFO Completion from [1] id:340 status:true
Sep 22 23:15:10.266 INFO [341/752] Repair commands completed
Sep 22 23:15:10.266 INFO Pop front: ReconcileIO { id: ReconciliationId(341), op: ExtentClose { repair_id: ReconciliationId(341), extent_id: 136 }, state: ClientData([New, New, New]) }
Sep 22 23:15:10.266 INFO Sent repair work, now wait for resp
Sep 22 23:15:10.266 INFO [0] received reconcile message
Sep 22 23:15:10.266 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(341), op: ExtentClose { repair_id: ReconciliationId(341), extent_id: 136 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:10.266 INFO [0] client ExtentClose { repair_id: ReconciliationId(341), extent_id: 136 }
Sep 22 23:15:10.267 INFO [1] received reconcile message
Sep 22 23:15:10.267 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(341), op: ExtentClose { repair_id: ReconciliationId(341), extent_id: 136 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:10.267 INFO [1] client ExtentClose { repair_id: ReconciliationId(341), extent_id: 136 }
Sep 22 23:15:10.267 INFO [2] received reconcile message
Sep 22 23:15:10.267 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(341), op: ExtentClose { repair_id: ReconciliationId(341), extent_id: 136 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:10.267 INFO [2] client ExtentClose { repair_id: ReconciliationId(341), extent_id: 136 }
Sep 22 23:15:10.267 DEBG 341 Close extent 136
Sep 22 23:15:10.267 DEBG 341 Close extent 136
Sep 22 23:15:10.267 DEBG 341 Close extent 136
Sep 22 23:15:10.268 DEBG [2] It's time to notify for 341
Sep 22 23:15:10.268 INFO Completion from [2] id:341 status:true
Sep 22 23:15:10.268 INFO [342/752] Repair commands completed
Sep 22 23:15:10.268 INFO Pop front: ReconcileIO { id: ReconciliationId(342), op: ExtentRepair { repair_id: ReconciliationId(342), extent_id: 136, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:10.268 INFO Sent repair work, now wait for resp
Sep 22 23:15:10.268 INFO [0] received reconcile message
Sep 22 23:15:10.268 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(342), op: ExtentRepair { repair_id: ReconciliationId(342), extent_id: 136, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:10.268 INFO [0] client ExtentRepair { repair_id: ReconciliationId(342), extent_id: 136, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:10.268 INFO [0] Sending repair request ReconciliationId(342)
Sep 22 23:15:10.268 INFO [1] received reconcile message
Sep 22 23:15:10.268 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(342), op: ExtentRepair { repair_id: ReconciliationId(342), extent_id: 136, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:10.268 INFO [1] client ExtentRepair { repair_id: ReconciliationId(342), extent_id: 136, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:10.268 INFO [1] No action required ReconciliationId(342)
Sep 22 23:15:10.268 INFO [2] received reconcile message
Sep 22 23:15:10.268 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(342), op: ExtentRepair { repair_id: ReconciliationId(342), extent_id: 136, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:10.268 INFO [2] client ExtentRepair { repair_id: ReconciliationId(342), extent_id: 136, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:10.268 INFO [2] No action required ReconciliationId(342)
Sep 22 23:15:10.268 DEBG 342 Repair extent 136 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:10.268 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/088.copy"
Sep 22 23:15:10.271 DEBG up_ds_listen was notified
Sep 22 23:15:10.271 DEBG up_ds_listen process 1081
Sep 22 23:15:10.271 DEBG [A] ack job 1081:82, : downstairs
Sep 22 23:15:10.271 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:10.278 DEBG IO Flush 1083 has deps [JobId(1082), JobId(1081)]
Sep 22 23:15:10.278 INFO [lossy] sleeping 1 second
Sep 22 23:15:10.334 INFO accepted connection, remote_addr: 127.0.0.1:42290, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.334 TRCE incoming request, uri: /extent/136/files, method: GET, req_id: 3a5634e7-c00d-4520-8b85-67f909c1934c, remote_addr: 127.0.0.1:42290, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.334 INFO request completed, latency_us: 260, response_code: 200, uri: /extent/136/files, method: GET, req_id: 3a5634e7-c00d-4520-8b85-67f909c1934c, remote_addr: 127.0.0.1:42290, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.335 INFO eid:136 Found repair files: ["088", "088.db"]
Sep 22 23:15:10.335 TRCE incoming request, uri: /newextent/136/data, method: GET, req_id: 243f0fba-3f9c-4d67-86db-13aee6bc2efc, remote_addr: 127.0.0.1:42290, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.335 INFO request completed, latency_us: 365, response_code: 200, uri: /newextent/136/data, method: GET, req_id: 243f0fba-3f9c-4d67-86db-13aee6bc2efc, remote_addr: 127.0.0.1:42290, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.340 TRCE incoming request, uri: /newextent/136/db, method: GET, req_id: d49eaf16-cb43-4ede-ac66-874f208665d0, remote_addr: 127.0.0.1:42290, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.341 INFO request completed, latency_us: 292, response_code: 200, uri: /newextent/136/db, method: GET, req_id: d49eaf16-cb43-4ede-ac66-874f208665d0, remote_addr: 127.0.0.1:42290, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.342 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/088.copy" to "/tmp/downstairs-vrx8aK6L/00/000/088.replace"
Sep 22 23:15:10.342 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:10.343 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/088.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:10.343 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/088"
Sep 22 23:15:10.343 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/088.db"
Sep 22 23:15:10.343 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:10.343 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/088.replace" to "/tmp/downstairs-vrx8aK6L/00/000/088.completed"
Sep 22 23:15:10.343 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:10.343 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:10.343 DEBG [0] It's time to notify for 342
Sep 22 23:15:10.344 INFO Completion from [0] id:342 status:true
Sep 22 23:15:10.344 INFO [343/752] Repair commands completed
Sep 22 23:15:10.344 INFO Pop front: ReconcileIO { id: ReconciliationId(343), op: ExtentReopen { repair_id: ReconciliationId(343), extent_id: 136 }, state: ClientData([New, New, New]) }
Sep 22 23:15:10.344 INFO Sent repair work, now wait for resp
Sep 22 23:15:10.344 INFO [0] received reconcile message
Sep 22 23:15:10.344 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(343), op: ExtentReopen { repair_id: ReconciliationId(343), extent_id: 136 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:10.344 INFO [0] client ExtentReopen { repair_id: ReconciliationId(343), extent_id: 136 }
Sep 22 23:15:10.344 INFO [1] received reconcile message
Sep 22 23:15:10.344 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(343), op: ExtentReopen { repair_id: ReconciliationId(343), extent_id: 136 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:10.344 INFO [1] client ExtentReopen { repair_id: ReconciliationId(343), extent_id: 136 }
Sep 22 23:15:10.344 INFO [2] received reconcile message
Sep 22 23:15:10.344 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(343), op: ExtentReopen { repair_id: ReconciliationId(343), extent_id: 136 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:10.344 INFO [2] client ExtentReopen { repair_id: ReconciliationId(343), extent_id: 136 }
Sep 22 23:15:10.344 DEBG 343 Reopen extent 136
Sep 22 23:15:10.345 DEBG 343 Reopen extent 136
Sep 22 23:15:10.345 DEBG 343 Reopen extent 136
Sep 22 23:15:10.346 DEBG [2] It's time to notify for 343
Sep 22 23:15:10.346 INFO Completion from [2] id:343 status:true
Sep 22 23:15:10.346 INFO [344/752] Repair commands completed
Sep 22 23:15:10.346 INFO Pop front: ReconcileIO { id: ReconciliationId(344), op: ExtentFlush { repair_id: ReconciliationId(344), extent_id: 176, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:10.346 INFO Sent repair work, now wait for resp
Sep 22 23:15:10.346 INFO [0] received reconcile message
Sep 22 23:15:10.346 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(344), op: ExtentFlush { repair_id: ReconciliationId(344), extent_id: 176, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:10.346 INFO [0] client ExtentFlush { repair_id: ReconciliationId(344), extent_id: 176, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:10.346 INFO [1] received reconcile message
Sep 22 23:15:10.346 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(344), op: ExtentFlush { repair_id: ReconciliationId(344), extent_id: 176, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:10.346 INFO [1] client ExtentFlush { repair_id: ReconciliationId(344), extent_id: 176, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:10.346 INFO [2] received reconcile message
Sep 22 23:15:10.346 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(344), op: ExtentFlush { repair_id: ReconciliationId(344), extent_id: 176, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:10.346 INFO [2] client ExtentFlush { repair_id: ReconciliationId(344), extent_id: 176, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:10.346 DEBG 344 Flush extent 176 with f:2 g:2
Sep 22 23:15:10.346 DEBG Flush just extent 176 with f:2 and g:2
Sep 22 23:15:10.347 DEBG [1] It's time to notify for 344
Sep 22 23:15:10.347 INFO Completion from [1] id:344 status:true
Sep 22 23:15:10.347 INFO [345/752] Repair commands completed
Sep 22 23:15:10.347 INFO Pop front: ReconcileIO { id: ReconciliationId(345), op: ExtentClose { repair_id: ReconciliationId(345), extent_id: 176 }, state: ClientData([New, New, New]) }
Sep 22 23:15:10.347 INFO Sent repair work, now wait for resp
Sep 22 23:15:10.347 INFO [0] received reconcile message
Sep 22 23:15:10.347 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(345), op: ExtentClose { repair_id: ReconciliationId(345), extent_id: 176 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:10.347 INFO [0] client ExtentClose { repair_id: ReconciliationId(345), extent_id: 176 }
Sep 22 23:15:10.347 INFO [1] received reconcile message
Sep 22 23:15:10.347 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(345), op: ExtentClose { repair_id: ReconciliationId(345), extent_id: 176 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:10.347 INFO [1] client ExtentClose { repair_id: ReconciliationId(345), extent_id: 176 }
Sep 22 23:15:10.347 INFO [2] received reconcile message
Sep 22 23:15:10.347 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(345), op: ExtentClose { repair_id: ReconciliationId(345), extent_id: 176 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:10.347 INFO [2] client ExtentClose { repair_id: ReconciliationId(345), extent_id: 176 }
Sep 22 23:15:10.347 DEBG 345 Close extent 176
Sep 22 23:15:10.347 DEBG 345 Close extent 176
Sep 22 23:15:10.347 DEBG 345 Close extent 176
Sep 22 23:15:10.348 DEBG [2] It's time to notify for 345
Sep 22 23:15:10.348 INFO Completion from [2] id:345 status:true
Sep 22 23:15:10.348 INFO [346/752] Repair commands completed
Sep 22 23:15:10.348 INFO Pop front: ReconcileIO { id: ReconciliationId(346), op: ExtentRepair { repair_id: ReconciliationId(346), extent_id: 176, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:10.348 INFO Sent repair work, now wait for resp
Sep 22 23:15:10.348 INFO [0] received reconcile message
Sep 22 23:15:10.348 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(346), op: ExtentRepair { repair_id: ReconciliationId(346), extent_id: 176, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:10.348 INFO [0] client ExtentRepair { repair_id: ReconciliationId(346), extent_id: 176, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:10.348 INFO [0] Sending repair request ReconciliationId(346)
Sep 22 23:15:10.348 INFO [1] received reconcile message
Sep 22 23:15:10.348 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(346), op: ExtentRepair { repair_id: ReconciliationId(346), extent_id: 176, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:10.348 INFO [1] client ExtentRepair { repair_id: ReconciliationId(346), extent_id: 176, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:10.348 INFO [1] No action required ReconciliationId(346)
Sep 22 23:15:10.348 INFO [2] received reconcile message
Sep 22 23:15:10.348 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(346), op: ExtentRepair { repair_id: ReconciliationId(346), extent_id: 176, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:10.348 INFO [2] client ExtentRepair { repair_id: ReconciliationId(346), extent_id: 176, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:10.348 INFO [2] No action required ReconciliationId(346)
Sep 22 23:15:10.348 DEBG 346 Repair extent 176 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:10.348 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/0B0.copy"
Sep 22 23:15:10.412 INFO accepted connection, remote_addr: 127.0.0.1:43530, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.412 TRCE incoming request, uri: /extent/176/files, method: GET, req_id: fb775b24-9680-4ca8-8975-4843c619161d, remote_addr: 127.0.0.1:43530, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.412 INFO request completed, latency_us: 192, response_code: 200, uri: /extent/176/files, method: GET, req_id: fb775b24-9680-4ca8-8975-4843c619161d, remote_addr: 127.0.0.1:43530, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.412 INFO eid:176 Found repair files: ["0B0", "0B0.db"]
Sep 22 23:15:10.413 TRCE incoming request, uri: /newextent/176/data, method: GET, req_id: 32907a1f-247c-414a-b1f9-d59f0fb65c44, remote_addr: 127.0.0.1:43530, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.413 INFO request completed, latency_us: 296, response_code: 200, uri: /newextent/176/data, method: GET, req_id: 32907a1f-247c-414a-b1f9-d59f0fb65c44, remote_addr: 127.0.0.1:43530, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.418 TRCE incoming request, uri: /newextent/176/db, method: GET, req_id: aa3c9787-5d3c-4696-936a-4989710bcd12, remote_addr: 127.0.0.1:43530, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.418 INFO request completed, latency_us: 297, response_code: 200, uri: /newextent/176/db, method: GET, req_id: aa3c9787-5d3c-4696-936a-4989710bcd12, remote_addr: 127.0.0.1:43530, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.419 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/0B0.copy" to "/tmp/downstairs-vrx8aK6L/00/000/0B0.replace"
Sep 22 23:15:10.420 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:10.420 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/0B0.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:10.420 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/0B0"
Sep 22 23:15:10.421 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/0B0.db"
Sep 22 23:15:10.421 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:10.421 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/0B0.replace" to "/tmp/downstairs-vrx8aK6L/00/000/0B0.completed"
Sep 22 23:15:10.421 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:10.421 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:10.421 DEBG [0] It's time to notify for 346
Sep 22 23:15:10.421 INFO Completion from [0] id:346 status:true
Sep 22 23:15:10.421 INFO [347/752] Repair commands completed
Sep 22 23:15:10.421 INFO Pop front: ReconcileIO { id: ReconciliationId(347), op: ExtentReopen { repair_id: ReconciliationId(347), extent_id: 176 }, state: ClientData([New, New, New]) }
Sep 22 23:15:10.421 INFO Sent repair work, now wait for resp
Sep 22 23:15:10.421 INFO [0] received reconcile message
Sep 22 23:15:10.421 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(347), op: ExtentReopen { repair_id: ReconciliationId(347), extent_id: 176 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:10.421 INFO [0] client ExtentReopen { repair_id: ReconciliationId(347), extent_id: 176 }
Sep 22 23:15:10.421 INFO [1] received reconcile message
Sep 22 23:15:10.421 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(347), op: ExtentReopen { repair_id: ReconciliationId(347), extent_id: 176 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:10.421 INFO [1] client ExtentReopen { repair_id: ReconciliationId(347), extent_id: 176 }
Sep 22 23:15:10.421 INFO [2] received reconcile message
Sep 22 23:15:10.421 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(347), op: ExtentReopen { repair_id: ReconciliationId(347), extent_id: 176 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:10.421 INFO [2] client ExtentReopen { repair_id: ReconciliationId(347), extent_id: 176 }
Sep 22 23:15:10.421 DEBG 347 Reopen extent 176
Sep 22 23:15:10.422 DEBG 347 Reopen extent 176
Sep 22 23:15:10.422 DEBG 347 Reopen extent 176
Sep 22 23:15:10.423 DEBG [2] It's time to notify for 347
Sep 22 23:15:10.423 INFO Completion from [2] id:347 status:true
Sep 22 23:15:10.423 INFO [348/752] Repair commands completed
Sep 22 23:15:10.423 INFO Pop front: ReconcileIO { id: ReconciliationId(348), op: ExtentFlush { repair_id: ReconciliationId(348), extent_id: 125, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:10.423 INFO Sent repair work, now wait for resp
Sep 22 23:15:10.423 INFO [0] received reconcile message
Sep 22 23:15:10.423 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(348), op: ExtentFlush { repair_id: ReconciliationId(348), extent_id: 125, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:10.423 INFO [0] client ExtentFlush { repair_id: ReconciliationId(348), extent_id: 125, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:10.423 INFO [1] received reconcile message
Sep 22 23:15:10.423 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(348), op: ExtentFlush { repair_id: ReconciliationId(348), extent_id: 125, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:10.423 INFO [1] client ExtentFlush { repair_id: ReconciliationId(348), extent_id: 125, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:10.423 INFO [2] received reconcile message
Sep 22 23:15:10.423 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(348), op: ExtentFlush { repair_id: ReconciliationId(348), extent_id: 125, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:10.423 INFO [2] client ExtentFlush { repair_id: ReconciliationId(348), extent_id: 125, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:10.423 DEBG 348 Flush extent 125 with f:2 g:2
Sep 22 23:15:10.423 DEBG Flush just extent 125 with f:2 and g:2
Sep 22 23:15:10.424 DEBG [1] It's time to notify for 348
Sep 22 23:15:10.424 INFO Completion from [1] id:348 status:true
Sep 22 23:15:10.424 INFO [349/752] Repair commands completed
Sep 22 23:15:10.424 INFO Pop front: ReconcileIO { id: ReconciliationId(349), op: ExtentClose { repair_id: ReconciliationId(349), extent_id: 125 }, state: ClientData([New, New, New]) }
Sep 22 23:15:10.424 INFO Sent repair work, now wait for resp
Sep 22 23:15:10.424 INFO [0] received reconcile message
Sep 22 23:15:10.424 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(349), op: ExtentClose { repair_id: ReconciliationId(349), extent_id: 125 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:10.424 INFO [0] client ExtentClose { repair_id: ReconciliationId(349), extent_id: 125 }
Sep 22 23:15:10.424 INFO [1] received reconcile message
Sep 22 23:15:10.424 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(349), op: ExtentClose { repair_id: ReconciliationId(349), extent_id: 125 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:10.424 INFO [1] client ExtentClose { repair_id: ReconciliationId(349), extent_id: 125 }
Sep 22 23:15:10.424 INFO [2] received reconcile message
Sep 22 23:15:10.424 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(349), op: ExtentClose { repair_id: ReconciliationId(349), extent_id: 125 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:10.424 INFO [2] client ExtentClose { repair_id: ReconciliationId(349), extent_id: 125 }
Sep 22 23:15:10.424 DEBG 349 Close extent 125
Sep 22 23:15:10.424 DEBG 349 Close extent 125
Sep 22 23:15:10.425 DEBG 349 Close extent 125
Sep 22 23:15:10.425 DEBG [2] It's time to notify for 349
Sep 22 23:15:10.425 INFO Completion from [2] id:349 status:true
Sep 22 23:15:10.425 INFO [350/752] Repair commands completed
Sep 22 23:15:10.425 INFO Pop front: ReconcileIO { id: ReconciliationId(350), op: ExtentRepair { repair_id: ReconciliationId(350), extent_id: 125, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:10.425 INFO Sent repair work, now wait for resp
Sep 22 23:15:10.425 INFO [0] received reconcile message
Sep 22 23:15:10.425 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(350), op: ExtentRepair { repair_id: ReconciliationId(350), extent_id: 125, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:10.425 INFO [0] client ExtentRepair { repair_id: ReconciliationId(350), extent_id: 125, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:10.425 INFO [0] Sending repair request ReconciliationId(350)
Sep 22 23:15:10.425 INFO [1] received reconcile message
Sep 22 23:15:10.425 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(350), op: ExtentRepair { repair_id: ReconciliationId(350), extent_id: 125, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:10.425 INFO [1] client ExtentRepair { repair_id: ReconciliationId(350), extent_id: 125, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:10.425 INFO [1] No action required ReconciliationId(350)
Sep 22 23:15:10.425 INFO [2] received reconcile message
Sep 22 23:15:10.425 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(350), op: ExtentRepair { repair_id: ReconciliationId(350), extent_id: 125, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:10.425 INFO [2] client ExtentRepair { repair_id: ReconciliationId(350), extent_id: 125, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:10.425 INFO [2] No action required ReconciliationId(350)
Sep 22 23:15:10.425 DEBG 350 Repair extent 125 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:10.425 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/07D.copy"
Sep 22 23:15:10.490 INFO accepted connection, remote_addr: 127.0.0.1:43501, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.490 TRCE incoming request, uri: /extent/125/files, method: GET, req_id: d5b88c7e-704a-499f-b964-1a5cee27bb60, remote_addr: 127.0.0.1:43501, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.490 INFO request completed, latency_us: 199, response_code: 200, uri: /extent/125/files, method: GET, req_id: d5b88c7e-704a-499f-b964-1a5cee27bb60, remote_addr: 127.0.0.1:43501, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.491 INFO eid:125 Found repair files: ["07D", "07D.db"]
Sep 22 23:15:10.491 TRCE incoming request, uri: /newextent/125/data, method: GET, req_id: 0f1dd2d1-2ceb-4827-807e-72cfdc7edabb, remote_addr: 127.0.0.1:43501, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.491 INFO request completed, latency_us: 260, response_code: 200, uri: /newextent/125/data, method: GET, req_id: 0f1dd2d1-2ceb-4827-807e-72cfdc7edabb, remote_addr: 127.0.0.1:43501, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.496 TRCE incoming request, uri: /newextent/125/db, method: GET, req_id: e0535523-61ad-446d-8e9b-b3a3176bd6d2, remote_addr: 127.0.0.1:43501, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.497 INFO request completed, latency_us: 296, response_code: 200, uri: /newextent/125/db, method: GET, req_id: e0535523-61ad-446d-8e9b-b3a3176bd6d2, remote_addr: 127.0.0.1:43501, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.498 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/07D.copy" to "/tmp/downstairs-vrx8aK6L/00/000/07D.replace"
Sep 22 23:15:10.498 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:10.498 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/07D.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:10.499 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/07D"
Sep 22 23:15:10.499 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/07D.db"
Sep 22 23:15:10.499 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:10.499 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/07D.replace" to "/tmp/downstairs-vrx8aK6L/00/000/07D.completed"
Sep 22 23:15:10.499 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:10.499 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:10.499 DEBG [0] It's time to notify for 350
Sep 22 23:15:10.499 INFO Completion from [0] id:350 status:true
Sep 22 23:15:10.499 INFO [351/752] Repair commands completed
Sep 22 23:15:10.499 INFO Pop front: ReconcileIO { id: ReconciliationId(351), op: ExtentReopen { repair_id: ReconciliationId(351), extent_id: 125 }, state: ClientData([New, New, New]) }
Sep 22 23:15:10.499 INFO Sent repair work, now wait for resp
Sep 22 23:15:10.499 INFO [0] received reconcile message
Sep 22 23:15:10.499 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(351), op: ExtentReopen { repair_id: ReconciliationId(351), extent_id: 125 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:10.499 INFO [0] client ExtentReopen { repair_id: ReconciliationId(351), extent_id: 125 }
Sep 22 23:15:10.499 INFO [1] received reconcile message
Sep 22 23:15:10.499 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(351), op: ExtentReopen { repair_id: ReconciliationId(351), extent_id: 125 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:10.499 INFO [1] client ExtentReopen { repair_id: ReconciliationId(351), extent_id: 125 }
Sep 22 23:15:10.500 INFO [2] received reconcile message
Sep 22 23:15:10.500 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(351), op: ExtentReopen { repair_id: ReconciliationId(351), extent_id: 125 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:10.500 INFO [2] client ExtentReopen { repair_id: ReconciliationId(351), extent_id: 125 }
Sep 22 23:15:10.500 DEBG 351 Reopen extent 125
Sep 22 23:15:10.500 DEBG 351 Reopen extent 125
Sep 22 23:15:10.501 DEBG 351 Reopen extent 125
Sep 22 23:15:10.501 DEBG [2] It's time to notify for 351
Sep 22 23:15:10.501 INFO Completion from [2] id:351 status:true
Sep 22 23:15:10.501 INFO [352/752] Repair commands completed
Sep 22 23:15:10.501 INFO Pop front: ReconcileIO { id: ReconciliationId(352), op: ExtentFlush { repair_id: ReconciliationId(352), extent_id: 90, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:10.501 INFO Sent repair work, now wait for resp
Sep 22 23:15:10.502 INFO [0] received reconcile message
Sep 22 23:15:10.502 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(352), op: ExtentFlush { repair_id: ReconciliationId(352), extent_id: 90, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:10.502 INFO [0] client ExtentFlush { repair_id: ReconciliationId(352), extent_id: 90, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:10.502 INFO [1] received reconcile message
Sep 22 23:15:10.502 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(352), op: ExtentFlush { repair_id: ReconciliationId(352), extent_id: 90, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:10.502 INFO [1] client ExtentFlush { repair_id: ReconciliationId(352), extent_id: 90, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:10.502 INFO [2] received reconcile message
Sep 22 23:15:10.502 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(352), op: ExtentFlush { repair_id: ReconciliationId(352), extent_id: 90, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:10.502 INFO [2] client ExtentFlush { repair_id: ReconciliationId(352), extent_id: 90, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:10.502 DEBG 352 Flush extent 90 with f:2 g:2
Sep 22 23:15:10.502 DEBG Flush just extent 90 with f:2 and g:2
Sep 22 23:15:10.502 DEBG [1] It's time to notify for 352
Sep 22 23:15:10.502 INFO [lossy] skipping 1082
Sep 22 23:15:10.502 INFO Completion from [1] id:352 status:true
Sep 22 23:15:10.502 INFO [353/752] Repair commands completed
Sep 22 23:15:10.502 INFO [lossy] skipping 1082
Sep 22 23:15:10.502 INFO Pop front: ReconcileIO { id: ReconciliationId(353), op: ExtentClose { repair_id: ReconciliationId(353), extent_id: 90 }, state: ClientData([New, New, New]) }
Sep 22 23:15:10.502 INFO Sent repair work, now wait for resp
Sep 22 23:15:10.502 INFO [0] received reconcile message
Sep 22 23:15:10.502 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(353), op: ExtentClose { repair_id: ReconciliationId(353), extent_id: 90 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:10.502 INFO [0] client ExtentClose { repair_id: ReconciliationId(353), extent_id: 90 }
Sep 22 23:15:10.502 INFO [1] received reconcile message
Sep 22 23:15:10.502 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(353), op: ExtentClose { repair_id: ReconciliationId(353), extent_id: 90 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:10.502 INFO [1] client ExtentClose { repair_id: ReconciliationId(353), extent_id: 90 }
Sep 22 23:15:10.502 INFO [2] received reconcile message
Sep 22 23:15:10.502 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(353), op: ExtentClose { repair_id: ReconciliationId(353), extent_id: 90 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:10.502 INFO [2] client ExtentClose { repair_id: ReconciliationId(353), extent_id: 90 }
Sep 22 23:15:10.502 DEBG 353 Close extent 90
Sep 22 23:15:10.503 DEBG 353 Close extent 90
Sep 22 23:15:10.503 DEBG 353 Close extent 90
Sep 22 23:15:10.503 DEBG [2] It's time to notify for 353
Sep 22 23:15:10.503 INFO Completion from [2] id:353 status:true
Sep 22 23:15:10.503 INFO [354/752] Repair commands completed
Sep 22 23:15:10.503 INFO Pop front: ReconcileIO { id: ReconciliationId(354), op: ExtentRepair { repair_id: ReconciliationId(354), extent_id: 90, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:10.503 INFO Sent repair work, now wait for resp
Sep 22 23:15:10.503 INFO [0] received reconcile message
Sep 22 23:15:10.504 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(354), op: ExtentRepair { repair_id: ReconciliationId(354), extent_id: 90, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:10.504 INFO [0] client ExtentRepair { repair_id: ReconciliationId(354), extent_id: 90, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:10.504 INFO [0] Sending repair request ReconciliationId(354)
Sep 22 23:15:10.504 INFO [1] received reconcile message
Sep 22 23:15:10.504 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(354), op: ExtentRepair { repair_id: ReconciliationId(354), extent_id: 90, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:10.504 INFO [1] client ExtentRepair { repair_id: ReconciliationId(354), extent_id: 90, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:10.504 INFO [1] No action required ReconciliationId(354)
Sep 22 23:15:10.504 INFO [2] received reconcile message
Sep 22 23:15:10.504 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(354), op: ExtentRepair { repair_id: ReconciliationId(354), extent_id: 90, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:10.504 INFO [2] client ExtentRepair { repair_id: ReconciliationId(354), extent_id: 90, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:10.504 INFO [2] No action required ReconciliationId(354)
Sep 22 23:15:10.504 DEBG 354 Repair extent 90 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:10.504 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/05A.copy"
Sep 22 23:15:10.508 DEBG Read      :1082 deps:[JobId(1081)] res:true
Sep 22 23:15:10.530 DEBG Flush     :1081 extent_limit None deps:[JobId(1080)] res:true f:30 g:1
Sep 22 23:15:10.537 DEBG Read      :1082 deps:[JobId(1081)] res:true
Sep 22 23:15:10.558 DEBG [rc] retire 1081 clears [JobId(1080), JobId(1081)], : downstairs
Sep 22 23:15:10.561 DEBG Flush     :1083 extent_limit None deps:[JobId(1082), JobId(1081)] res:true f:31 g:1
Sep 22 23:15:10.561 INFO [lossy] sleeping 1 second
Sep 22 23:15:10.565 INFO accepted connection, remote_addr: 127.0.0.1:44671, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.565 TRCE incoming request, uri: /extent/90/files, method: GET, req_id: 7a6a6a87-5bb7-4d0a-9b9f-f14a496cd1fb, remote_addr: 127.0.0.1:44671, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.566 INFO request completed, latency_us: 230, response_code: 200, uri: /extent/90/files, method: GET, req_id: 7a6a6a87-5bb7-4d0a-9b9f-f14a496cd1fb, remote_addr: 127.0.0.1:44671, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.566 INFO eid:90 Found repair files: ["05A", "05A.db"]
Sep 22 23:15:10.566 TRCE incoming request, uri: /newextent/90/data, method: GET, req_id: 42a459ec-ba3e-4c6a-9b63-ee47b24302a7, remote_addr: 127.0.0.1:44671, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.567 INFO request completed, latency_us: 345, response_code: 200, uri: /newextent/90/data, method: GET, req_id: 42a459ec-ba3e-4c6a-9b63-ee47b24302a7, remote_addr: 127.0.0.1:44671, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.572 TRCE incoming request, uri: /newextent/90/db, method: GET, req_id: 21092c30-d57e-4f68-98cf-1bae1ffbe5f4, remote_addr: 127.0.0.1:44671, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.572 INFO request completed, latency_us: 293, response_code: 200, uri: /newextent/90/db, method: GET, req_id: 21092c30-d57e-4f68-98cf-1bae1ffbe5f4, remote_addr: 127.0.0.1:44671, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.573 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/05A.copy" to "/tmp/downstairs-vrx8aK6L/00/000/05A.replace"
Sep 22 23:15:10.573 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:10.574 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/05A.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:10.574 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/05A"
Sep 22 23:15:10.575 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/05A.db"
Sep 22 23:15:10.575 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:10.575 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/05A.replace" to "/tmp/downstairs-vrx8aK6L/00/000/05A.completed"
Sep 22 23:15:10.575 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:10.575 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:10.575 DEBG [0] It's time to notify for 354
Sep 22 23:15:10.575 INFO Completion from [0] id:354 status:true
Sep 22 23:15:10.575 INFO [355/752] Repair commands completed
Sep 22 23:15:10.575 INFO Pop front: ReconcileIO { id: ReconciliationId(355), op: ExtentReopen { repair_id: ReconciliationId(355), extent_id: 90 }, state: ClientData([New, New, New]) }
Sep 22 23:15:10.575 INFO Sent repair work, now wait for resp
Sep 22 23:15:10.575 INFO [0] received reconcile message
Sep 22 23:15:10.575 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(355), op: ExtentReopen { repair_id: ReconciliationId(355), extent_id: 90 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:10.575 INFO [0] client ExtentReopen { repair_id: ReconciliationId(355), extent_id: 90 }
Sep 22 23:15:10.575 INFO [1] received reconcile message
Sep 22 23:15:10.575 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(355), op: ExtentReopen { repair_id: ReconciliationId(355), extent_id: 90 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:10.575 INFO [1] client ExtentReopen { repair_id: ReconciliationId(355), extent_id: 90 }
Sep 22 23:15:10.575 INFO [2] received reconcile message
Sep 22 23:15:10.575 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(355), op: ExtentReopen { repair_id: ReconciliationId(355), extent_id: 90 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:10.575 INFO [2] client ExtentReopen { repair_id: ReconciliationId(355), extent_id: 90 }
Sep 22 23:15:10.575 DEBG 355 Reopen extent 90
Sep 22 23:15:10.576 DEBG 355 Reopen extent 90
Sep 22 23:15:10.577 DEBG 355 Reopen extent 90
Sep 22 23:15:10.577 DEBG [2] It's time to notify for 355
Sep 22 23:15:10.577 INFO Completion from [2] id:355 status:true
Sep 22 23:15:10.577 INFO [356/752] Repair commands completed
Sep 22 23:15:10.577 INFO Pop front: ReconcileIO { id: ReconciliationId(356), op: ExtentFlush { repair_id: ReconciliationId(356), extent_id: 144, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:10.577 INFO Sent repair work, now wait for resp
Sep 22 23:15:10.577 INFO [0] received reconcile message
Sep 22 23:15:10.577 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(356), op: ExtentFlush { repair_id: ReconciliationId(356), extent_id: 144, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:10.577 INFO [0] client ExtentFlush { repair_id: ReconciliationId(356), extent_id: 144, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:10.577 INFO [1] received reconcile message
Sep 22 23:15:10.577 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(356), op: ExtentFlush { repair_id: ReconciliationId(356), extent_id: 144, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:10.577 INFO [1] client ExtentFlush { repair_id: ReconciliationId(356), extent_id: 144, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:10.578 INFO [2] received reconcile message
Sep 22 23:15:10.578 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(356), op: ExtentFlush { repair_id: ReconciliationId(356), extent_id: 144, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:10.578 INFO [2] client ExtentFlush { repair_id: ReconciliationId(356), extent_id: 144, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:10.578 DEBG 356 Flush extent 144 with f:2 g:2
Sep 22 23:15:10.578 DEBG Flush just extent 144 with f:2 and g:2
Sep 22 23:15:10.578 DEBG [1] It's time to notify for 356
Sep 22 23:15:10.578 INFO Completion from [1] id:356 status:true
Sep 22 23:15:10.578 INFO [357/752] Repair commands completed
Sep 22 23:15:10.578 INFO Pop front: ReconcileIO { id: ReconciliationId(357), op: ExtentClose { repair_id: ReconciliationId(357), extent_id: 144 }, state: ClientData([New, New, New]) }
Sep 22 23:15:10.578 INFO Sent repair work, now wait for resp
Sep 22 23:15:10.578 INFO [0] received reconcile message
Sep 22 23:15:10.578 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(357), op: ExtentClose { repair_id: ReconciliationId(357), extent_id: 144 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:10.578 INFO [0] client ExtentClose { repair_id: ReconciliationId(357), extent_id: 144 }
Sep 22 23:15:10.578 INFO [1] received reconcile message
Sep 22 23:15:10.578 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(357), op: ExtentClose { repair_id: ReconciliationId(357), extent_id: 144 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:10.578 INFO [1] client ExtentClose { repair_id: ReconciliationId(357), extent_id: 144 }
Sep 22 23:15:10.578 INFO [2] received reconcile message
Sep 22 23:15:10.578 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(357), op: ExtentClose { repair_id: ReconciliationId(357), extent_id: 144 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:10.578 INFO [2] client ExtentClose { repair_id: ReconciliationId(357), extent_id: 144 }
Sep 22 23:15:10.578 DEBG 357 Close extent 144
Sep 22 23:15:10.578 DEBG 357 Close extent 144
Sep 22 23:15:10.579 DEBG 357 Close extent 144
Sep 22 23:15:10.579 DEBG [2] It's time to notify for 357
Sep 22 23:15:10.579 INFO Completion from [2] id:357 status:true
Sep 22 23:15:10.579 INFO [358/752] Repair commands completed
Sep 22 23:15:10.579 INFO Pop front: ReconcileIO { id: ReconciliationId(358), op: ExtentRepair { repair_id: ReconciliationId(358), extent_id: 144, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:10.579 INFO Sent repair work, now wait for resp
Sep 22 23:15:10.579 INFO [0] received reconcile message
Sep 22 23:15:10.579 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(358), op: ExtentRepair { repair_id: ReconciliationId(358), extent_id: 144, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:10.579 INFO [0] client ExtentRepair { repair_id: ReconciliationId(358), extent_id: 144, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:10.579 INFO [0] Sending repair request ReconciliationId(358)
Sep 22 23:15:10.579 INFO [1] received reconcile message
Sep 22 23:15:10.579 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(358), op: ExtentRepair { repair_id: ReconciliationId(358), extent_id: 144, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:10.579 INFO [1] client ExtentRepair { repair_id: ReconciliationId(358), extent_id: 144, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:10.580 INFO [1] No action required ReconciliationId(358)
Sep 22 23:15:10.580 INFO [2] received reconcile message
Sep 22 23:15:10.580 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(358), op: ExtentRepair { repair_id: ReconciliationId(358), extent_id: 144, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:10.580 INFO [2] client ExtentRepair { repair_id: ReconciliationId(358), extent_id: 144, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:10.580 INFO [2] No action required ReconciliationId(358)
Sep 22 23:15:10.580 DEBG 358 Repair extent 144 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:10.580 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/090.copy"
Sep 22 23:15:10.609 WARN returning error on flush!
Sep 22 23:15:10.609 DEBG Flush     :1083 extent_limit None deps:[JobId(1082), JobId(1081)] res:false f:31 g:1
Sep 22 23:15:10.609 WARN returning error on flush!
Sep 22 23:15:10.609 DEBG Flush     :1083 extent_limit None deps:[JobId(1082), JobId(1081)] res:false f:31 g:1
Sep 22 23:15:10.609 DEBG Flush     :1083 extent_limit None deps:[JobId(1082), JobId(1081)] res:true f:31 g:1
Sep 22 23:15:10.609 INFO [lossy] sleeping 1 second
Sep 22 23:15:10.642 INFO accepted connection, remote_addr: 127.0.0.1:65157, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.642 TRCE incoming request, uri: /extent/144/files, method: GET, req_id: abba7a0e-afcf-49f0-a1b0-c0f41d603f2f, remote_addr: 127.0.0.1:65157, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.642 INFO request completed, latency_us: 206, response_code: 200, uri: /extent/144/files, method: GET, req_id: abba7a0e-afcf-49f0-a1b0-c0f41d603f2f, remote_addr: 127.0.0.1:65157, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.642 INFO eid:144 Found repair files: ["090", "090.db"]
Sep 22 23:15:10.643 TRCE incoming request, uri: /newextent/144/data, method: GET, req_id: d8b39c29-a6b4-43ef-a4a6-bdb9660bf227, remote_addr: 127.0.0.1:65157, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.643 INFO request completed, latency_us: 313, response_code: 200, uri: /newextent/144/data, method: GET, req_id: d8b39c29-a6b4-43ef-a4a6-bdb9660bf227, remote_addr: 127.0.0.1:65157, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.648 TRCE incoming request, uri: /newextent/144/db, method: GET, req_id: 6d897a7c-a234-4792-a2f1-9b62fcb5b1a0, remote_addr: 127.0.0.1:65157, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.648 INFO request completed, latency_us: 298, response_code: 200, uri: /newextent/144/db, method: GET, req_id: 6d897a7c-a234-4792-a2f1-9b62fcb5b1a0, remote_addr: 127.0.0.1:65157, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.649 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/090.copy" to "/tmp/downstairs-vrx8aK6L/00/000/090.replace"
Sep 22 23:15:10.649 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:10.650 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/090.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:10.651 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/090"
Sep 22 23:15:10.651 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/090.db"
Sep 22 23:15:10.651 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:10.651 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/090.replace" to "/tmp/downstairs-vrx8aK6L/00/000/090.completed"
Sep 22 23:15:10.651 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:10.651 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:10.651 DEBG [0] It's time to notify for 358
Sep 22 23:15:10.651 INFO Completion from [0] id:358 status:true
Sep 22 23:15:10.651 INFO [359/752] Repair commands completed
Sep 22 23:15:10.651 INFO Pop front: ReconcileIO { id: ReconciliationId(359), op: ExtentReopen { repair_id: ReconciliationId(359), extent_id: 144 }, state: ClientData([New, New, New]) }
Sep 22 23:15:10.651 INFO Sent repair work, now wait for resp
Sep 22 23:15:10.651 INFO [0] received reconcile message
Sep 22 23:15:10.651 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(359), op: ExtentReopen { repair_id: ReconciliationId(359), extent_id: 144 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:10.651 INFO [0] client ExtentReopen { repair_id: ReconciliationId(359), extent_id: 144 }
Sep 22 23:15:10.651 INFO [1] received reconcile message
Sep 22 23:15:10.651 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(359), op: ExtentReopen { repair_id: ReconciliationId(359), extent_id: 144 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:10.651 INFO [1] client ExtentReopen { repair_id: ReconciliationId(359), extent_id: 144 }
Sep 22 23:15:10.651 INFO [2] received reconcile message
Sep 22 23:15:10.651 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(359), op: ExtentReopen { repair_id: ReconciliationId(359), extent_id: 144 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:10.651 INFO [2] client ExtentReopen { repair_id: ReconciliationId(359), extent_id: 144 }
Sep 22 23:15:10.652 DEBG 359 Reopen extent 144
Sep 22 23:15:10.652 DEBG 359 Reopen extent 144
Sep 22 23:15:10.653 DEBG 359 Reopen extent 144
Sep 22 23:15:10.653 DEBG [2] It's time to notify for 359
Sep 22 23:15:10.653 INFO Completion from [2] id:359 status:true
Sep 22 23:15:10.653 INFO [360/752] Repair commands completed
Sep 22 23:15:10.653 INFO Pop front: ReconcileIO { id: ReconciliationId(360), op: ExtentFlush { repair_id: ReconciliationId(360), extent_id: 156, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:10.653 INFO Sent repair work, now wait for resp
Sep 22 23:15:10.653 INFO [0] received reconcile message
Sep 22 23:15:10.653 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(360), op: ExtentFlush { repair_id: ReconciliationId(360), extent_id: 156, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:10.653 INFO [0] client ExtentFlush { repair_id: ReconciliationId(360), extent_id: 156, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:10.653 INFO [1] received reconcile message
Sep 22 23:15:10.654 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(360), op: ExtentFlush { repair_id: ReconciliationId(360), extent_id: 156, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:10.654 INFO [1] client ExtentFlush { repair_id: ReconciliationId(360), extent_id: 156, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:10.654 INFO [2] received reconcile message
Sep 22 23:15:10.654 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(360), op: ExtentFlush { repair_id: ReconciliationId(360), extent_id: 156, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:10.654 INFO [2] client ExtentFlush { repair_id: ReconciliationId(360), extent_id: 156, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:10.654 DEBG 360 Flush extent 156 with f:2 g:2
Sep 22 23:15:10.654 DEBG Flush just extent 156 with f:2 and g:2
Sep 22 23:15:10.654 DEBG [1] It's time to notify for 360
Sep 22 23:15:10.654 INFO Completion from [1] id:360 status:true
Sep 22 23:15:10.654 INFO [361/752] Repair commands completed
Sep 22 23:15:10.654 INFO Pop front: ReconcileIO { id: ReconciliationId(361), op: ExtentClose { repair_id: ReconciliationId(361), extent_id: 156 }, state: ClientData([New, New, New]) }
Sep 22 23:15:10.654 INFO Sent repair work, now wait for resp
Sep 22 23:15:10.654 INFO [0] received reconcile message
Sep 22 23:15:10.654 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(361), op: ExtentClose { repair_id: ReconciliationId(361), extent_id: 156 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:10.654 INFO [0] client ExtentClose { repair_id: ReconciliationId(361), extent_id: 156 }
Sep 22 23:15:10.654 INFO [1] received reconcile message
Sep 22 23:15:10.654 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(361), op: ExtentClose { repair_id: ReconciliationId(361), extent_id: 156 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:10.654 INFO [1] client ExtentClose { repair_id: ReconciliationId(361), extent_id: 156 }
Sep 22 23:15:10.654 INFO [2] received reconcile message
Sep 22 23:15:10.654 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(361), op: ExtentClose { repair_id: ReconciliationId(361), extent_id: 156 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:10.654 INFO [2] client ExtentClose { repair_id: ReconciliationId(361), extent_id: 156 }
Sep 22 23:15:10.654 DEBG 361 Close extent 156
Sep 22 23:15:10.655 DEBG 361 Close extent 156
Sep 22 23:15:10.655 DEBG 361 Close extent 156
Sep 22 23:15:10.655 DEBG [2] It's time to notify for 361
Sep 22 23:15:10.655 INFO Completion from [2] id:361 status:true
Sep 22 23:15:10.655 INFO [362/752] Repair commands completed
Sep 22 23:15:10.655 INFO Pop front: ReconcileIO { id: ReconciliationId(362), op: ExtentRepair { repair_id: ReconciliationId(362), extent_id: 156, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:10.655 INFO Sent repair work, now wait for resp
Sep 22 23:15:10.655 INFO [0] received reconcile message
Sep 22 23:15:10.655 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(362), op: ExtentRepair { repair_id: ReconciliationId(362), extent_id: 156, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:10.655 INFO [0] client ExtentRepair { repair_id: ReconciliationId(362), extent_id: 156, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:10.655 INFO [0] Sending repair request ReconciliationId(362)
Sep 22 23:15:10.655 INFO [1] received reconcile message
Sep 22 23:15:10.656 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(362), op: ExtentRepair { repair_id: ReconciliationId(362), extent_id: 156, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:10.656 INFO [1] client ExtentRepair { repair_id: ReconciliationId(362), extent_id: 156, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:10.656 INFO [1] No action required ReconciliationId(362)
Sep 22 23:15:10.656 INFO [2] received reconcile message
Sep 22 23:15:10.656 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(362), op: ExtentRepair { repair_id: ReconciliationId(362), extent_id: 156, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:10.656 INFO [2] client ExtentRepair { repair_id: ReconciliationId(362), extent_id: 156, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:10.656 INFO [2] No action required ReconciliationId(362)
Sep 22 23:15:10.656 DEBG 362 Repair extent 156 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:10.656 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/09C.copy"
Sep 22 23:15:10.717 INFO accepted connection, remote_addr: 127.0.0.1:46629, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.718 TRCE incoming request, uri: /extent/156/files, method: GET, req_id: 2dff1d28-5851-482e-bde7-dbfce6e7e751, remote_addr: 127.0.0.1:46629, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.718 INFO request completed, latency_us: 200, response_code: 200, uri: /extent/156/files, method: GET, req_id: 2dff1d28-5851-482e-bde7-dbfce6e7e751, remote_addr: 127.0.0.1:46629, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.718 INFO eid:156 Found repair files: ["09C", "09C.db"]
Sep 22 23:15:10.718 TRCE incoming request, uri: /newextent/156/data, method: GET, req_id: 5dd4a693-bd0b-47a9-ab59-e626a55dbbb9, remote_addr: 127.0.0.1:46629, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.719 INFO request completed, latency_us: 313, response_code: 200, uri: /newextent/156/data, method: GET, req_id: 5dd4a693-bd0b-47a9-ab59-e626a55dbbb9, remote_addr: 127.0.0.1:46629, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.724 TRCE incoming request, uri: /newextent/156/db, method: GET, req_id: f8705a8f-e230-4cd2-99f9-65d61c8bd17c, remote_addr: 127.0.0.1:46629, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.724 INFO request completed, latency_us: 292, response_code: 200, uri: /newextent/156/db, method: GET, req_id: f8705a8f-e230-4cd2-99f9-65d61c8bd17c, remote_addr: 127.0.0.1:46629, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.725 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/09C.copy" to "/tmp/downstairs-vrx8aK6L/00/000/09C.replace"
Sep 22 23:15:10.725 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:10.726 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/09C.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:10.726 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/09C"
Sep 22 23:15:10.726 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/09C.db"
Sep 22 23:15:10.726 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:10.726 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/09C.replace" to "/tmp/downstairs-vrx8aK6L/00/000/09C.completed"
Sep 22 23:15:10.726 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:10.727 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:10.727 DEBG [0] It's time to notify for 362
Sep 22 23:15:10.727 INFO Completion from [0] id:362 status:true
Sep 22 23:15:10.727 INFO [363/752] Repair commands completed
Sep 22 23:15:10.727 INFO Pop front: ReconcileIO { id: ReconciliationId(363), op: ExtentReopen { repair_id: ReconciliationId(363), extent_id: 156 }, state: ClientData([New, New, New]) }
Sep 22 23:15:10.727 INFO Sent repair work, now wait for resp
Sep 22 23:15:10.727 INFO [0] received reconcile message
Sep 22 23:15:10.727 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(363), op: ExtentReopen { repair_id: ReconciliationId(363), extent_id: 156 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:10.727 INFO [0] client ExtentReopen { repair_id: ReconciliationId(363), extent_id: 156 }
Sep 22 23:15:10.727 INFO [1] received reconcile message
Sep 22 23:15:10.727 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(363), op: ExtentReopen { repair_id: ReconciliationId(363), extent_id: 156 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:10.727 INFO [1] client ExtentReopen { repair_id: ReconciliationId(363), extent_id: 156 }
Sep 22 23:15:10.727 INFO [2] received reconcile message
Sep 22 23:15:10.727 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(363), op: ExtentReopen { repair_id: ReconciliationId(363), extent_id: 156 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:10.727 INFO [2] client ExtentReopen { repair_id: ReconciliationId(363), extent_id: 156 }
Sep 22 23:15:10.727 DEBG 363 Reopen extent 156
Sep 22 23:15:10.728 DEBG 363 Reopen extent 156
Sep 22 23:15:10.728 DEBG 363 Reopen extent 156
Sep 22 23:15:10.729 DEBG [2] It's time to notify for 363
Sep 22 23:15:10.729 INFO Completion from [2] id:363 status:true
Sep 22 23:15:10.729 INFO [364/752] Repair commands completed
Sep 22 23:15:10.729 INFO Pop front: ReconcileIO { id: ReconciliationId(364), op: ExtentFlush { repair_id: ReconciliationId(364), extent_id: 33, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:10.729 INFO Sent repair work, now wait for resp
Sep 22 23:15:10.729 INFO [0] received reconcile message
Sep 22 23:15:10.729 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(364), op: ExtentFlush { repair_id: ReconciliationId(364), extent_id: 33, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:10.729 INFO [0] client ExtentFlush { repair_id: ReconciliationId(364), extent_id: 33, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:10.729 INFO [1] received reconcile message
Sep 22 23:15:10.729 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(364), op: ExtentFlush { repair_id: ReconciliationId(364), extent_id: 33, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:10.729 INFO [1] client ExtentFlush { repair_id: ReconciliationId(364), extent_id: 33, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:10.729 INFO [2] received reconcile message
Sep 22 23:15:10.729 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(364), op: ExtentFlush { repair_id: ReconciliationId(364), extent_id: 33, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:10.729 INFO [2] client ExtentFlush { repair_id: ReconciliationId(364), extent_id: 33, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:10.729 DEBG 364 Flush extent 33 with f:2 g:2
Sep 22 23:15:10.729 DEBG Flush just extent 33 with f:2 and g:2
Sep 22 23:15:10.730 DEBG [1] It's time to notify for 364
Sep 22 23:15:10.730 INFO Completion from [1] id:364 status:true
Sep 22 23:15:10.730 INFO [365/752] Repair commands completed
Sep 22 23:15:10.730 INFO Pop front: ReconcileIO { id: ReconciliationId(365), op: ExtentClose { repair_id: ReconciliationId(365), extent_id: 33 }, state: ClientData([New, New, New]) }
Sep 22 23:15:10.730 INFO Sent repair work, now wait for resp
Sep 22 23:15:10.730 INFO [0] received reconcile message
Sep 22 23:15:10.730 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(365), op: ExtentClose { repair_id: ReconciliationId(365), extent_id: 33 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:10.730 INFO [0] client ExtentClose { repair_id: ReconciliationId(365), extent_id: 33 }
Sep 22 23:15:10.730 INFO [1] received reconcile message
Sep 22 23:15:10.730 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(365), op: ExtentClose { repair_id: ReconciliationId(365), extent_id: 33 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:10.730 INFO [1] client ExtentClose { repair_id: ReconciliationId(365), extent_id: 33 }
Sep 22 23:15:10.730 INFO [2] received reconcile message
Sep 22 23:15:10.730 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(365), op: ExtentClose { repair_id: ReconciliationId(365), extent_id: 33 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:10.730 INFO [2] client ExtentClose { repair_id: ReconciliationId(365), extent_id: 33 }
Sep 22 23:15:10.730 DEBG 365 Close extent 33
Sep 22 23:15:10.730 DEBG 365 Close extent 33
Sep 22 23:15:10.731 DEBG 365 Close extent 33
Sep 22 23:15:10.731 DEBG [2] It's time to notify for 365
Sep 22 23:15:10.731 INFO Completion from [2] id:365 status:true
Sep 22 23:15:10.731 INFO [366/752] Repair commands completed
Sep 22 23:15:10.731 INFO Pop front: ReconcileIO { id: ReconciliationId(366), op: ExtentRepair { repair_id: ReconciliationId(366), extent_id: 33, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:10.731 INFO Sent repair work, now wait for resp
Sep 22 23:15:10.731 INFO [0] received reconcile message
Sep 22 23:15:10.731 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(366), op: ExtentRepair { repair_id: ReconciliationId(366), extent_id: 33, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:10.731 INFO [0] client ExtentRepair { repair_id: ReconciliationId(366), extent_id: 33, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:10.731 INFO [0] Sending repair request ReconciliationId(366)
Sep 22 23:15:10.731 INFO [1] received reconcile message
Sep 22 23:15:10.731 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(366), op: ExtentRepair { repair_id: ReconciliationId(366), extent_id: 33, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:10.731 INFO [1] client ExtentRepair { repair_id: ReconciliationId(366), extent_id: 33, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:10.731 INFO [1] No action required ReconciliationId(366)
Sep 22 23:15:10.731 INFO [2] received reconcile message
Sep 22 23:15:10.731 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(366), op: ExtentRepair { repair_id: ReconciliationId(366), extent_id: 33, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:10.731 INFO [2] client ExtentRepair { repair_id: ReconciliationId(366), extent_id: 33, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:10.731 INFO [2] No action required ReconciliationId(366)
Sep 22 23:15:10.731 DEBG 366 Repair extent 33 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:10.732 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/021.copy"
Sep 22 23:15:10.794 INFO accepted connection, remote_addr: 127.0.0.1:40276, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.794 TRCE incoming request, uri: /extent/33/files, method: GET, req_id: 0164b5b8-6b31-44a1-b287-7e4d2c5cdbb4, remote_addr: 127.0.0.1:40276, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.794 INFO request completed, latency_us: 242, response_code: 200, uri: /extent/33/files, method: GET, req_id: 0164b5b8-6b31-44a1-b287-7e4d2c5cdbb4, remote_addr: 127.0.0.1:40276, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.794 INFO eid:33 Found repair files: ["021", "021.db"]
Sep 22 23:15:10.795 TRCE incoming request, uri: /newextent/33/data, method: GET, req_id: f905dba8-bdcc-4829-9585-6ab9bf0a6092, remote_addr: 127.0.0.1:40276, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.795 INFO request completed, latency_us: 332, response_code: 200, uri: /newextent/33/data, method: GET, req_id: f905dba8-bdcc-4829-9585-6ab9bf0a6092, remote_addr: 127.0.0.1:40276, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.800 TRCE incoming request, uri: /newextent/33/db, method: GET, req_id: 5777fda6-75b2-48da-a6b0-ffd3d277438c, remote_addr: 127.0.0.1:40276, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.801 INFO request completed, latency_us: 296, response_code: 200, uri: /newextent/33/db, method: GET, req_id: 5777fda6-75b2-48da-a6b0-ffd3d277438c, remote_addr: 127.0.0.1:40276, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.802 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/021.copy" to "/tmp/downstairs-vrx8aK6L/00/000/021.replace"
Sep 22 23:15:10.802 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:10.803 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/021.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:10.803 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/021"
Sep 22 23:15:10.803 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/021.db"
Sep 22 23:15:10.803 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:10.803 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/021.replace" to "/tmp/downstairs-vrx8aK6L/00/000/021.completed"
Sep 22 23:15:10.803 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:10.803 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:10.803 DEBG [0] It's time to notify for 366
Sep 22 23:15:10.804 INFO Completion from [0] id:366 status:true
Sep 22 23:15:10.804 INFO [367/752] Repair commands completed
Sep 22 23:15:10.804 INFO Pop front: ReconcileIO { id: ReconciliationId(367), op: ExtentReopen { repair_id: ReconciliationId(367), extent_id: 33 }, state: ClientData([New, New, New]) }
Sep 22 23:15:10.804 INFO Sent repair work, now wait for resp
Sep 22 23:15:10.804 INFO [0] received reconcile message
Sep 22 23:15:10.804 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(367), op: ExtentReopen { repair_id: ReconciliationId(367), extent_id: 33 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:10.804 INFO [0] client ExtentReopen { repair_id: ReconciliationId(367), extent_id: 33 }
Sep 22 23:15:10.804 INFO [1] received reconcile message
Sep 22 23:15:10.804 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(367), op: ExtentReopen { repair_id: ReconciliationId(367), extent_id: 33 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:10.804 INFO [1] client ExtentReopen { repair_id: ReconciliationId(367), extent_id: 33 }
Sep 22 23:15:10.804 INFO [2] received reconcile message
Sep 22 23:15:10.804 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(367), op: ExtentReopen { repair_id: ReconciliationId(367), extent_id: 33 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:10.804 INFO [2] client ExtentReopen { repair_id: ReconciliationId(367), extent_id: 33 }
Sep 22 23:15:10.804 DEBG 367 Reopen extent 33
Sep 22 23:15:10.805 DEBG 367 Reopen extent 33
Sep 22 23:15:10.805 DEBG 367 Reopen extent 33
Sep 22 23:15:10.806 DEBG [2] It's time to notify for 367
Sep 22 23:15:10.806 INFO Completion from [2] id:367 status:true
Sep 22 23:15:10.806 INFO [368/752] Repair commands completed
Sep 22 23:15:10.806 INFO Pop front: ReconcileIO { id: ReconciliationId(368), op: ExtentFlush { repair_id: ReconciliationId(368), extent_id: 48, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:10.806 INFO Sent repair work, now wait for resp
Sep 22 23:15:10.806 INFO [0] received reconcile message
Sep 22 23:15:10.806 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(368), op: ExtentFlush { repair_id: ReconciliationId(368), extent_id: 48, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:10.806 INFO [0] client ExtentFlush { repair_id: ReconciliationId(368), extent_id: 48, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:10.806 INFO [1] received reconcile message
Sep 22 23:15:10.806 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(368), op: ExtentFlush { repair_id: ReconciliationId(368), extent_id: 48, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:10.806 INFO [1] client ExtentFlush { repair_id: ReconciliationId(368), extent_id: 48, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:10.806 INFO [2] received reconcile message
Sep 22 23:15:10.806 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(368), op: ExtentFlush { repair_id: ReconciliationId(368), extent_id: 48, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:10.806 INFO [2] client ExtentFlush { repair_id: ReconciliationId(368), extent_id: 48, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:10.806 DEBG 368 Flush extent 48 with f:2 g:2
Sep 22 23:15:10.806 DEBG Flush just extent 48 with f:2 and g:2
Sep 22 23:15:10.806 DEBG [1] It's time to notify for 368
Sep 22 23:15:10.806 INFO Completion from [1] id:368 status:true
Sep 22 23:15:10.807 INFO [369/752] Repair commands completed
Sep 22 23:15:10.807 INFO Pop front: ReconcileIO { id: ReconciliationId(369), op: ExtentClose { repair_id: ReconciliationId(369), extent_id: 48 }, state: ClientData([New, New, New]) }
Sep 22 23:15:10.807 INFO Sent repair work, now wait for resp
Sep 22 23:15:10.807 INFO [0] received reconcile message
Sep 22 23:15:10.807 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(369), op: ExtentClose { repair_id: ReconciliationId(369), extent_id: 48 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:10.807 INFO [0] client ExtentClose { repair_id: ReconciliationId(369), extent_id: 48 }
Sep 22 23:15:10.807 INFO [1] received reconcile message
Sep 22 23:15:10.807 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(369), op: ExtentClose { repair_id: ReconciliationId(369), extent_id: 48 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:10.807 INFO [1] client ExtentClose { repair_id: ReconciliationId(369), extent_id: 48 }
Sep 22 23:15:10.807 INFO [2] received reconcile message
Sep 22 23:15:10.807 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(369), op: ExtentClose { repair_id: ReconciliationId(369), extent_id: 48 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:10.807 INFO [2] client ExtentClose { repair_id: ReconciliationId(369), extent_id: 48 }
Sep 22 23:15:10.807 DEBG 369 Close extent 48
Sep 22 23:15:10.807 DEBG 369 Close extent 48
Sep 22 23:15:10.807 DEBG 369 Close extent 48
Sep 22 23:15:10.808 DEBG [2] It's time to notify for 369
Sep 22 23:15:10.808 INFO Completion from [2] id:369 status:true
Sep 22 23:15:10.808 INFO [370/752] Repair commands completed
Sep 22 23:15:10.808 INFO Pop front: ReconcileIO { id: ReconciliationId(370), op: ExtentRepair { repair_id: ReconciliationId(370), extent_id: 48, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:10.808 INFO Sent repair work, now wait for resp
Sep 22 23:15:10.808 INFO [0] received reconcile message
Sep 22 23:15:10.808 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(370), op: ExtentRepair { repair_id: ReconciliationId(370), extent_id: 48, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:10.808 INFO [0] client ExtentRepair { repair_id: ReconciliationId(370), extent_id: 48, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:10.808 INFO [0] Sending repair request ReconciliationId(370)
Sep 22 23:15:10.808 INFO [1] received reconcile message
Sep 22 23:15:10.808 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(370), op: ExtentRepair { repair_id: ReconciliationId(370), extent_id: 48, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:10.808 INFO [1] client ExtentRepair { repair_id: ReconciliationId(370), extent_id: 48, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:10.808 INFO [1] No action required ReconciliationId(370)
Sep 22 23:15:10.808 INFO [2] received reconcile message
Sep 22 23:15:10.808 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(370), op: ExtentRepair { repair_id: ReconciliationId(370), extent_id: 48, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:10.808 INFO [2] client ExtentRepair { repair_id: ReconciliationId(370), extent_id: 48, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:10.808 INFO [2] No action required ReconciliationId(370)
Sep 22 23:15:10.808 DEBG 370 Repair extent 48 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:10.808 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/030.copy"
Sep 22 23:15:10.869 INFO accepted connection, remote_addr: 127.0.0.1:42406, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.869 TRCE incoming request, uri: /extent/48/files, method: GET, req_id: bda9bef3-73c2-4aef-8477-4f9bade941a5, remote_addr: 127.0.0.1:42406, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.870 INFO request completed, latency_us: 213, response_code: 200, uri: /extent/48/files, method: GET, req_id: bda9bef3-73c2-4aef-8477-4f9bade941a5, remote_addr: 127.0.0.1:42406, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.870 INFO eid:48 Found repair files: ["030", "030.db"]
Sep 22 23:15:10.870 TRCE incoming request, uri: /newextent/48/data, method: GET, req_id: 20fc99f7-bc7d-429d-ad31-d2d63c0c4b88, remote_addr: 127.0.0.1:42406, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.870 INFO request completed, latency_us: 324, response_code: 200, uri: /newextent/48/data, method: GET, req_id: 20fc99f7-bc7d-429d-ad31-d2d63c0c4b88, remote_addr: 127.0.0.1:42406, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.876 TRCE incoming request, uri: /newextent/48/db, method: GET, req_id: 1f84c39a-6e91-4723-bc73-16df8890dc59, remote_addr: 127.0.0.1:42406, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.876 INFO request completed, latency_us: 297, response_code: 200, uri: /newextent/48/db, method: GET, req_id: 1f84c39a-6e91-4723-bc73-16df8890dc59, remote_addr: 127.0.0.1:42406, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.877 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/030.copy" to "/tmp/downstairs-vrx8aK6L/00/000/030.replace"
Sep 22 23:15:10.877 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:10.878 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/030.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:10.878 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/030"
Sep 22 23:15:10.878 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/030.db"
Sep 22 23:15:10.878 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:10.878 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/030.replace" to "/tmp/downstairs-vrx8aK6L/00/000/030.completed"
Sep 22 23:15:10.878 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:10.878 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:10.879 DEBG [0] It's time to notify for 370
Sep 22 23:15:10.879 INFO Completion from [0] id:370 status:true
Sep 22 23:15:10.879 INFO [371/752] Repair commands completed
Sep 22 23:15:10.879 INFO Pop front: ReconcileIO { id: ReconciliationId(371), op: ExtentReopen { repair_id: ReconciliationId(371), extent_id: 48 }, state: ClientData([New, New, New]) }
Sep 22 23:15:10.879 INFO Sent repair work, now wait for resp
Sep 22 23:15:10.879 INFO [0] received reconcile message
Sep 22 23:15:10.879 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(371), op: ExtentReopen { repair_id: ReconciliationId(371), extent_id: 48 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:10.879 INFO [0] client ExtentReopen { repair_id: ReconciliationId(371), extent_id: 48 }
Sep 22 23:15:10.879 INFO [1] received reconcile message
Sep 22 23:15:10.879 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(371), op: ExtentReopen { repair_id: ReconciliationId(371), extent_id: 48 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:10.879 INFO [1] client ExtentReopen { repair_id: ReconciliationId(371), extent_id: 48 }
Sep 22 23:15:10.879 INFO [2] received reconcile message
Sep 22 23:15:10.879 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(371), op: ExtentReopen { repair_id: ReconciliationId(371), extent_id: 48 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:10.879 INFO [2] client ExtentReopen { repair_id: ReconciliationId(371), extent_id: 48 }
Sep 22 23:15:10.879 DEBG 371 Reopen extent 48
Sep 22 23:15:10.880 DEBG 371 Reopen extent 48
Sep 22 23:15:10.880 DEBG 371 Reopen extent 48
Sep 22 23:15:10.881 DEBG [2] It's time to notify for 371
Sep 22 23:15:10.881 INFO Completion from [2] id:371 status:true
Sep 22 23:15:10.881 INFO [372/752] Repair commands completed
Sep 22 23:15:10.881 INFO Pop front: ReconcileIO { id: ReconciliationId(372), op: ExtentFlush { repair_id: ReconciliationId(372), extent_id: 78, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:10.881 INFO Sent repair work, now wait for resp
Sep 22 23:15:10.881 INFO [0] received reconcile message
Sep 22 23:15:10.881 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(372), op: ExtentFlush { repair_id: ReconciliationId(372), extent_id: 78, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:10.881 INFO [0] client ExtentFlush { repair_id: ReconciliationId(372), extent_id: 78, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:10.881 INFO [1] received reconcile message
Sep 22 23:15:10.881 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(372), op: ExtentFlush { repair_id: ReconciliationId(372), extent_id: 78, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:10.881 INFO [1] client ExtentFlush { repair_id: ReconciliationId(372), extent_id: 78, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:10.881 INFO [2] received reconcile message
Sep 22 23:15:10.881 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(372), op: ExtentFlush { repair_id: ReconciliationId(372), extent_id: 78, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:10.881 INFO [2] client ExtentFlush { repair_id: ReconciliationId(372), extent_id: 78, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:10.881 DEBG 372 Flush extent 78 with f:2 g:2
Sep 22 23:15:10.881 DEBG Flush just extent 78 with f:2 and g:2
Sep 22 23:15:10.881 DEBG [1] It's time to notify for 372
Sep 22 23:15:10.882 INFO Completion from [1] id:372 status:true
Sep 22 23:15:10.882 INFO [373/752] Repair commands completed
Sep 22 23:15:10.882 INFO Pop front: ReconcileIO { id: ReconciliationId(373), op: ExtentClose { repair_id: ReconciliationId(373), extent_id: 78 }, state: ClientData([New, New, New]) }
Sep 22 23:15:10.882 INFO Sent repair work, now wait for resp
Sep 22 23:15:10.882 INFO [0] received reconcile message
Sep 22 23:15:10.882 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(373), op: ExtentClose { repair_id: ReconciliationId(373), extent_id: 78 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:10.882 INFO [0] client ExtentClose { repair_id: ReconciliationId(373), extent_id: 78 }
Sep 22 23:15:10.882 INFO [1] received reconcile message
Sep 22 23:15:10.882 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(373), op: ExtentClose { repair_id: ReconciliationId(373), extent_id: 78 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:10.882 INFO [1] client ExtentClose { repair_id: ReconciliationId(373), extent_id: 78 }
Sep 22 23:15:10.882 INFO [2] received reconcile message
Sep 22 23:15:10.882 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(373), op: ExtentClose { repair_id: ReconciliationId(373), extent_id: 78 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:10.882 INFO [2] client ExtentClose { repair_id: ReconciliationId(373), extent_id: 78 }
Sep 22 23:15:10.882 DEBG 373 Close extent 78
Sep 22 23:15:10.882 DEBG 373 Close extent 78
Sep 22 23:15:10.882 DEBG 373 Close extent 78
Sep 22 23:15:10.883 DEBG [2] It's time to notify for 373
Sep 22 23:15:10.883 INFO Completion from [2] id:373 status:true
Sep 22 23:15:10.883 INFO [374/752] Repair commands completed
Sep 22 23:15:10.883 INFO Pop front: ReconcileIO { id: ReconciliationId(374), op: ExtentRepair { repair_id: ReconciliationId(374), extent_id: 78, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:10.883 INFO Sent repair work, now wait for resp
Sep 22 23:15:10.883 INFO [0] received reconcile message
Sep 22 23:15:10.883 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(374), op: ExtentRepair { repair_id: ReconciliationId(374), extent_id: 78, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:10.883 INFO [0] client ExtentRepair { repair_id: ReconciliationId(374), extent_id: 78, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:10.883 INFO [0] Sending repair request ReconciliationId(374)
Sep 22 23:15:10.883 INFO [1] received reconcile message
Sep 22 23:15:10.883 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(374), op: ExtentRepair { repair_id: ReconciliationId(374), extent_id: 78, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:10.883 INFO [1] client ExtentRepair { repair_id: ReconciliationId(374), extent_id: 78, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:10.883 INFO [1] No action required ReconciliationId(374)
Sep 22 23:15:10.883 INFO [2] received reconcile message
Sep 22 23:15:10.883 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(374), op: ExtentRepair { repair_id: ReconciliationId(374), extent_id: 78, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:10.883 INFO [2] client ExtentRepair { repair_id: ReconciliationId(374), extent_id: 78, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:10.883 INFO [2] No action required ReconciliationId(374)
Sep 22 23:15:10.883 DEBG 374 Repair extent 78 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:10.883 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/04E.copy"
Sep 22 23:15:10.939 DEBG [1] Read AckReady 1082, : downstairs
Sep 22 23:15:10.940 DEBG up_ds_listen was notified
Sep 22 23:15:10.940 DEBG up_ds_listen process 1082
Sep 22 23:15:10.940 DEBG [A] ack job 1082:83, : downstairs
Sep 22 23:15:10.946 INFO accepted connection, remote_addr: 127.0.0.1:51487, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.946 TRCE incoming request, uri: /extent/78/files, method: GET, req_id: e1d1a172-ca0b-4280-8698-76786dcc073c, remote_addr: 127.0.0.1:51487, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.946 INFO request completed, latency_us: 214, response_code: 200, uri: /extent/78/files, method: GET, req_id: e1d1a172-ca0b-4280-8698-76786dcc073c, remote_addr: 127.0.0.1:51487, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.947 INFO eid:78 Found repair files: ["04E", "04E.db"]
Sep 22 23:15:10.947 TRCE incoming request, uri: /newextent/78/data, method: GET, req_id: f7688a5e-efa4-4398-84ff-80ee6bf08bd8, remote_addr: 127.0.0.1:51487, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.947 INFO request completed, latency_us: 329, response_code: 200, uri: /newextent/78/data, method: GET, req_id: f7688a5e-efa4-4398-84ff-80ee6bf08bd8, remote_addr: 127.0.0.1:51487, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.952 TRCE incoming request, uri: /newextent/78/db, method: GET, req_id: 887a2791-c791-4d39-b2da-0354100cecd8, remote_addr: 127.0.0.1:51487, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.952 INFO request completed, latency_us: 291, response_code: 200, uri: /newextent/78/db, method: GET, req_id: 887a2791-c791-4d39-b2da-0354100cecd8, remote_addr: 127.0.0.1:51487, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:10.954 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/04E.copy" to "/tmp/downstairs-vrx8aK6L/00/000/04E.replace"
Sep 22 23:15:10.954 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:10.954 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/04E.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:10.955 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/04E"
Sep 22 23:15:10.955 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/04E.db"
Sep 22 23:15:10.955 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:10.955 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/04E.replace" to "/tmp/downstairs-vrx8aK6L/00/000/04E.completed"
Sep 22 23:15:10.955 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:10.955 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:10.955 DEBG [0] It's time to notify for 374
Sep 22 23:15:10.955 INFO Completion from [0] id:374 status:true
Sep 22 23:15:10.955 INFO [375/752] Repair commands completed
Sep 22 23:15:10.955 INFO Pop front: ReconcileIO { id: ReconciliationId(375), op: ExtentReopen { repair_id: ReconciliationId(375), extent_id: 78 }, state: ClientData([New, New, New]) }
Sep 22 23:15:10.955 INFO Sent repair work, now wait for resp
Sep 22 23:15:10.955 INFO [0] received reconcile message
Sep 22 23:15:10.955 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(375), op: ExtentReopen { repair_id: ReconciliationId(375), extent_id: 78 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:10.955 INFO [0] client ExtentReopen { repair_id: ReconciliationId(375), extent_id: 78 }
Sep 22 23:15:10.955 INFO [1] received reconcile message
Sep 22 23:15:10.955 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(375), op: ExtentReopen { repair_id: ReconciliationId(375), extent_id: 78 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:10.956 INFO [1] client ExtentReopen { repair_id: ReconciliationId(375), extent_id: 78 }
Sep 22 23:15:10.956 INFO [2] received reconcile message
Sep 22 23:15:10.956 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(375), op: ExtentReopen { repair_id: ReconciliationId(375), extent_id: 78 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:10.956 INFO [2] client ExtentReopen { repair_id: ReconciliationId(375), extent_id: 78 }
Sep 22 23:15:10.956 DEBG 375 Reopen extent 78
Sep 22 23:15:10.956 DEBG 375 Reopen extent 78
Sep 22 23:15:10.957 DEBG 375 Reopen extent 78
Sep 22 23:15:10.957 DEBG [2] It's time to notify for 375
Sep 22 23:15:10.958 INFO Completion from [2] id:375 status:true
Sep 22 23:15:10.958 INFO [376/752] Repair commands completed
Sep 22 23:15:10.958 INFO Pop front: ReconcileIO { id: ReconciliationId(376), op: ExtentFlush { repair_id: ReconciliationId(376), extent_id: 88, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:10.958 INFO Sent repair work, now wait for resp
Sep 22 23:15:10.958 INFO [0] received reconcile message
Sep 22 23:15:10.958 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(376), op: ExtentFlush { repair_id: ReconciliationId(376), extent_id: 88, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:10.958 INFO [0] client ExtentFlush { repair_id: ReconciliationId(376), extent_id: 88, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:10.958 INFO [1] received reconcile message
Sep 22 23:15:10.958 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(376), op: ExtentFlush { repair_id: ReconciliationId(376), extent_id: 88, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:10.958 INFO [1] client ExtentFlush { repair_id: ReconciliationId(376), extent_id: 88, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:10.958 INFO [2] received reconcile message
Sep 22 23:15:10.958 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(376), op: ExtentFlush { repair_id: ReconciliationId(376), extent_id: 88, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:10.958 INFO [2] client ExtentFlush { repair_id: ReconciliationId(376), extent_id: 88, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:10.958 DEBG 376 Flush extent 88 with f:2 g:2
Sep 22 23:15:10.958 DEBG Flush just extent 88 with f:2 and g:2
Sep 22 23:15:10.958 DEBG [1] It's time to notify for 376
Sep 22 23:15:10.958 INFO Completion from [1] id:376 status:true
Sep 22 23:15:10.958 INFO [377/752] Repair commands completed
Sep 22 23:15:10.958 INFO Pop front: ReconcileIO { id: ReconciliationId(377), op: ExtentClose { repair_id: ReconciliationId(377), extent_id: 88 }, state: ClientData([New, New, New]) }
Sep 22 23:15:10.958 INFO Sent repair work, now wait for resp
Sep 22 23:15:10.958 INFO [0] received reconcile message
Sep 22 23:15:10.958 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(377), op: ExtentClose { repair_id: ReconciliationId(377), extent_id: 88 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:10.958 INFO [0] client ExtentClose { repair_id: ReconciliationId(377), extent_id: 88 }
Sep 22 23:15:10.958 INFO [1] received reconcile message
Sep 22 23:15:10.958 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(377), op: ExtentClose { repair_id: ReconciliationId(377), extent_id: 88 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:10.958 INFO [1] client ExtentClose { repair_id: ReconciliationId(377), extent_id: 88 }
Sep 22 23:15:10.958 INFO [2] received reconcile message
Sep 22 23:15:10.958 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(377), op: ExtentClose { repair_id: ReconciliationId(377), extent_id: 88 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:10.958 INFO [2] client ExtentClose { repair_id: ReconciliationId(377), extent_id: 88 }
Sep 22 23:15:10.959 DEBG 377 Close extent 88
Sep 22 23:15:10.959 DEBG 377 Close extent 88
Sep 22 23:15:10.959 DEBG 377 Close extent 88
Sep 22 23:15:10.959 DEBG [2] It's time to notify for 377
Sep 22 23:15:10.959 INFO Completion from [2] id:377 status:true
Sep 22 23:15:10.960 INFO [378/752] Repair commands completed
Sep 22 23:15:10.960 INFO Pop front: ReconcileIO { id: ReconciliationId(378), op: ExtentRepair { repair_id: ReconciliationId(378), extent_id: 88, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:10.960 INFO Sent repair work, now wait for resp
Sep 22 23:15:10.960 INFO [0] received reconcile message
Sep 22 23:15:10.960 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(378), op: ExtentRepair { repair_id: ReconciliationId(378), extent_id: 88, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:10.960 INFO [0] client ExtentRepair { repair_id: ReconciliationId(378), extent_id: 88, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:10.960 INFO [0] Sending repair request ReconciliationId(378)
Sep 22 23:15:10.960 INFO [1] received reconcile message
Sep 22 23:15:10.960 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(378), op: ExtentRepair { repair_id: ReconciliationId(378), extent_id: 88, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:10.960 INFO [1] client ExtentRepair { repair_id: ReconciliationId(378), extent_id: 88, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:10.960 INFO [1] No action required ReconciliationId(378)
Sep 22 23:15:10.960 INFO [2] received reconcile message
Sep 22 23:15:10.960 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(378), op: ExtentRepair { repair_id: ReconciliationId(378), extent_id: 88, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:10.960 INFO [2] client ExtentRepair { repair_id: ReconciliationId(378), extent_id: 88, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:10.960 INFO [2] No action required ReconciliationId(378)
Sep 22 23:15:10.960 DEBG 378 Repair extent 88 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:10.960 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/058.copy"
Sep 22 23:15:10.993 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:11.025 INFO accepted connection, remote_addr: 127.0.0.1:56140, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.026 TRCE incoming request, uri: /extent/88/files, method: GET, req_id: 3e9ff68a-6866-4f07-99fa-9bc4ecbb0071, remote_addr: 127.0.0.1:56140, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.026 INFO request completed, latency_us: 262, response_code: 200, uri: /extent/88/files, method: GET, req_id: 3e9ff68a-6866-4f07-99fa-9bc4ecbb0071, remote_addr: 127.0.0.1:56140, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.026 INFO eid:88 Found repair files: ["058", "058.db"]
Sep 22 23:15:11.027 TRCE incoming request, uri: /newextent/88/data, method: GET, req_id: 8ba18d7d-ab82-4522-8f19-171ca18ab892, remote_addr: 127.0.0.1:56140, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.027 INFO request completed, latency_us: 342, response_code: 200, uri: /newextent/88/data, method: GET, req_id: 8ba18d7d-ab82-4522-8f19-171ca18ab892, remote_addr: 127.0.0.1:56140, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.032 TRCE incoming request, uri: /newextent/88/db, method: GET, req_id: 2f697082-c02d-4298-8fdb-3a4fd5e21750, remote_addr: 127.0.0.1:56140, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.032 INFO request completed, latency_us: 294, response_code: 200, uri: /newextent/88/db, method: GET, req_id: 2f697082-c02d-4298-8fdb-3a4fd5e21750, remote_addr: 127.0.0.1:56140, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.033 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/058.copy" to "/tmp/downstairs-vrx8aK6L/00/000/058.replace"
Sep 22 23:15:11.034 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:11.035 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/058.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:11.035 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/058"
Sep 22 23:15:11.035 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/058.db"
Sep 22 23:15:11.035 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:11.035 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/058.replace" to "/tmp/downstairs-vrx8aK6L/00/000/058.completed"
Sep 22 23:15:11.035 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:11.035 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:11.035 DEBG [0] It's time to notify for 378
Sep 22 23:15:11.035 INFO Completion from [0] id:378 status:true
Sep 22 23:15:11.035 INFO [379/752] Repair commands completed
Sep 22 23:15:11.035 INFO Pop front: ReconcileIO { id: ReconciliationId(379), op: ExtentReopen { repair_id: ReconciliationId(379), extent_id: 88 }, state: ClientData([New, New, New]) }
Sep 22 23:15:11.036 INFO Sent repair work, now wait for resp
Sep 22 23:15:11.036 INFO [0] received reconcile message
Sep 22 23:15:11.036 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(379), op: ExtentReopen { repair_id: ReconciliationId(379), extent_id: 88 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:11.036 INFO [0] client ExtentReopen { repair_id: ReconciliationId(379), extent_id: 88 }
Sep 22 23:15:11.036 INFO [1] received reconcile message
Sep 22 23:15:11.036 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(379), op: ExtentReopen { repair_id: ReconciliationId(379), extent_id: 88 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:11.036 INFO [1] client ExtentReopen { repair_id: ReconciliationId(379), extent_id: 88 }
Sep 22 23:15:11.036 INFO [2] received reconcile message
Sep 22 23:15:11.036 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(379), op: ExtentReopen { repair_id: ReconciliationId(379), extent_id: 88 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:11.036 INFO [2] client ExtentReopen { repair_id: ReconciliationId(379), extent_id: 88 }
Sep 22 23:15:11.036 DEBG 379 Reopen extent 88
Sep 22 23:15:11.037 DEBG 379 Reopen extent 88
Sep 22 23:15:11.037 DEBG 379 Reopen extent 88
Sep 22 23:15:11.038 DEBG [2] It's time to notify for 379
Sep 22 23:15:11.038 INFO Completion from [2] id:379 status:true
Sep 22 23:15:11.038 INFO [380/752] Repair commands completed
Sep 22 23:15:11.038 INFO Pop front: ReconcileIO { id: ReconciliationId(380), op: ExtentFlush { repair_id: ReconciliationId(380), extent_id: 82, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:11.038 INFO Sent repair work, now wait for resp
Sep 22 23:15:11.038 INFO [0] received reconcile message
Sep 22 23:15:11.038 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(380), op: ExtentFlush { repair_id: ReconciliationId(380), extent_id: 82, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:11.038 INFO [0] client ExtentFlush { repair_id: ReconciliationId(380), extent_id: 82, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:11.038 INFO [1] received reconcile message
Sep 22 23:15:11.038 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(380), op: ExtentFlush { repair_id: ReconciliationId(380), extent_id: 82, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:11.038 INFO [1] client ExtentFlush { repair_id: ReconciliationId(380), extent_id: 82, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:11.038 INFO [2] received reconcile message
Sep 22 23:15:11.038 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(380), op: ExtentFlush { repair_id: ReconciliationId(380), extent_id: 82, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:11.038 INFO [2] client ExtentFlush { repair_id: ReconciliationId(380), extent_id: 82, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:11.038 DEBG 380 Flush extent 82 with f:2 g:2
Sep 22 23:15:11.038 DEBG Flush just extent 82 with f:2 and g:2
Sep 22 23:15:11.038 DEBG [1] It's time to notify for 380
Sep 22 23:15:11.038 INFO Completion from [1] id:380 status:true
Sep 22 23:15:11.038 INFO [381/752] Repair commands completed
Sep 22 23:15:11.038 INFO Pop front: ReconcileIO { id: ReconciliationId(381), op: ExtentClose { repair_id: ReconciliationId(381), extent_id: 82 }, state: ClientData([New, New, New]) }
Sep 22 23:15:11.039 INFO Sent repair work, now wait for resp
Sep 22 23:15:11.039 INFO [0] received reconcile message
Sep 22 23:15:11.039 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(381), op: ExtentClose { repair_id: ReconciliationId(381), extent_id: 82 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:11.039 INFO [0] client ExtentClose { repair_id: ReconciliationId(381), extent_id: 82 }
Sep 22 23:15:11.039 INFO [1] received reconcile message
Sep 22 23:15:11.039 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(381), op: ExtentClose { repair_id: ReconciliationId(381), extent_id: 82 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:11.039 INFO [1] client ExtentClose { repair_id: ReconciliationId(381), extent_id: 82 }
Sep 22 23:15:11.039 INFO [2] received reconcile message
Sep 22 23:15:11.039 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(381), op: ExtentClose { repair_id: ReconciliationId(381), extent_id: 82 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:11.039 INFO [2] client ExtentClose { repair_id: ReconciliationId(381), extent_id: 82 }
Sep 22 23:15:11.039 DEBG 381 Close extent 82
Sep 22 23:15:11.039 DEBG 381 Close extent 82
Sep 22 23:15:11.039 DEBG 381 Close extent 82
Sep 22 23:15:11.040 DEBG [2] It's time to notify for 381
Sep 22 23:15:11.040 INFO Completion from [2] id:381 status:true
Sep 22 23:15:11.040 INFO [382/752] Repair commands completed
Sep 22 23:15:11.040 INFO Pop front: ReconcileIO { id: ReconciliationId(382), op: ExtentRepair { repair_id: ReconciliationId(382), extent_id: 82, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:11.040 INFO Sent repair work, now wait for resp
Sep 22 23:15:11.040 INFO [0] received reconcile message
Sep 22 23:15:11.040 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(382), op: ExtentRepair { repair_id: ReconciliationId(382), extent_id: 82, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:11.040 INFO [0] client ExtentRepair { repair_id: ReconciliationId(382), extent_id: 82, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:11.040 INFO [0] Sending repair request ReconciliationId(382)
Sep 22 23:15:11.040 INFO [1] received reconcile message
Sep 22 23:15:11.040 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(382), op: ExtentRepair { repair_id: ReconciliationId(382), extent_id: 82, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:11.040 INFO [1] client ExtentRepair { repair_id: ReconciliationId(382), extent_id: 82, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:11.040 INFO [1] No action required ReconciliationId(382)
Sep 22 23:15:11.040 INFO [2] received reconcile message
Sep 22 23:15:11.040 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(382), op: ExtentRepair { repair_id: ReconciliationId(382), extent_id: 82, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:11.040 INFO [2] client ExtentRepair { repair_id: ReconciliationId(382), extent_id: 82, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:11.040 INFO [2] No action required ReconciliationId(382)
Sep 22 23:15:11.040 DEBG 382 Repair extent 82 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:11.040 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/052.copy"
Sep 22 23:15:11.103 INFO accepted connection, remote_addr: 127.0.0.1:43157, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.103 TRCE incoming request, uri: /extent/82/files, method: GET, req_id: 29a729e2-51fa-47c7-bac6-926c59854eda, remote_addr: 127.0.0.1:43157, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.103 INFO request completed, latency_us: 213, response_code: 200, uri: /extent/82/files, method: GET, req_id: 29a729e2-51fa-47c7-bac6-926c59854eda, remote_addr: 127.0.0.1:43157, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.104 INFO eid:82 Found repair files: ["052", "052.db"]
Sep 22 23:15:11.104 TRCE incoming request, uri: /newextent/82/data, method: GET, req_id: 0cc2d882-1c0d-4d89-a4a2-aa63b291d923, remote_addr: 127.0.0.1:43157, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.104 INFO request completed, latency_us: 337, response_code: 200, uri: /newextent/82/data, method: GET, req_id: 0cc2d882-1c0d-4d89-a4a2-aa63b291d923, remote_addr: 127.0.0.1:43157, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.109 TRCE incoming request, uri: /newextent/82/db, method: GET, req_id: e8d9fe7f-f633-4461-ac56-fec2a6ac9a82, remote_addr: 127.0.0.1:43157, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.110 INFO request completed, latency_us: 292, response_code: 200, uri: /newextent/82/db, method: GET, req_id: e8d9fe7f-f633-4461-ac56-fec2a6ac9a82, remote_addr: 127.0.0.1:43157, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.111 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/052.copy" to "/tmp/downstairs-vrx8aK6L/00/000/052.replace"
Sep 22 23:15:11.111 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:11.112 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/052.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:11.112 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/052"
Sep 22 23:15:11.112 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/052.db"
Sep 22 23:15:11.112 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:11.112 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/052.replace" to "/tmp/downstairs-vrx8aK6L/00/000/052.completed"
Sep 22 23:15:11.112 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:11.112 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:11.112 DEBG [0] It's time to notify for 382
Sep 22 23:15:11.112 INFO Completion from [0] id:382 status:true
Sep 22 23:15:11.112 INFO [383/752] Repair commands completed
Sep 22 23:15:11.113 INFO Pop front: ReconcileIO { id: ReconciliationId(383), op: ExtentReopen { repair_id: ReconciliationId(383), extent_id: 82 }, state: ClientData([New, New, New]) }
Sep 22 23:15:11.113 INFO Sent repair work, now wait for resp
Sep 22 23:15:11.113 INFO [0] received reconcile message
Sep 22 23:15:11.113 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(383), op: ExtentReopen { repair_id: ReconciliationId(383), extent_id: 82 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:11.113 INFO [0] client ExtentReopen { repair_id: ReconciliationId(383), extent_id: 82 }
Sep 22 23:15:11.113 INFO [1] received reconcile message
Sep 22 23:15:11.113 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(383), op: ExtentReopen { repair_id: ReconciliationId(383), extent_id: 82 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:11.113 INFO [1] client ExtentReopen { repair_id: ReconciliationId(383), extent_id: 82 }
Sep 22 23:15:11.113 INFO [2] received reconcile message
Sep 22 23:15:11.113 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(383), op: ExtentReopen { repair_id: ReconciliationId(383), extent_id: 82 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:11.113 INFO [2] client ExtentReopen { repair_id: ReconciliationId(383), extent_id: 82 }
Sep 22 23:15:11.113 DEBG 383 Reopen extent 82
Sep 22 23:15:11.114 DEBG 383 Reopen extent 82
Sep 22 23:15:11.114 DEBG 383 Reopen extent 82
Sep 22 23:15:11.115 DEBG [2] It's time to notify for 383
Sep 22 23:15:11.115 INFO Completion from [2] id:383 status:true
Sep 22 23:15:11.115 INFO [384/752] Repair commands completed
Sep 22 23:15:11.115 INFO Pop front: ReconcileIO { id: ReconciliationId(384), op: ExtentFlush { repair_id: ReconciliationId(384), extent_id: 86, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:11.115 INFO Sent repair work, now wait for resp
Sep 22 23:15:11.115 INFO [0] received reconcile message
Sep 22 23:15:11.115 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(384), op: ExtentFlush { repair_id: ReconciliationId(384), extent_id: 86, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:11.115 INFO [0] client ExtentFlush { repair_id: ReconciliationId(384), extent_id: 86, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:11.115 INFO [1] received reconcile message
Sep 22 23:15:11.115 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(384), op: ExtentFlush { repair_id: ReconciliationId(384), extent_id: 86, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:11.115 INFO [1] client ExtentFlush { repair_id: ReconciliationId(384), extent_id: 86, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:11.115 INFO [2] received reconcile message
Sep 22 23:15:11.115 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(384), op: ExtentFlush { repair_id: ReconciliationId(384), extent_id: 86, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:11.115 INFO [2] client ExtentFlush { repair_id: ReconciliationId(384), extent_id: 86, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:11.115 DEBG 384 Flush extent 86 with f:2 g:2
Sep 22 23:15:11.115 DEBG Flush just extent 86 with f:2 and g:2
Sep 22 23:15:11.115 DEBG [1] It's time to notify for 384
Sep 22 23:15:11.115 INFO Completion from [1] id:384 status:true
Sep 22 23:15:11.115 INFO [385/752] Repair commands completed
Sep 22 23:15:11.115 INFO Pop front: ReconcileIO { id: ReconciliationId(385), op: ExtentClose { repair_id: ReconciliationId(385), extent_id: 86 }, state: ClientData([New, New, New]) }
Sep 22 23:15:11.115 INFO Sent repair work, now wait for resp
Sep 22 23:15:11.115 INFO [0] received reconcile message
Sep 22 23:15:11.115 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(385), op: ExtentClose { repair_id: ReconciliationId(385), extent_id: 86 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:11.115 INFO [0] client ExtentClose { repair_id: ReconciliationId(385), extent_id: 86 }
Sep 22 23:15:11.115 INFO [1] received reconcile message
Sep 22 23:15:11.115 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(385), op: ExtentClose { repair_id: ReconciliationId(385), extent_id: 86 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:11.115 INFO [1] client ExtentClose { repair_id: ReconciliationId(385), extent_id: 86 }
Sep 22 23:15:11.115 INFO [2] received reconcile message
Sep 22 23:15:11.116 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(385), op: ExtentClose { repair_id: ReconciliationId(385), extent_id: 86 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:11.116 INFO [2] client ExtentClose { repair_id: ReconciliationId(385), extent_id: 86 }
Sep 22 23:15:11.116 DEBG 385 Close extent 86
Sep 22 23:15:11.116 DEBG 385 Close extent 86
Sep 22 23:15:11.116 DEBG 385 Close extent 86
Sep 22 23:15:11.117 DEBG [2] It's time to notify for 385
Sep 22 23:15:11.117 INFO Completion from [2] id:385 status:true
Sep 22 23:15:11.117 INFO [386/752] Repair commands completed
Sep 22 23:15:11.117 INFO Pop front: ReconcileIO { id: ReconciliationId(386), op: ExtentRepair { repair_id: ReconciliationId(386), extent_id: 86, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:11.117 INFO Sent repair work, now wait for resp
Sep 22 23:15:11.117 INFO [0] received reconcile message
Sep 22 23:15:11.117 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(386), op: ExtentRepair { repair_id: ReconciliationId(386), extent_id: 86, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:11.117 INFO [0] client ExtentRepair { repair_id: ReconciliationId(386), extent_id: 86, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:11.117 INFO [0] Sending repair request ReconciliationId(386)
Sep 22 23:15:11.117 INFO [1] received reconcile message
Sep 22 23:15:11.117 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(386), op: ExtentRepair { repair_id: ReconciliationId(386), extent_id: 86, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:11.117 INFO [1] client ExtentRepair { repair_id: ReconciliationId(386), extent_id: 86, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:11.117 INFO [1] No action required ReconciliationId(386)
Sep 22 23:15:11.117 INFO [2] received reconcile message
Sep 22 23:15:11.117 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(386), op: ExtentRepair { repair_id: ReconciliationId(386), extent_id: 86, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:11.117 INFO [2] client ExtentRepair { repair_id: ReconciliationId(386), extent_id: 86, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:11.117 INFO [2] No action required ReconciliationId(386)
Sep 22 23:15:11.117 DEBG 386 Repair extent 86 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:11.117 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/056.copy"
Sep 22 23:15:11.181 INFO accepted connection, remote_addr: 127.0.0.1:49281, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.181 TRCE incoming request, uri: /extent/86/files, method: GET, req_id: 28b29fe2-23ca-4872-95e1-b6cd82a70413, remote_addr: 127.0.0.1:49281, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.181 INFO request completed, latency_us: 204, response_code: 200, uri: /extent/86/files, method: GET, req_id: 28b29fe2-23ca-4872-95e1-b6cd82a70413, remote_addr: 127.0.0.1:49281, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.182 INFO eid:86 Found repair files: ["056", "056.db"]
Sep 22 23:15:11.182 TRCE incoming request, uri: /newextent/86/data, method: GET, req_id: e22ad7ab-0baf-4d2f-8f29-0c5c4eb3c278, remote_addr: 127.0.0.1:49281, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.182 INFO request completed, latency_us: 251, response_code: 200, uri: /newextent/86/data, method: GET, req_id: e22ad7ab-0baf-4d2f-8f29-0c5c4eb3c278, remote_addr: 127.0.0.1:49281, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.187 TRCE incoming request, uri: /newextent/86/db, method: GET, req_id: f123ad6f-ec93-45af-ac28-ceb1400879f2, remote_addr: 127.0.0.1:49281, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.187 INFO request completed, latency_us: 295, response_code: 200, uri: /newextent/86/db, method: GET, req_id: f123ad6f-ec93-45af-ac28-ceb1400879f2, remote_addr: 127.0.0.1:49281, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.189 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/056.copy" to "/tmp/downstairs-vrx8aK6L/00/000/056.replace"
Sep 22 23:15:11.189 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:11.189 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/056.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:11.190 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/056"
Sep 22 23:15:11.190 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/056.db"
Sep 22 23:15:11.190 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:11.190 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/056.replace" to "/tmp/downstairs-vrx8aK6L/00/000/056.completed"
Sep 22 23:15:11.190 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:11.190 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:11.190 DEBG [0] It's time to notify for 386
Sep 22 23:15:11.190 INFO Completion from [0] id:386 status:true
Sep 22 23:15:11.190 INFO [387/752] Repair commands completed
Sep 22 23:15:11.190 INFO Pop front: ReconcileIO { id: ReconciliationId(387), op: ExtentReopen { repair_id: ReconciliationId(387), extent_id: 86 }, state: ClientData([New, New, New]) }
Sep 22 23:15:11.190 INFO Sent repair work, now wait for resp
Sep 22 23:15:11.190 INFO [0] received reconcile message
Sep 22 23:15:11.190 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(387), op: ExtentReopen { repair_id: ReconciliationId(387), extent_id: 86 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:11.190 INFO [0] client ExtentReopen { repair_id: ReconciliationId(387), extent_id: 86 }
Sep 22 23:15:11.190 INFO [1] received reconcile message
Sep 22 23:15:11.190 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(387), op: ExtentReopen { repair_id: ReconciliationId(387), extent_id: 86 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:11.190 INFO [1] client ExtentReopen { repair_id: ReconciliationId(387), extent_id: 86 }
Sep 22 23:15:11.191 INFO [2] received reconcile message
Sep 22 23:15:11.191 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(387), op: ExtentReopen { repair_id: ReconciliationId(387), extent_id: 86 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:11.191 INFO [2] client ExtentReopen { repair_id: ReconciliationId(387), extent_id: 86 }
Sep 22 23:15:11.191 DEBG 387 Reopen extent 86
Sep 22 23:15:11.191 DEBG 387 Reopen extent 86
Sep 22 23:15:11.192 DEBG 387 Reopen extent 86
Sep 22 23:15:11.192 DEBG [2] It's time to notify for 387
Sep 22 23:15:11.192 INFO Completion from [2] id:387 status:true
Sep 22 23:15:11.192 INFO [388/752] Repair commands completed
Sep 22 23:15:11.192 INFO Pop front: ReconcileIO { id: ReconciliationId(388), op: ExtentFlush { repair_id: ReconciliationId(388), extent_id: 94, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:11.192 INFO Sent repair work, now wait for resp
Sep 22 23:15:11.193 INFO [0] received reconcile message
Sep 22 23:15:11.193 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(388), op: ExtentFlush { repair_id: ReconciliationId(388), extent_id: 94, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:11.193 INFO [0] client ExtentFlush { repair_id: ReconciliationId(388), extent_id: 94, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:11.193 INFO [1] received reconcile message
Sep 22 23:15:11.193 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(388), op: ExtentFlush { repair_id: ReconciliationId(388), extent_id: 94, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:11.193 INFO [1] client ExtentFlush { repair_id: ReconciliationId(388), extent_id: 94, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:11.193 INFO [2] received reconcile message
Sep 22 23:15:11.193 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(388), op: ExtentFlush { repair_id: ReconciliationId(388), extent_id: 94, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:11.193 INFO [2] client ExtentFlush { repair_id: ReconciliationId(388), extent_id: 94, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:11.193 DEBG 388 Flush extent 94 with f:2 g:2
Sep 22 23:15:11.193 DEBG Flush just extent 94 with f:2 and g:2
Sep 22 23:15:11.193 DEBG [1] It's time to notify for 388
Sep 22 23:15:11.193 INFO Completion from [1] id:388 status:true
Sep 22 23:15:11.193 INFO [389/752] Repair commands completed
Sep 22 23:15:11.193 INFO Pop front: ReconcileIO { id: ReconciliationId(389), op: ExtentClose { repair_id: ReconciliationId(389), extent_id: 94 }, state: ClientData([New, New, New]) }
Sep 22 23:15:11.193 INFO Sent repair work, now wait for resp
Sep 22 23:15:11.193 INFO [0] received reconcile message
Sep 22 23:15:11.193 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(389), op: ExtentClose { repair_id: ReconciliationId(389), extent_id: 94 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:11.193 INFO [0] client ExtentClose { repair_id: ReconciliationId(389), extent_id: 94 }
Sep 22 23:15:11.193 INFO [1] received reconcile message
Sep 22 23:15:11.193 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(389), op: ExtentClose { repair_id: ReconciliationId(389), extent_id: 94 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:11.193 INFO [1] client ExtentClose { repair_id: ReconciliationId(389), extent_id: 94 }
Sep 22 23:15:11.193 INFO [2] received reconcile message
Sep 22 23:15:11.193 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(389), op: ExtentClose { repair_id: ReconciliationId(389), extent_id: 94 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:11.193 INFO [2] client ExtentClose { repair_id: ReconciliationId(389), extent_id: 94 }
Sep 22 23:15:11.193 DEBG 389 Close extent 94
Sep 22 23:15:11.194 DEBG 389 Close extent 94
Sep 22 23:15:11.194 DEBG 389 Close extent 94
Sep 22 23:15:11.194 DEBG [2] It's time to notify for 389
Sep 22 23:15:11.194 INFO Completion from [2] id:389 status:true
Sep 22 23:15:11.194 INFO [390/752] Repair commands completed
Sep 22 23:15:11.194 INFO Pop front: ReconcileIO { id: ReconciliationId(390), op: ExtentRepair { repair_id: ReconciliationId(390), extent_id: 94, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:11.194 INFO Sent repair work, now wait for resp
Sep 22 23:15:11.194 INFO [0] received reconcile message
Sep 22 23:15:11.194 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(390), op: ExtentRepair { repair_id: ReconciliationId(390), extent_id: 94, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:11.195 INFO [0] client ExtentRepair { repair_id: ReconciliationId(390), extent_id: 94, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:11.195 INFO [0] Sending repair request ReconciliationId(390)
Sep 22 23:15:11.195 INFO [1] received reconcile message
Sep 22 23:15:11.195 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(390), op: ExtentRepair { repair_id: ReconciliationId(390), extent_id: 94, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:11.195 INFO [1] client ExtentRepair { repair_id: ReconciliationId(390), extent_id: 94, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:11.195 INFO [1] No action required ReconciliationId(390)
Sep 22 23:15:11.195 INFO [2] received reconcile message
Sep 22 23:15:11.195 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(390), op: ExtentRepair { repair_id: ReconciliationId(390), extent_id: 94, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:11.195 INFO [2] client ExtentRepair { repair_id: ReconciliationId(390), extent_id: 94, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:11.195 INFO [2] No action required ReconciliationId(390)
Sep 22 23:15:11.195 DEBG 390 Repair extent 94 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:11.195 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/05E.copy"
Sep 22 23:15:11.258 INFO accepted connection, remote_addr: 127.0.0.1:52448, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.259 TRCE incoming request, uri: /extent/94/files, method: GET, req_id: 69627f5d-a5af-4826-95fc-a878ab23d9d5, remote_addr: 127.0.0.1:52448, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.259 INFO request completed, latency_us: 193, response_code: 200, uri: /extent/94/files, method: GET, req_id: 69627f5d-a5af-4826-95fc-a878ab23d9d5, remote_addr: 127.0.0.1:52448, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.259 INFO eid:94 Found repair files: ["05E", "05E.db"]
Sep 22 23:15:11.259 TRCE incoming request, uri: /newextent/94/data, method: GET, req_id: 5bfb62ea-9451-4ec0-a544-a54522c10853, remote_addr: 127.0.0.1:52448, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.260 INFO request completed, latency_us: 301, response_code: 200, uri: /newextent/94/data, method: GET, req_id: 5bfb62ea-9451-4ec0-a544-a54522c10853, remote_addr: 127.0.0.1:52448, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.265 TRCE incoming request, uri: /newextent/94/db, method: GET, req_id: 61f60823-9fd7-4462-a4df-3bc313fc8e6c, remote_addr: 127.0.0.1:52448, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.265 INFO request completed, latency_us: 293, response_code: 200, uri: /newextent/94/db, method: GET, req_id: 61f60823-9fd7-4462-a4df-3bc313fc8e6c, remote_addr: 127.0.0.1:52448, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.266 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/05E.copy" to "/tmp/downstairs-vrx8aK6L/00/000/05E.replace"
Sep 22 23:15:11.266 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:11.267 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/05E.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:11.267 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/05E"
Sep 22 23:15:11.267 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/05E.db"
Sep 22 23:15:11.267 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:11.267 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/05E.replace" to "/tmp/downstairs-vrx8aK6L/00/000/05E.completed"
Sep 22 23:15:11.267 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:11.267 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:11.267 DEBG [0] It's time to notify for 390
Sep 22 23:15:11.268 INFO Completion from [0] id:390 status:true
Sep 22 23:15:11.268 INFO [391/752] Repair commands completed
Sep 22 23:15:11.268 INFO Pop front: ReconcileIO { id: ReconciliationId(391), op: ExtentReopen { repair_id: ReconciliationId(391), extent_id: 94 }, state: ClientData([New, New, New]) }
Sep 22 23:15:11.268 INFO Sent repair work, now wait for resp
Sep 22 23:15:11.268 INFO [0] received reconcile message
Sep 22 23:15:11.268 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(391), op: ExtentReopen { repair_id: ReconciliationId(391), extent_id: 94 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:11.268 INFO [0] client ExtentReopen { repair_id: ReconciliationId(391), extent_id: 94 }
Sep 22 23:15:11.268 INFO [1] received reconcile message
Sep 22 23:15:11.268 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(391), op: ExtentReopen { repair_id: ReconciliationId(391), extent_id: 94 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:11.268 INFO [1] client ExtentReopen { repair_id: ReconciliationId(391), extent_id: 94 }
Sep 22 23:15:11.268 INFO [2] received reconcile message
Sep 22 23:15:11.268 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(391), op: ExtentReopen { repair_id: ReconciliationId(391), extent_id: 94 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:11.268 INFO [2] client ExtentReopen { repair_id: ReconciliationId(391), extent_id: 94 }
Sep 22 23:15:11.268 DEBG 391 Reopen extent 94
Sep 22 23:15:11.269 DEBG 391 Reopen extent 94
Sep 22 23:15:11.269 DEBG 391 Reopen extent 94
Sep 22 23:15:11.270 DEBG [2] It's time to notify for 391
Sep 22 23:15:11.270 INFO Completion from [2] id:391 status:true
Sep 22 23:15:11.270 INFO [392/752] Repair commands completed
Sep 22 23:15:11.270 INFO Pop front: ReconcileIO { id: ReconciliationId(392), op: ExtentFlush { repair_id: ReconciliationId(392), extent_id: 145, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:11.270 INFO Sent repair work, now wait for resp
Sep 22 23:15:11.270 INFO [0] received reconcile message
Sep 22 23:15:11.270 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(392), op: ExtentFlush { repair_id: ReconciliationId(392), extent_id: 145, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:11.270 INFO [0] client ExtentFlush { repair_id: ReconciliationId(392), extent_id: 145, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:11.270 INFO [1] received reconcile message
Sep 22 23:15:11.270 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(392), op: ExtentFlush { repair_id: ReconciliationId(392), extent_id: 145, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:11.270 INFO [1] client ExtentFlush { repair_id: ReconciliationId(392), extent_id: 145, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:11.270 INFO [2] received reconcile message
Sep 22 23:15:11.270 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(392), op: ExtentFlush { repair_id: ReconciliationId(392), extent_id: 145, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:11.270 INFO [2] client ExtentFlush { repair_id: ReconciliationId(392), extent_id: 145, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:11.270 DEBG 392 Flush extent 145 with f:2 g:2
Sep 22 23:15:11.270 DEBG Flush just extent 145 with f:2 and g:2
Sep 22 23:15:11.270 DEBG [1] It's time to notify for 392
Sep 22 23:15:11.270 INFO Completion from [1] id:392 status:true
Sep 22 23:15:11.270 INFO [393/752] Repair commands completed
Sep 22 23:15:11.270 INFO Pop front: ReconcileIO { id: ReconciliationId(393), op: ExtentClose { repair_id: ReconciliationId(393), extent_id: 145 }, state: ClientData([New, New, New]) }
Sep 22 23:15:11.270 INFO Sent repair work, now wait for resp
Sep 22 23:15:11.270 INFO [0] received reconcile message
Sep 22 23:15:11.270 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(393), op: ExtentClose { repair_id: ReconciliationId(393), extent_id: 145 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:11.270 INFO [0] client ExtentClose { repair_id: ReconciliationId(393), extent_id: 145 }
Sep 22 23:15:11.270 INFO [1] received reconcile message
Sep 22 23:15:11.270 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(393), op: ExtentClose { repair_id: ReconciliationId(393), extent_id: 145 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:11.270 INFO [1] client ExtentClose { repair_id: ReconciliationId(393), extent_id: 145 }
Sep 22 23:15:11.271 INFO [2] received reconcile message
Sep 22 23:15:11.271 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(393), op: ExtentClose { repair_id: ReconciliationId(393), extent_id: 145 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:11.271 INFO [2] client ExtentClose { repair_id: ReconciliationId(393), extent_id: 145 }
Sep 22 23:15:11.271 DEBG 393 Close extent 145
Sep 22 23:15:11.271 DEBG 393 Close extent 145
Sep 22 23:15:11.271 DEBG 393 Close extent 145
Sep 22 23:15:11.272 DEBG [2] It's time to notify for 393
Sep 22 23:15:11.272 INFO Completion from [2] id:393 status:true
Sep 22 23:15:11.272 INFO [394/752] Repair commands completed
Sep 22 23:15:11.272 INFO Pop front: ReconcileIO { id: ReconciliationId(394), op: ExtentRepair { repair_id: ReconciliationId(394), extent_id: 145, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:11.272 INFO Sent repair work, now wait for resp
Sep 22 23:15:11.272 INFO [0] received reconcile message
Sep 22 23:15:11.272 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(394), op: ExtentRepair { repair_id: ReconciliationId(394), extent_id: 145, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:11.272 INFO [0] client ExtentRepair { repair_id: ReconciliationId(394), extent_id: 145, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:11.272 INFO [0] Sending repair request ReconciliationId(394)
Sep 22 23:15:11.272 INFO [1] received reconcile message
Sep 22 23:15:11.272 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(394), op: ExtentRepair { repair_id: ReconciliationId(394), extent_id: 145, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:11.272 INFO [1] client ExtentRepair { repair_id: ReconciliationId(394), extent_id: 145, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:11.272 INFO [1] No action required ReconciliationId(394)
Sep 22 23:15:11.272 INFO [2] received reconcile message
Sep 22 23:15:11.272 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(394), op: ExtentRepair { repair_id: ReconciliationId(394), extent_id: 145, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:11.272 INFO [2] client ExtentRepair { repair_id: ReconciliationId(394), extent_id: 145, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:11.272 INFO [2] No action required ReconciliationId(394)
Sep 22 23:15:11.272 DEBG 394 Repair extent 145 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:11.272 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/091.copy"
Sep 22 23:15:11.337 INFO accepted connection, remote_addr: 127.0.0.1:62775, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.337 TRCE incoming request, uri: /extent/145/files, method: GET, req_id: e921412f-6fd6-44eb-b757-74d15bcae950, remote_addr: 127.0.0.1:62775, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.337 INFO request completed, latency_us: 236, response_code: 200, uri: /extent/145/files, method: GET, req_id: e921412f-6fd6-44eb-b757-74d15bcae950, remote_addr: 127.0.0.1:62775, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.338 INFO eid:145 Found repair files: ["091", "091.db"]
Sep 22 23:15:11.338 TRCE incoming request, uri: /newextent/145/data, method: GET, req_id: 2df0297d-8ed1-4248-ba06-a68f8b38460f, remote_addr: 127.0.0.1:62775, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.338 INFO request completed, latency_us: 348, response_code: 200, uri: /newextent/145/data, method: GET, req_id: 2df0297d-8ed1-4248-ba06-a68f8b38460f, remote_addr: 127.0.0.1:62775, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.343 TRCE incoming request, uri: /newextent/145/db, method: GET, req_id: 5d806757-8399-463a-9bfd-8c0ac9810c1f, remote_addr: 127.0.0.1:62775, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.344 INFO request completed, latency_us: 293, response_code: 200, uri: /newextent/145/db, method: GET, req_id: 5d806757-8399-463a-9bfd-8c0ac9810c1f, remote_addr: 127.0.0.1:62775, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.345 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/091.copy" to "/tmp/downstairs-vrx8aK6L/00/000/091.replace"
Sep 22 23:15:11.345 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:11.346 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/091.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:11.346 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/091"
Sep 22 23:15:11.346 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/091.db"
Sep 22 23:15:11.346 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:11.346 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/091.replace" to "/tmp/downstairs-vrx8aK6L/00/000/091.completed"
Sep 22 23:15:11.346 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:11.346 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:11.346 DEBG [0] It's time to notify for 394
Sep 22 23:15:11.347 INFO Completion from [0] id:394 status:true
Sep 22 23:15:11.347 INFO [395/752] Repair commands completed
Sep 22 23:15:11.347 INFO Pop front: ReconcileIO { id: ReconciliationId(395), op: ExtentReopen { repair_id: ReconciliationId(395), extent_id: 145 }, state: ClientData([New, New, New]) }
Sep 22 23:15:11.347 INFO Sent repair work, now wait for resp
Sep 22 23:15:11.347 INFO [0] received reconcile message
Sep 22 23:15:11.347 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(395), op: ExtentReopen { repair_id: ReconciliationId(395), extent_id: 145 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:11.347 INFO [0] client ExtentReopen { repair_id: ReconciliationId(395), extent_id: 145 }
Sep 22 23:15:11.347 INFO [1] received reconcile message
Sep 22 23:15:11.347 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(395), op: ExtentReopen { repair_id: ReconciliationId(395), extent_id: 145 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:11.347 INFO [1] client ExtentReopen { repair_id: ReconciliationId(395), extent_id: 145 }
Sep 22 23:15:11.347 INFO [2] received reconcile message
Sep 22 23:15:11.347 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(395), op: ExtentReopen { repair_id: ReconciliationId(395), extent_id: 145 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:11.347 INFO [2] client ExtentReopen { repair_id: ReconciliationId(395), extent_id: 145 }
Sep 22 23:15:11.347 DEBG 395 Reopen extent 145
Sep 22 23:15:11.348 DEBG 395 Reopen extent 145
Sep 22 23:15:11.348 DEBG 395 Reopen extent 145
Sep 22 23:15:11.349 DEBG [2] It's time to notify for 395
Sep 22 23:15:11.349 INFO Completion from [2] id:395 status:true
Sep 22 23:15:11.349 INFO [396/752] Repair commands completed
Sep 22 23:15:11.349 INFO Pop front: ReconcileIO { id: ReconciliationId(396), op: ExtentFlush { repair_id: ReconciliationId(396), extent_id: 168, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:11.349 INFO Sent repair work, now wait for resp
Sep 22 23:15:11.349 INFO [0] received reconcile message
Sep 22 23:15:11.349 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(396), op: ExtentFlush { repair_id: ReconciliationId(396), extent_id: 168, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:11.349 INFO [0] client ExtentFlush { repair_id: ReconciliationId(396), extent_id: 168, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:11.349 INFO [1] received reconcile message
Sep 22 23:15:11.349 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(396), op: ExtentFlush { repair_id: ReconciliationId(396), extent_id: 168, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:11.349 INFO [1] client ExtentFlush { repair_id: ReconciliationId(396), extent_id: 168, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:11.349 INFO [2] received reconcile message
Sep 22 23:15:11.349 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(396), op: ExtentFlush { repair_id: ReconciliationId(396), extent_id: 168, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:11.349 INFO [2] client ExtentFlush { repair_id: ReconciliationId(396), extent_id: 168, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:11.349 DEBG 396 Flush extent 168 with f:2 g:2
Sep 22 23:15:11.349 DEBG Flush just extent 168 with f:2 and g:2
Sep 22 23:15:11.349 DEBG [1] It's time to notify for 396
Sep 22 23:15:11.349 INFO Completion from [1] id:396 status:true
Sep 22 23:15:11.349 INFO [397/752] Repair commands completed
Sep 22 23:15:11.349 INFO Pop front: ReconcileIO { id: ReconciliationId(397), op: ExtentClose { repair_id: ReconciliationId(397), extent_id: 168 }, state: ClientData([New, New, New]) }
Sep 22 23:15:11.350 INFO Sent repair work, now wait for resp
Sep 22 23:15:11.350 INFO [0] received reconcile message
Sep 22 23:15:11.350 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(397), op: ExtentClose { repair_id: ReconciliationId(397), extent_id: 168 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:11.350 INFO [0] client ExtentClose { repair_id: ReconciliationId(397), extent_id: 168 }
Sep 22 23:15:11.350 INFO [1] received reconcile message
Sep 22 23:15:11.350 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(397), op: ExtentClose { repair_id: ReconciliationId(397), extent_id: 168 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:11.350 INFO [1] client ExtentClose { repair_id: ReconciliationId(397), extent_id: 168 }
Sep 22 23:15:11.350 INFO [2] received reconcile message
Sep 22 23:15:11.350 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(397), op: ExtentClose { repair_id: ReconciliationId(397), extent_id: 168 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:11.350 INFO [2] client ExtentClose { repair_id: ReconciliationId(397), extent_id: 168 }
Sep 22 23:15:11.350 DEBG 397 Close extent 168
Sep 22 23:15:11.350 DEBG 397 Close extent 168
Sep 22 23:15:11.350 DEBG 397 Close extent 168
Sep 22 23:15:11.351 DEBG [2] It's time to notify for 397
Sep 22 23:15:11.351 INFO Completion from [2] id:397 status:true
Sep 22 23:15:11.351 INFO [398/752] Repair commands completed
Sep 22 23:15:11.351 INFO Pop front: ReconcileIO { id: ReconciliationId(398), op: ExtentRepair { repair_id: ReconciliationId(398), extent_id: 168, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:11.351 INFO Sent repair work, now wait for resp
Sep 22 23:15:11.351 INFO [0] received reconcile message
Sep 22 23:15:11.351 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(398), op: ExtentRepair { repair_id: ReconciliationId(398), extent_id: 168, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:11.351 INFO [0] client ExtentRepair { repair_id: ReconciliationId(398), extent_id: 168, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:11.351 INFO [0] Sending repair request ReconciliationId(398)
Sep 22 23:15:11.351 INFO [1] received reconcile message
Sep 22 23:15:11.351 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(398), op: ExtentRepair { repair_id: ReconciliationId(398), extent_id: 168, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:11.351 INFO [1] client ExtentRepair { repair_id: ReconciliationId(398), extent_id: 168, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:11.351 INFO [1] No action required ReconciliationId(398)
Sep 22 23:15:11.351 INFO [2] received reconcile message
Sep 22 23:15:11.351 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(398), op: ExtentRepair { repair_id: ReconciliationId(398), extent_id: 168, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:11.351 INFO [2] client ExtentRepair { repair_id: ReconciliationId(398), extent_id: 168, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:11.351 INFO [2] No action required ReconciliationId(398)
Sep 22 23:15:11.351 DEBG 398 Repair extent 168 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:11.351 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/0A8.copy"
Sep 22 23:15:11.375 ERRO [0] job id 1083 saw error GenericError("test error")
Sep 22 23:15:11.375 ERRO [0] job id 1083 saw error GenericError("test error")
Sep 22 23:15:11.375 DEBG up_ds_listen was notified
Sep 22 23:15:11.375 DEBG up_ds_listen process 1083
Sep 22 23:15:11.375 DEBG [A] ack job 1083:84, : downstairs
Sep 22 23:15:11.375 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:11.381 DEBG Read      :1082 deps:[JobId(1081)] res:true
Sep 22 23:15:11.403 DEBG IO Read  1084 has deps [JobId(1083)]
Sep 22 23:15:11.414 INFO accepted connection, remote_addr: 127.0.0.1:33934, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.415 TRCE incoming request, uri: /extent/168/files, method: GET, req_id: 183eb21a-6502-4d6f-884f-e9b87d04cf5d, remote_addr: 127.0.0.1:33934, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.415 INFO request completed, latency_us: 223, response_code: 200, uri: /extent/168/files, method: GET, req_id: 183eb21a-6502-4d6f-884f-e9b87d04cf5d, remote_addr: 127.0.0.1:33934, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.415 INFO eid:168 Found repair files: ["0A8", "0A8.db"]
Sep 22 23:15:11.415 TRCE incoming request, uri: /newextent/168/data, method: GET, req_id: 42229d6f-34f6-488e-8762-2277951d1815, remote_addr: 127.0.0.1:33934, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.416 INFO request completed, latency_us: 322, response_code: 200, uri: /newextent/168/data, method: GET, req_id: 42229d6f-34f6-488e-8762-2277951d1815, remote_addr: 127.0.0.1:33934, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.417 INFO [lossy] skipping 1083
Sep 22 23:15:11.417 DEBG Flush     :1083 extent_limit None deps:[JobId(1082), JobId(1081)] res:true f:31 g:1
Sep 22 23:15:11.417 INFO [lossy] skipping 1084
Sep 22 23:15:11.417 INFO [lossy] skipping 1084
Sep 22 23:15:11.417 INFO [lossy] skipping 1084
Sep 22 23:15:11.421 TRCE incoming request, uri: /newextent/168/db, method: GET, req_id: a5b7274c-d630-476d-8703-1dedd48c7b78, remote_addr: 127.0.0.1:33934, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.421 INFO request completed, latency_us: 292, response_code: 200, uri: /newextent/168/db, method: GET, req_id: a5b7274c-d630-476d-8703-1dedd48c7b78, remote_addr: 127.0.0.1:33934, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.422 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/0A8.copy" to "/tmp/downstairs-vrx8aK6L/00/000/0A8.replace"
Sep 22 23:15:11.422 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:11.423 DEBG Read      :1084 deps:[JobId(1083)] res:true
Sep 22 23:15:11.423 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/0A8.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:11.424 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/0A8"
Sep 22 23:15:11.424 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/0A8.db"
Sep 22 23:15:11.424 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:11.424 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/0A8.replace" to "/tmp/downstairs-vrx8aK6L/00/000/0A8.completed"
Sep 22 23:15:11.424 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:11.424 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:11.424 DEBG [0] It's time to notify for 398
Sep 22 23:15:11.424 INFO Completion from [0] id:398 status:true
Sep 22 23:15:11.424 INFO [399/752] Repair commands completed
Sep 22 23:15:11.424 INFO Pop front: ReconcileIO { id: ReconciliationId(399), op: ExtentReopen { repair_id: ReconciliationId(399), extent_id: 168 }, state: ClientData([New, New, New]) }
Sep 22 23:15:11.424 INFO Sent repair work, now wait for resp
Sep 22 23:15:11.424 INFO [0] received reconcile message
Sep 22 23:15:11.424 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(399), op: ExtentReopen { repair_id: ReconciliationId(399), extent_id: 168 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:11.424 INFO [0] client ExtentReopen { repair_id: ReconciliationId(399), extent_id: 168 }
Sep 22 23:15:11.424 INFO [1] received reconcile message
Sep 22 23:15:11.424 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(399), op: ExtentReopen { repair_id: ReconciliationId(399), extent_id: 168 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:11.424 INFO [1] client ExtentReopen { repair_id: ReconciliationId(399), extent_id: 168 }
Sep 22 23:15:11.424 INFO [2] received reconcile message
Sep 22 23:15:11.424 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(399), op: ExtentReopen { repair_id: ReconciliationId(399), extent_id: 168 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:11.424 INFO [2] client ExtentReopen { repair_id: ReconciliationId(399), extent_id: 168 }
Sep 22 23:15:11.425 DEBG 399 Reopen extent 168
Sep 22 23:15:11.425 DEBG 399 Reopen extent 168
Sep 22 23:15:11.426 DEBG 399 Reopen extent 168
Sep 22 23:15:11.426 DEBG [2] It's time to notify for 399
Sep 22 23:15:11.426 INFO Completion from [2] id:399 status:true
Sep 22 23:15:11.426 INFO [400/752] Repair commands completed
Sep 22 23:15:11.426 INFO Pop front: ReconcileIO { id: ReconciliationId(400), op: ExtentFlush { repair_id: ReconciliationId(400), extent_id: 127, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:11.426 INFO Sent repair work, now wait for resp
Sep 22 23:15:11.426 INFO [0] received reconcile message
Sep 22 23:15:11.427 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(400), op: ExtentFlush { repair_id: ReconciliationId(400), extent_id: 127, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:11.427 INFO [0] client ExtentFlush { repair_id: ReconciliationId(400), extent_id: 127, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:11.427 INFO [1] received reconcile message
Sep 22 23:15:11.427 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(400), op: ExtentFlush { repair_id: ReconciliationId(400), extent_id: 127, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:11.427 INFO [1] client ExtentFlush { repair_id: ReconciliationId(400), extent_id: 127, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:11.427 INFO [2] received reconcile message
Sep 22 23:15:11.427 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(400), op: ExtentFlush { repair_id: ReconciliationId(400), extent_id: 127, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:11.427 INFO [2] client ExtentFlush { repair_id: ReconciliationId(400), extent_id: 127, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:11.427 DEBG 400 Flush extent 127 with f:2 g:2
Sep 22 23:15:11.427 DEBG Flush just extent 127 with f:2 and g:2
Sep 22 23:15:11.427 DEBG [1] It's time to notify for 400
Sep 22 23:15:11.427 INFO Completion from [1] id:400 status:true
Sep 22 23:15:11.427 INFO [401/752] Repair commands completed
Sep 22 23:15:11.427 INFO Pop front: ReconcileIO { id: ReconciliationId(401), op: ExtentClose { repair_id: ReconciliationId(401), extent_id: 127 }, state: ClientData([New, New, New]) }
Sep 22 23:15:11.427 INFO Sent repair work, now wait for resp
Sep 22 23:15:11.427 INFO [0] received reconcile message
Sep 22 23:15:11.427 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(401), op: ExtentClose { repair_id: ReconciliationId(401), extent_id: 127 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:11.427 INFO [0] client ExtentClose { repair_id: ReconciliationId(401), extent_id: 127 }
Sep 22 23:15:11.427 INFO [1] received reconcile message
Sep 22 23:15:11.427 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(401), op: ExtentClose { repair_id: ReconciliationId(401), extent_id: 127 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:11.427 INFO [1] client ExtentClose { repair_id: ReconciliationId(401), extent_id: 127 }
Sep 22 23:15:11.427 INFO [2] received reconcile message
Sep 22 23:15:11.427 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(401), op: ExtentClose { repair_id: ReconciliationId(401), extent_id: 127 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:11.427 INFO [2] client ExtentClose { repair_id: ReconciliationId(401), extent_id: 127 }
Sep 22 23:15:11.427 DEBG 401 Close extent 127
Sep 22 23:15:11.428 DEBG 401 Close extent 127
Sep 22 23:15:11.428 DEBG 401 Close extent 127
Sep 22 23:15:11.428 DEBG [2] It's time to notify for 401
Sep 22 23:15:11.428 INFO Completion from [2] id:401 status:true
Sep 22 23:15:11.428 INFO [402/752] Repair commands completed
Sep 22 23:15:11.428 INFO Pop front: ReconcileIO { id: ReconciliationId(402), op: ExtentRepair { repair_id: ReconciliationId(402), extent_id: 127, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:11.428 INFO Sent repair work, now wait for resp
Sep 22 23:15:11.428 INFO [0] received reconcile message
Sep 22 23:15:11.428 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(402), op: ExtentRepair { repair_id: ReconciliationId(402), extent_id: 127, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:11.428 INFO [0] client ExtentRepair { repair_id: ReconciliationId(402), extent_id: 127, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:11.429 INFO [0] Sending repair request ReconciliationId(402)
Sep 22 23:15:11.429 INFO [1] received reconcile message
Sep 22 23:15:11.429 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(402), op: ExtentRepair { repair_id: ReconciliationId(402), extent_id: 127, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:11.429 INFO [1] client ExtentRepair { repair_id: ReconciliationId(402), extent_id: 127, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:11.429 INFO [1] No action required ReconciliationId(402)
Sep 22 23:15:11.429 INFO [2] received reconcile message
Sep 22 23:15:11.429 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(402), op: ExtentRepair { repair_id: ReconciliationId(402), extent_id: 127, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:11.429 INFO [2] client ExtentRepair { repair_id: ReconciliationId(402), extent_id: 127, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:11.429 INFO [2] No action required ReconciliationId(402)
Sep 22 23:15:11.429 DEBG 402 Repair extent 127 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:11.429 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/07F.copy"
Sep 22 23:15:11.493 INFO accepted connection, remote_addr: 127.0.0.1:33658, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.493 TRCE incoming request, uri: /extent/127/files, method: GET, req_id: 4a2b9aee-da6e-4e30-97db-d6a5edc6b8d5, remote_addr: 127.0.0.1:33658, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.493 INFO request completed, latency_us: 216, response_code: 200, uri: /extent/127/files, method: GET, req_id: 4a2b9aee-da6e-4e30-97db-d6a5edc6b8d5, remote_addr: 127.0.0.1:33658, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.494 INFO eid:127 Found repair files: ["07F", "07F.db"]
Sep 22 23:15:11.494 TRCE incoming request, uri: /newextent/127/data, method: GET, req_id: 81c1bbc9-076f-401d-b3cf-732a797ebf19, remote_addr: 127.0.0.1:33658, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.494 INFO request completed, latency_us: 257, response_code: 200, uri: /newextent/127/data, method: GET, req_id: 81c1bbc9-076f-401d-b3cf-732a797ebf19, remote_addr: 127.0.0.1:33658, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.499 TRCE incoming request, uri: /newextent/127/db, method: GET, req_id: 546a330f-36b7-4640-9219-ed90178f13d3, remote_addr: 127.0.0.1:33658, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.500 INFO request completed, latency_us: 293, response_code: 200, uri: /newextent/127/db, method: GET, req_id: 546a330f-36b7-4640-9219-ed90178f13d3, remote_addr: 127.0.0.1:33658, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.501 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/07F.copy" to "/tmp/downstairs-vrx8aK6L/00/000/07F.replace"
Sep 22 23:15:11.501 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:11.502 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/07F.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:11.502 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/07F"
Sep 22 23:15:11.502 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/07F.db"
Sep 22 23:15:11.502 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:11.502 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/07F.replace" to "/tmp/downstairs-vrx8aK6L/00/000/07F.completed"
Sep 22 23:15:11.502 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:11.502 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:11.502 DEBG [0] It's time to notify for 402
Sep 22 23:15:11.502 INFO Completion from [0] id:402 status:true
Sep 22 23:15:11.502 INFO [403/752] Repair commands completed
Sep 22 23:15:11.502 INFO Pop front: ReconcileIO { id: ReconciliationId(403), op: ExtentReopen { repair_id: ReconciliationId(403), extent_id: 127 }, state: ClientData([New, New, New]) }
Sep 22 23:15:11.502 INFO Sent repair work, now wait for resp
Sep 22 23:15:11.503 INFO [0] received reconcile message
Sep 22 23:15:11.503 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(403), op: ExtentReopen { repair_id: ReconciliationId(403), extent_id: 127 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:11.503 INFO [0] client ExtentReopen { repair_id: ReconciliationId(403), extent_id: 127 }
Sep 22 23:15:11.503 INFO [1] received reconcile message
Sep 22 23:15:11.503 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(403), op: ExtentReopen { repair_id: ReconciliationId(403), extent_id: 127 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:11.503 INFO [1] client ExtentReopen { repair_id: ReconciliationId(403), extent_id: 127 }
Sep 22 23:15:11.503 INFO [2] received reconcile message
Sep 22 23:15:11.503 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(403), op: ExtentReopen { repair_id: ReconciliationId(403), extent_id: 127 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:11.503 INFO [2] client ExtentReopen { repair_id: ReconciliationId(403), extent_id: 127 }
Sep 22 23:15:11.503 DEBG 403 Reopen extent 127
Sep 22 23:15:11.504 DEBG 403 Reopen extent 127
Sep 22 23:15:11.504 DEBG 403 Reopen extent 127
Sep 22 23:15:11.505 DEBG [2] It's time to notify for 403
Sep 22 23:15:11.505 INFO Completion from [2] id:403 status:true
Sep 22 23:15:11.505 INFO [404/752] Repair commands completed
Sep 22 23:15:11.505 INFO Pop front: ReconcileIO { id: ReconciliationId(404), op: ExtentFlush { repair_id: ReconciliationId(404), extent_id: 26, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:11.505 INFO Sent repair work, now wait for resp
Sep 22 23:15:11.505 INFO [0] received reconcile message
Sep 22 23:15:11.505 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(404), op: ExtentFlush { repair_id: ReconciliationId(404), extent_id: 26, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:11.505 INFO [0] client ExtentFlush { repair_id: ReconciliationId(404), extent_id: 26, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:11.505 INFO [1] received reconcile message
Sep 22 23:15:11.505 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(404), op: ExtentFlush { repair_id: ReconciliationId(404), extent_id: 26, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:11.505 INFO [1] client ExtentFlush { repair_id: ReconciliationId(404), extent_id: 26, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:11.505 INFO [2] received reconcile message
Sep 22 23:15:11.505 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(404), op: ExtentFlush { repair_id: ReconciliationId(404), extent_id: 26, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:11.505 INFO [2] client ExtentFlush { repair_id: ReconciliationId(404), extent_id: 26, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:11.505 DEBG 404 Flush extent 26 with f:2 g:2
Sep 22 23:15:11.505 DEBG Flush just extent 26 with f:2 and g:2
Sep 22 23:15:11.505 DEBG [1] It's time to notify for 404
Sep 22 23:15:11.505 INFO Completion from [1] id:404 status:true
Sep 22 23:15:11.505 INFO [405/752] Repair commands completed
Sep 22 23:15:11.505 INFO Pop front: ReconcileIO { id: ReconciliationId(405), op: ExtentClose { repair_id: ReconciliationId(405), extent_id: 26 }, state: ClientData([New, New, New]) }
Sep 22 23:15:11.505 INFO Sent repair work, now wait for resp
Sep 22 23:15:11.505 INFO [0] received reconcile message
Sep 22 23:15:11.505 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(405), op: ExtentClose { repair_id: ReconciliationId(405), extent_id: 26 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:11.505 INFO [0] client ExtentClose { repair_id: ReconciliationId(405), extent_id: 26 }
Sep 22 23:15:11.505 INFO [1] received reconcile message
Sep 22 23:15:11.505 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(405), op: ExtentClose { repair_id: ReconciliationId(405), extent_id: 26 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:11.505 INFO [1] client ExtentClose { repair_id: ReconciliationId(405), extent_id: 26 }
Sep 22 23:15:11.505 INFO [2] received reconcile message
Sep 22 23:15:11.505 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(405), op: ExtentClose { repair_id: ReconciliationId(405), extent_id: 26 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:11.506 INFO [2] client ExtentClose { repair_id: ReconciliationId(405), extent_id: 26 }
Sep 22 23:15:11.506 DEBG 405 Close extent 26
Sep 22 23:15:11.506 DEBG 405 Close extent 26
Sep 22 23:15:11.506 DEBG 405 Close extent 26
Sep 22 23:15:11.507 DEBG [2] It's time to notify for 405
Sep 22 23:15:11.507 INFO Completion from [2] id:405 status:true
Sep 22 23:15:11.507 INFO [406/752] Repair commands completed
Sep 22 23:15:11.507 INFO Pop front: ReconcileIO { id: ReconciliationId(406), op: ExtentRepair { repair_id: ReconciliationId(406), extent_id: 26, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:11.507 INFO Sent repair work, now wait for resp
Sep 22 23:15:11.507 INFO [0] received reconcile message
Sep 22 23:15:11.507 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(406), op: ExtentRepair { repair_id: ReconciliationId(406), extent_id: 26, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:11.507 INFO [0] client ExtentRepair { repair_id: ReconciliationId(406), extent_id: 26, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:11.507 INFO [0] Sending repair request ReconciliationId(406)
Sep 22 23:15:11.507 INFO [1] received reconcile message
Sep 22 23:15:11.507 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(406), op: ExtentRepair { repair_id: ReconciliationId(406), extent_id: 26, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:11.507 INFO [1] client ExtentRepair { repair_id: ReconciliationId(406), extent_id: 26, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:11.507 INFO [1] No action required ReconciliationId(406)
Sep 22 23:15:11.507 INFO [2] received reconcile message
Sep 22 23:15:11.507 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(406), op: ExtentRepair { repair_id: ReconciliationId(406), extent_id: 26, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:11.507 INFO [2] client ExtentRepair { repair_id: ReconciliationId(406), extent_id: 26, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:11.507 INFO [2] No action required ReconciliationId(406)
Sep 22 23:15:11.507 DEBG 406 Repair extent 26 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:11.507 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/01A.copy"
Sep 22 23:15:11.571 INFO accepted connection, remote_addr: 127.0.0.1:42529, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.571 TRCE incoming request, uri: /extent/26/files, method: GET, req_id: 1b9b10e1-8af0-42d6-9203-ae346e2b9b0a, remote_addr: 127.0.0.1:42529, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.572 INFO request completed, latency_us: 229, response_code: 200, uri: /extent/26/files, method: GET, req_id: 1b9b10e1-8af0-42d6-9203-ae346e2b9b0a, remote_addr: 127.0.0.1:42529, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.572 INFO eid:26 Found repair files: ["01A", "01A.db"]
Sep 22 23:15:11.572 TRCE incoming request, uri: /newextent/26/data, method: GET, req_id: 54ec14e7-3b7e-4e0d-81d6-0cbb2dacba46, remote_addr: 127.0.0.1:42529, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.572 INFO request completed, latency_us: 343, response_code: 200, uri: /newextent/26/data, method: GET, req_id: 54ec14e7-3b7e-4e0d-81d6-0cbb2dacba46, remote_addr: 127.0.0.1:42529, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.578 TRCE incoming request, uri: /newextent/26/db, method: GET, req_id: ea27c0d5-8311-4275-8b6d-d2f5d0e5a286, remote_addr: 127.0.0.1:42529, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.578 INFO request completed, latency_us: 293, response_code: 200, uri: /newextent/26/db, method: GET, req_id: ea27c0d5-8311-4275-8b6d-d2f5d0e5a286, remote_addr: 127.0.0.1:42529, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.579 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/01A.copy" to "/tmp/downstairs-vrx8aK6L/00/000/01A.replace"
Sep 22 23:15:11.579 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:11.580 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/01A.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:11.580 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/01A"
Sep 22 23:15:11.580 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/01A.db"
Sep 22 23:15:11.580 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:11.580 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/01A.replace" to "/tmp/downstairs-vrx8aK6L/00/000/01A.completed"
Sep 22 23:15:11.580 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:11.581 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:11.581 DEBG [0] It's time to notify for 406
Sep 22 23:15:11.581 INFO Completion from [0] id:406 status:true
Sep 22 23:15:11.581 INFO [407/752] Repair commands completed
Sep 22 23:15:11.581 INFO Pop front: ReconcileIO { id: ReconciliationId(407), op: ExtentReopen { repair_id: ReconciliationId(407), extent_id: 26 }, state: ClientData([New, New, New]) }
Sep 22 23:15:11.581 INFO Sent repair work, now wait for resp
Sep 22 23:15:11.581 INFO [0] received reconcile message
Sep 22 23:15:11.581 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(407), op: ExtentReopen { repair_id: ReconciliationId(407), extent_id: 26 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:11.581 INFO [0] client ExtentReopen { repair_id: ReconciliationId(407), extent_id: 26 }
Sep 22 23:15:11.581 INFO [1] received reconcile message
Sep 22 23:15:11.581 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(407), op: ExtentReopen { repair_id: ReconciliationId(407), extent_id: 26 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:11.581 INFO [1] client ExtentReopen { repair_id: ReconciliationId(407), extent_id: 26 }
Sep 22 23:15:11.581 INFO [2] received reconcile message
Sep 22 23:15:11.581 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(407), op: ExtentReopen { repair_id: ReconciliationId(407), extent_id: 26 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:11.581 INFO [2] client ExtentReopen { repair_id: ReconciliationId(407), extent_id: 26 }
Sep 22 23:15:11.581 DEBG 407 Reopen extent 26
Sep 22 23:15:11.582 DEBG 407 Reopen extent 26
Sep 22 23:15:11.583 DEBG 407 Reopen extent 26
Sep 22 23:15:11.583 DEBG [2] It's time to notify for 407
Sep 22 23:15:11.583 INFO Completion from [2] id:407 status:true
Sep 22 23:15:11.583 INFO [408/752] Repair commands completed
Sep 22 23:15:11.583 INFO Pop front: ReconcileIO { id: ReconciliationId(408), op: ExtentFlush { repair_id: ReconciliationId(408), extent_id: 34, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:11.583 INFO Sent repair work, now wait for resp
Sep 22 23:15:11.583 INFO [0] received reconcile message
Sep 22 23:15:11.583 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(408), op: ExtentFlush { repair_id: ReconciliationId(408), extent_id: 34, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:11.583 INFO [0] client ExtentFlush { repair_id: ReconciliationId(408), extent_id: 34, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:11.583 INFO [1] received reconcile message
Sep 22 23:15:11.583 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(408), op: ExtentFlush { repair_id: ReconciliationId(408), extent_id: 34, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:11.583 INFO [1] client ExtentFlush { repair_id: ReconciliationId(408), extent_id: 34, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:11.583 INFO [2] received reconcile message
Sep 22 23:15:11.583 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(408), op: ExtentFlush { repair_id: ReconciliationId(408), extent_id: 34, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:11.583 INFO [2] client ExtentFlush { repair_id: ReconciliationId(408), extent_id: 34, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:11.584 DEBG 408 Flush extent 34 with f:2 g:2
Sep 22 23:15:11.584 DEBG Flush just extent 34 with f:2 and g:2
Sep 22 23:15:11.584 DEBG [1] It's time to notify for 408
Sep 22 23:15:11.584 INFO Completion from [1] id:408 status:true
Sep 22 23:15:11.584 INFO [409/752] Repair commands completed
Sep 22 23:15:11.584 INFO Pop front: ReconcileIO { id: ReconciliationId(409), op: ExtentClose { repair_id: ReconciliationId(409), extent_id: 34 }, state: ClientData([New, New, New]) }
Sep 22 23:15:11.584 INFO Sent repair work, now wait for resp
Sep 22 23:15:11.584 INFO [0] received reconcile message
Sep 22 23:15:11.584 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(409), op: ExtentClose { repair_id: ReconciliationId(409), extent_id: 34 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:11.584 INFO [0] client ExtentClose { repair_id: ReconciliationId(409), extent_id: 34 }
Sep 22 23:15:11.584 INFO [1] received reconcile message
Sep 22 23:15:11.584 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(409), op: ExtentClose { repair_id: ReconciliationId(409), extent_id: 34 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:11.584 INFO [1] client ExtentClose { repair_id: ReconciliationId(409), extent_id: 34 }
Sep 22 23:15:11.584 INFO [2] received reconcile message
Sep 22 23:15:11.584 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(409), op: ExtentClose { repair_id: ReconciliationId(409), extent_id: 34 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:11.584 INFO [2] client ExtentClose { repair_id: ReconciliationId(409), extent_id: 34 }
Sep 22 23:15:11.584 DEBG 409 Close extent 34
Sep 22 23:15:11.584 DEBG 409 Close extent 34
Sep 22 23:15:11.585 DEBG 409 Close extent 34
Sep 22 23:15:11.585 DEBG [2] It's time to notify for 409
Sep 22 23:15:11.585 INFO Completion from [2] id:409 status:true
Sep 22 23:15:11.585 INFO [410/752] Repair commands completed
Sep 22 23:15:11.585 INFO Pop front: ReconcileIO { id: ReconciliationId(410), op: ExtentRepair { repair_id: ReconciliationId(410), extent_id: 34, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:11.585 INFO Sent repair work, now wait for resp
Sep 22 23:15:11.585 INFO [0] received reconcile message
Sep 22 23:15:11.585 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(410), op: ExtentRepair { repair_id: ReconciliationId(410), extent_id: 34, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:11.585 INFO [0] client ExtentRepair { repair_id: ReconciliationId(410), extent_id: 34, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:11.585 INFO [0] Sending repair request ReconciliationId(410)
Sep 22 23:15:11.585 INFO [1] received reconcile message
Sep 22 23:15:11.585 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(410), op: ExtentRepair { repair_id: ReconciliationId(410), extent_id: 34, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:11.585 INFO [1] client ExtentRepair { repair_id: ReconciliationId(410), extent_id: 34, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:11.585 INFO [1] No action required ReconciliationId(410)
Sep 22 23:15:11.585 INFO [2] received reconcile message
Sep 22 23:15:11.585 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(410), op: ExtentRepair { repair_id: ReconciliationId(410), extent_id: 34, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:11.585 INFO [2] client ExtentRepair { repair_id: ReconciliationId(410), extent_id: 34, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:11.585 INFO [2] No action required ReconciliationId(410)
Sep 22 23:15:11.586 DEBG 410 Repair extent 34 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:11.586 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/022.copy"
Sep 22 23:15:11.650 INFO accepted connection, remote_addr: 127.0.0.1:34604, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.650 TRCE incoming request, uri: /extent/34/files, method: GET, req_id: 75a10963-4838-4311-b957-e4167bac9ada, remote_addr: 127.0.0.1:34604, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.650 INFO request completed, latency_us: 197, response_code: 200, uri: /extent/34/files, method: GET, req_id: 75a10963-4838-4311-b957-e4167bac9ada, remote_addr: 127.0.0.1:34604, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.651 INFO eid:34 Found repair files: ["022", "022.db"]
Sep 22 23:15:11.651 TRCE incoming request, uri: /newextent/34/data, method: GET, req_id: e6cbc666-4bc9-4395-8a26-30e718184c95, remote_addr: 127.0.0.1:34604, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.651 INFO request completed, latency_us: 316, response_code: 200, uri: /newextent/34/data, method: GET, req_id: e6cbc666-4bc9-4395-8a26-30e718184c95, remote_addr: 127.0.0.1:34604, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.656 TRCE incoming request, uri: /newextent/34/db, method: GET, req_id: ee3d11e4-e9d8-449d-a1ac-6bd04655168d, remote_addr: 127.0.0.1:34604, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.657 INFO request completed, latency_us: 293, response_code: 200, uri: /newextent/34/db, method: GET, req_id: ee3d11e4-e9d8-449d-a1ac-6bd04655168d, remote_addr: 127.0.0.1:34604, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.658 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/022.copy" to "/tmp/downstairs-vrx8aK6L/00/000/022.replace"
Sep 22 23:15:11.658 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:11.659 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/022.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:11.659 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/022"
Sep 22 23:15:11.659 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/022.db"
Sep 22 23:15:11.659 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:11.659 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/022.replace" to "/tmp/downstairs-vrx8aK6L/00/000/022.completed"
Sep 22 23:15:11.659 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:11.659 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:11.659 DEBG [0] It's time to notify for 410
Sep 22 23:15:11.659 INFO Completion from [0] id:410 status:true
Sep 22 23:15:11.659 INFO [411/752] Repair commands completed
Sep 22 23:15:11.659 INFO Pop front: ReconcileIO { id: ReconciliationId(411), op: ExtentReopen { repair_id: ReconciliationId(411), extent_id: 34 }, state: ClientData([New, New, New]) }
Sep 22 23:15:11.659 INFO Sent repair work, now wait for resp
Sep 22 23:15:11.660 INFO [0] received reconcile message
Sep 22 23:15:11.660 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(411), op: ExtentReopen { repair_id: ReconciliationId(411), extent_id: 34 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:11.660 INFO [0] client ExtentReopen { repair_id: ReconciliationId(411), extent_id: 34 }
Sep 22 23:15:11.660 INFO [1] received reconcile message
Sep 22 23:15:11.660 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(411), op: ExtentReopen { repair_id: ReconciliationId(411), extent_id: 34 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:11.660 INFO [1] client ExtentReopen { repair_id: ReconciliationId(411), extent_id: 34 }
Sep 22 23:15:11.660 INFO [2] received reconcile message
Sep 22 23:15:11.660 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(411), op: ExtentReopen { repair_id: ReconciliationId(411), extent_id: 34 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:11.660 INFO [2] client ExtentReopen { repair_id: ReconciliationId(411), extent_id: 34 }
Sep 22 23:15:11.660 DEBG 411 Reopen extent 34
Sep 22 23:15:11.660 DEBG 411 Reopen extent 34
Sep 22 23:15:11.661 DEBG 411 Reopen extent 34
Sep 22 23:15:11.661 DEBG [2] It's time to notify for 411
Sep 22 23:15:11.662 INFO Completion from [2] id:411 status:true
Sep 22 23:15:11.662 INFO [412/752] Repair commands completed
Sep 22 23:15:11.662 INFO Pop front: ReconcileIO { id: ReconciliationId(412), op: ExtentFlush { repair_id: ReconciliationId(412), extent_id: 12, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:11.662 INFO Sent repair work, now wait for resp
Sep 22 23:15:11.662 INFO [0] received reconcile message
Sep 22 23:15:11.662 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(412), op: ExtentFlush { repair_id: ReconciliationId(412), extent_id: 12, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:11.662 INFO [0] client ExtentFlush { repair_id: ReconciliationId(412), extent_id: 12, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:11.662 INFO [1] received reconcile message
Sep 22 23:15:11.662 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(412), op: ExtentFlush { repair_id: ReconciliationId(412), extent_id: 12, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:11.662 INFO [1] client ExtentFlush { repair_id: ReconciliationId(412), extent_id: 12, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:11.662 INFO [2] received reconcile message
Sep 22 23:15:11.662 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(412), op: ExtentFlush { repair_id: ReconciliationId(412), extent_id: 12, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:11.662 INFO [2] client ExtentFlush { repair_id: ReconciliationId(412), extent_id: 12, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:11.662 DEBG 412 Flush extent 12 with f:2 g:2
Sep 22 23:15:11.662 DEBG Flush just extent 12 with f:2 and g:2
Sep 22 23:15:11.662 DEBG [1] It's time to notify for 412
Sep 22 23:15:11.662 INFO Completion from [1] id:412 status:true
Sep 22 23:15:11.662 INFO [413/752] Repair commands completed
Sep 22 23:15:11.662 INFO Pop front: ReconcileIO { id: ReconciliationId(413), op: ExtentClose { repair_id: ReconciliationId(413), extent_id: 12 }, state: ClientData([New, New, New]) }
Sep 22 23:15:11.662 INFO Sent repair work, now wait for resp
Sep 22 23:15:11.662 INFO [0] received reconcile message
Sep 22 23:15:11.662 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(413), op: ExtentClose { repair_id: ReconciliationId(413), extent_id: 12 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:11.662 INFO [0] client ExtentClose { repair_id: ReconciliationId(413), extent_id: 12 }
Sep 22 23:15:11.662 INFO [1] received reconcile message
Sep 22 23:15:11.662 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(413), op: ExtentClose { repair_id: ReconciliationId(413), extent_id: 12 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:11.662 INFO [1] client ExtentClose { repair_id: ReconciliationId(413), extent_id: 12 }
Sep 22 23:15:11.662 INFO [2] received reconcile message
Sep 22 23:15:11.662 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(413), op: ExtentClose { repair_id: ReconciliationId(413), extent_id: 12 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:11.662 INFO [2] client ExtentClose { repair_id: ReconciliationId(413), extent_id: 12 }
Sep 22 23:15:11.663 DEBG 413 Close extent 12
Sep 22 23:15:11.663 DEBG 413 Close extent 12
Sep 22 23:15:11.663 DEBG 413 Close extent 12
Sep 22 23:15:11.663 DEBG [2] It's time to notify for 413
Sep 22 23:15:11.663 INFO Completion from [2] id:413 status:true
Sep 22 23:15:11.664 INFO [414/752] Repair commands completed
Sep 22 23:15:11.664 INFO Pop front: ReconcileIO { id: ReconciliationId(414), op: ExtentRepair { repair_id: ReconciliationId(414), extent_id: 12, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:11.664 INFO Sent repair work, now wait for resp
Sep 22 23:15:11.664 INFO [0] received reconcile message
Sep 22 23:15:11.664 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(414), op: ExtentRepair { repair_id: ReconciliationId(414), extent_id: 12, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:11.664 INFO [0] client ExtentRepair { repair_id: ReconciliationId(414), extent_id: 12, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:11.664 INFO [0] Sending repair request ReconciliationId(414)
Sep 22 23:15:11.664 INFO [1] received reconcile message
Sep 22 23:15:11.664 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(414), op: ExtentRepair { repair_id: ReconciliationId(414), extent_id: 12, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:11.664 INFO [1] client ExtentRepair { repair_id: ReconciliationId(414), extent_id: 12, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:11.664 INFO [1] No action required ReconciliationId(414)
Sep 22 23:15:11.664 INFO [2] received reconcile message
Sep 22 23:15:11.664 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(414), op: ExtentRepair { repair_id: ReconciliationId(414), extent_id: 12, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:11.664 INFO [2] client ExtentRepair { repair_id: ReconciliationId(414), extent_id: 12, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:11.664 INFO [2] No action required ReconciliationId(414)
Sep 22 23:15:11.664 DEBG 414 Repair extent 12 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:11.664 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/00C.copy"
Sep 22 23:15:11.726 INFO accepted connection, remote_addr: 127.0.0.1:62790, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.727 TRCE incoming request, uri: /extent/12/files, method: GET, req_id: 07f68c1f-ee80-4eaf-af07-5c54812f0f8a, remote_addr: 127.0.0.1:62790, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.727 INFO request completed, latency_us: 246, response_code: 200, uri: /extent/12/files, method: GET, req_id: 07f68c1f-ee80-4eaf-af07-5c54812f0f8a, remote_addr: 127.0.0.1:62790, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.727 INFO eid:12 Found repair files: ["00C", "00C.db"]
Sep 22 23:15:11.727 TRCE incoming request, uri: /newextent/12/data, method: GET, req_id: c5019864-7b10-448c-89e6-d000ae79eccc, remote_addr: 127.0.0.1:62790, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.728 INFO request completed, latency_us: 340, response_code: 200, uri: /newextent/12/data, method: GET, req_id: c5019864-7b10-448c-89e6-d000ae79eccc, remote_addr: 127.0.0.1:62790, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.733 TRCE incoming request, uri: /newextent/12/db, method: GET, req_id: 5a5ebc74-abd4-4638-b647-a81d3527c464, remote_addr: 127.0.0.1:62790, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.733 INFO request completed, latency_us: 351, response_code: 200, uri: /newextent/12/db, method: GET, req_id: 5a5ebc74-abd4-4638-b647-a81d3527c464, remote_addr: 127.0.0.1:62790, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.735 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/00C.copy" to "/tmp/downstairs-vrx8aK6L/00/000/00C.replace"
Sep 22 23:15:11.735 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:11.736 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/00C.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:11.736 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/00C"
Sep 22 23:15:11.736 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/00C.db"
Sep 22 23:15:11.736 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:11.736 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/00C.replace" to "/tmp/downstairs-vrx8aK6L/00/000/00C.completed"
Sep 22 23:15:11.736 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:11.736 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:11.737 DEBG [0] It's time to notify for 414
Sep 22 23:15:11.737 INFO Completion from [0] id:414 status:true
Sep 22 23:15:11.737 INFO [415/752] Repair commands completed
Sep 22 23:15:11.737 INFO Pop front: ReconcileIO { id: ReconciliationId(415), op: ExtentReopen { repair_id: ReconciliationId(415), extent_id: 12 }, state: ClientData([New, New, New]) }
Sep 22 23:15:11.737 INFO Sent repair work, now wait for resp
Sep 22 23:15:11.737 INFO [0] received reconcile message
Sep 22 23:15:11.737 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(415), op: ExtentReopen { repair_id: ReconciliationId(415), extent_id: 12 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:11.737 INFO [0] client ExtentReopen { repair_id: ReconciliationId(415), extent_id: 12 }
Sep 22 23:15:11.737 INFO [1] received reconcile message
Sep 22 23:15:11.737 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(415), op: ExtentReopen { repair_id: ReconciliationId(415), extent_id: 12 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:11.737 INFO [1] client ExtentReopen { repair_id: ReconciliationId(415), extent_id: 12 }
Sep 22 23:15:11.737 INFO [2] received reconcile message
Sep 22 23:15:11.737 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(415), op: ExtentReopen { repair_id: ReconciliationId(415), extent_id: 12 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:11.737 INFO [2] client ExtentReopen { repair_id: ReconciliationId(415), extent_id: 12 }
Sep 22 23:15:11.737 DEBG 415 Reopen extent 12
Sep 22 23:15:11.738 DEBG 415 Reopen extent 12
Sep 22 23:15:11.739 DEBG 415 Reopen extent 12
Sep 22 23:15:11.739 DEBG [2] It's time to notify for 415
Sep 22 23:15:11.739 INFO Completion from [2] id:415 status:true
Sep 22 23:15:11.739 INFO [416/752] Repair commands completed
Sep 22 23:15:11.739 INFO Pop front: ReconcileIO { id: ReconciliationId(416), op: ExtentFlush { repair_id: ReconciliationId(416), extent_id: 172, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:11.739 INFO Sent repair work, now wait for resp
Sep 22 23:15:11.739 INFO [0] received reconcile message
Sep 22 23:15:11.739 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(416), op: ExtentFlush { repair_id: ReconciliationId(416), extent_id: 172, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:11.739 INFO [0] client ExtentFlush { repair_id: ReconciliationId(416), extent_id: 172, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:11.740 INFO [1] received reconcile message
Sep 22 23:15:11.740 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(416), op: ExtentFlush { repair_id: ReconciliationId(416), extent_id: 172, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:11.740 INFO [1] client ExtentFlush { repair_id: ReconciliationId(416), extent_id: 172, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:11.740 INFO [2] received reconcile message
Sep 22 23:15:11.740 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(416), op: ExtentFlush { repair_id: ReconciliationId(416), extent_id: 172, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:11.740 INFO [2] client ExtentFlush { repair_id: ReconciliationId(416), extent_id: 172, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:11.740 DEBG 416 Flush extent 172 with f:2 g:2
Sep 22 23:15:11.740 DEBG Flush just extent 172 with f:2 and g:2
Sep 22 23:15:11.740 DEBG [1] It's time to notify for 416
Sep 22 23:15:11.740 INFO Completion from [1] id:416 status:true
Sep 22 23:15:11.740 INFO [417/752] Repair commands completed
Sep 22 23:15:11.740 INFO Pop front: ReconcileIO { id: ReconciliationId(417), op: ExtentClose { repair_id: ReconciliationId(417), extent_id: 172 }, state: ClientData([New, New, New]) }
Sep 22 23:15:11.740 INFO Sent repair work, now wait for resp
Sep 22 23:15:11.740 INFO [0] received reconcile message
Sep 22 23:15:11.740 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(417), op: ExtentClose { repair_id: ReconciliationId(417), extent_id: 172 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:11.740 INFO [0] client ExtentClose { repair_id: ReconciliationId(417), extent_id: 172 }
Sep 22 23:15:11.740 INFO [1] received reconcile message
Sep 22 23:15:11.740 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(417), op: ExtentClose { repair_id: ReconciliationId(417), extent_id: 172 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:11.740 INFO [1] client ExtentClose { repair_id: ReconciliationId(417), extent_id: 172 }
Sep 22 23:15:11.740 INFO [2] received reconcile message
Sep 22 23:15:11.740 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(417), op: ExtentClose { repair_id: ReconciliationId(417), extent_id: 172 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:11.740 INFO [2] client ExtentClose { repair_id: ReconciliationId(417), extent_id: 172 }
Sep 22 23:15:11.740 DEBG 417 Close extent 172
Sep 22 23:15:11.741 DEBG 417 Close extent 172
Sep 22 23:15:11.741 DEBG 417 Close extent 172
Sep 22 23:15:11.741 DEBG [2] It's time to notify for 417
Sep 22 23:15:11.741 INFO Completion from [2] id:417 status:true
Sep 22 23:15:11.741 INFO [418/752] Repair commands completed
Sep 22 23:15:11.741 INFO Pop front: ReconcileIO { id: ReconciliationId(418), op: ExtentRepair { repair_id: ReconciliationId(418), extent_id: 172, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:11.741 INFO Sent repair work, now wait for resp
Sep 22 23:15:11.741 INFO [0] received reconcile message
Sep 22 23:15:11.741 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(418), op: ExtentRepair { repair_id: ReconciliationId(418), extent_id: 172, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:11.741 INFO [0] client ExtentRepair { repair_id: ReconciliationId(418), extent_id: 172, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:11.742 INFO [0] Sending repair request ReconciliationId(418)
Sep 22 23:15:11.742 INFO [1] received reconcile message
Sep 22 23:15:11.742 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(418), op: ExtentRepair { repair_id: ReconciliationId(418), extent_id: 172, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:11.742 INFO [1] client ExtentRepair { repair_id: ReconciliationId(418), extent_id: 172, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:11.742 INFO [1] No action required ReconciliationId(418)
Sep 22 23:15:11.742 INFO [2] received reconcile message
Sep 22 23:15:11.742 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(418), op: ExtentRepair { repair_id: ReconciliationId(418), extent_id: 172, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:11.742 INFO [2] client ExtentRepair { repair_id: ReconciliationId(418), extent_id: 172, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:11.742 INFO [2] No action required ReconciliationId(418)
Sep 22 23:15:11.742 DEBG 418 Repair extent 172 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:11.742 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/0AC.copy"
Sep 22 23:15:11.804 INFO accepted connection, remote_addr: 127.0.0.1:37584, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.805 TRCE incoming request, uri: /extent/172/files, method: GET, req_id: 6d40fc74-e2c7-47a0-afb4-8db6534fbcd7, remote_addr: 127.0.0.1:37584, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.805 INFO request completed, latency_us: 243, response_code: 200, uri: /extent/172/files, method: GET, req_id: 6d40fc74-e2c7-47a0-afb4-8db6534fbcd7, remote_addr: 127.0.0.1:37584, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.805 INFO eid:172 Found repair files: ["0AC", "0AC.db"]
Sep 22 23:15:11.806 TRCE incoming request, uri: /newextent/172/data, method: GET, req_id: e195532e-1c5e-4006-af9e-7d704a7e663a, remote_addr: 127.0.0.1:37584, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.806 INFO request completed, latency_us: 347, response_code: 200, uri: /newextent/172/data, method: GET, req_id: e195532e-1c5e-4006-af9e-7d704a7e663a, remote_addr: 127.0.0.1:37584, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.811 TRCE incoming request, uri: /newextent/172/db, method: GET, req_id: 284e8531-3182-4ac8-9d72-b75dc82d53c3, remote_addr: 127.0.0.1:37584, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.811 INFO request completed, latency_us: 296, response_code: 200, uri: /newextent/172/db, method: GET, req_id: 284e8531-3182-4ac8-9d72-b75dc82d53c3, remote_addr: 127.0.0.1:37584, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.813 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/0AC.copy" to "/tmp/downstairs-vrx8aK6L/00/000/0AC.replace"
Sep 22 23:15:11.813 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:11.814 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/0AC.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:11.814 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/0AC"
Sep 22 23:15:11.814 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/0AC.db"
Sep 22 23:15:11.814 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:11.814 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/0AC.replace" to "/tmp/downstairs-vrx8aK6L/00/000/0AC.completed"
Sep 22 23:15:11.814 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:11.814 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:11.814 DEBG [0] It's time to notify for 418
Sep 22 23:15:11.815 INFO Completion from [0] id:418 status:true
Sep 22 23:15:11.815 INFO [419/752] Repair commands completed
Sep 22 23:15:11.815 INFO Pop front: ReconcileIO { id: ReconciliationId(419), op: ExtentReopen { repair_id: ReconciliationId(419), extent_id: 172 }, state: ClientData([New, New, New]) }
Sep 22 23:15:11.815 INFO Sent repair work, now wait for resp
Sep 22 23:15:11.815 INFO [0] received reconcile message
Sep 22 23:15:11.815 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(419), op: ExtentReopen { repair_id: ReconciliationId(419), extent_id: 172 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:11.815 INFO [0] client ExtentReopen { repair_id: ReconciliationId(419), extent_id: 172 }
Sep 22 23:15:11.815 INFO [1] received reconcile message
Sep 22 23:15:11.815 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(419), op: ExtentReopen { repair_id: ReconciliationId(419), extent_id: 172 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:11.815 INFO [1] client ExtentReopen { repair_id: ReconciliationId(419), extent_id: 172 }
Sep 22 23:15:11.815 INFO [2] received reconcile message
Sep 22 23:15:11.815 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(419), op: ExtentReopen { repair_id: ReconciliationId(419), extent_id: 172 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:11.815 INFO [2] client ExtentReopen { repair_id: ReconciliationId(419), extent_id: 172 }
Sep 22 23:15:11.815 DEBG 419 Reopen extent 172
Sep 22 23:15:11.816 DEBG 419 Reopen extent 172
Sep 22 23:15:11.816 DEBG 419 Reopen extent 172
Sep 22 23:15:11.817 DEBG [2] It's time to notify for 419
Sep 22 23:15:11.817 INFO Completion from [2] id:419 status:true
Sep 22 23:15:11.817 INFO [420/752] Repair commands completed
Sep 22 23:15:11.817 INFO Pop front: ReconcileIO { id: ReconciliationId(420), op: ExtentFlush { repair_id: ReconciliationId(420), extent_id: 53, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:11.817 INFO Sent repair work, now wait for resp
Sep 22 23:15:11.817 INFO [0] received reconcile message
Sep 22 23:15:11.817 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(420), op: ExtentFlush { repair_id: ReconciliationId(420), extent_id: 53, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:11.817 INFO [0] client ExtentFlush { repair_id: ReconciliationId(420), extent_id: 53, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:11.817 INFO [1] received reconcile message
Sep 22 23:15:11.817 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(420), op: ExtentFlush { repair_id: ReconciliationId(420), extent_id: 53, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:11.817 INFO [1] client ExtentFlush { repair_id: ReconciliationId(420), extent_id: 53, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:11.817 INFO [2] received reconcile message
Sep 22 23:15:11.817 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(420), op: ExtentFlush { repair_id: ReconciliationId(420), extent_id: 53, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:11.817 INFO [2] client ExtentFlush { repair_id: ReconciliationId(420), extent_id: 53, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:11.817 DEBG 420 Flush extent 53 with f:2 g:2
Sep 22 23:15:11.817 DEBG Flush just extent 53 with f:2 and g:2
Sep 22 23:15:11.817 DEBG [1] It's time to notify for 420
Sep 22 23:15:11.817 INFO Completion from [1] id:420 status:true
Sep 22 23:15:11.817 INFO [421/752] Repair commands completed
Sep 22 23:15:11.817 INFO Pop front: ReconcileIO { id: ReconciliationId(421), op: ExtentClose { repair_id: ReconciliationId(421), extent_id: 53 }, state: ClientData([New, New, New]) }
Sep 22 23:15:11.818 INFO Sent repair work, now wait for resp
Sep 22 23:15:11.818 INFO [0] received reconcile message
Sep 22 23:15:11.818 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(421), op: ExtentClose { repair_id: ReconciliationId(421), extent_id: 53 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:11.818 INFO [0] client ExtentClose { repair_id: ReconciliationId(421), extent_id: 53 }
Sep 22 23:15:11.818 INFO [1] received reconcile message
Sep 22 23:15:11.818 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(421), op: ExtentClose { repair_id: ReconciliationId(421), extent_id: 53 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:11.818 INFO [1] client ExtentClose { repair_id: ReconciliationId(421), extent_id: 53 }
Sep 22 23:15:11.818 INFO [2] received reconcile message
Sep 22 23:15:11.818 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(421), op: ExtentClose { repair_id: ReconciliationId(421), extent_id: 53 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:11.818 INFO [2] client ExtentClose { repair_id: ReconciliationId(421), extent_id: 53 }
Sep 22 23:15:11.818 DEBG 421 Close extent 53
Sep 22 23:15:11.818 DEBG 421 Close extent 53
Sep 22 23:15:11.818 DEBG 421 Close extent 53
Sep 22 23:15:11.819 DEBG [2] It's time to notify for 421
Sep 22 23:15:11.819 INFO Completion from [2] id:421 status:true
Sep 22 23:15:11.819 INFO [422/752] Repair commands completed
Sep 22 23:15:11.819 INFO Pop front: ReconcileIO { id: ReconciliationId(422), op: ExtentRepair { repair_id: ReconciliationId(422), extent_id: 53, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:11.819 INFO Sent repair work, now wait for resp
Sep 22 23:15:11.819 INFO [0] received reconcile message
Sep 22 23:15:11.819 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(422), op: ExtentRepair { repair_id: ReconciliationId(422), extent_id: 53, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:11.819 INFO [0] client ExtentRepair { repair_id: ReconciliationId(422), extent_id: 53, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:11.819 INFO [0] Sending repair request ReconciliationId(422)
Sep 22 23:15:11.819 INFO [1] received reconcile message
Sep 22 23:15:11.819 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(422), op: ExtentRepair { repair_id: ReconciliationId(422), extent_id: 53, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:11.819 INFO [1] client ExtentRepair { repair_id: ReconciliationId(422), extent_id: 53, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:11.819 INFO [1] No action required ReconciliationId(422)
Sep 22 23:15:11.819 INFO [2] received reconcile message
Sep 22 23:15:11.819 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(422), op: ExtentRepair { repair_id: ReconciliationId(422), extent_id: 53, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:11.819 INFO [2] client ExtentRepair { repair_id: ReconciliationId(422), extent_id: 53, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:11.819 INFO [2] No action required ReconciliationId(422)
Sep 22 23:15:11.819 DEBG 422 Repair extent 53 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:11.819 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/035.copy"
Sep 22 23:15:11.825 DEBG [rc] retire 1083 clears [JobId(1082), JobId(1083)], : downstairs
Sep 22 23:15:11.825 DEBG IO Flush 1085 has deps [JobId(1084)]
Sep 22 23:15:11.831 DEBG Read      :1084 deps:[JobId(1083)] res:true
Sep 22 23:15:11.859 DEBG Read      :1084 deps:[JobId(1083)] res:true
Sep 22 23:15:11.882 INFO accepted connection, remote_addr: 127.0.0.1:49693, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.883 TRCE incoming request, uri: /extent/53/files, method: GET, req_id: e9c5ddb7-bff8-43e1-a5c1-d10cc24b99ea, remote_addr: 127.0.0.1:49693, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.883 INFO request completed, latency_us: 225, response_code: 200, uri: /extent/53/files, method: GET, req_id: e9c5ddb7-bff8-43e1-a5c1-d10cc24b99ea, remote_addr: 127.0.0.1:49693, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.883 INFO eid:53 Found repair files: ["035", "035.db"]
Sep 22 23:15:11.883 TRCE incoming request, uri: /newextent/53/data, method: GET, req_id: 62b63dc1-f589-4f65-b006-ea840af99148, remote_addr: 127.0.0.1:49693, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.884 DEBG Flush     :1085 extent_limit None deps:[JobId(1084)] res:true f:32 g:1
Sep 22 23:15:11.884 INFO [lossy] sleeping 1 second
Sep 22 23:15:11.884 INFO request completed, latency_us: 321, response_code: 200, uri: /newextent/53/data, method: GET, req_id: 62b63dc1-f589-4f65-b006-ea840af99148, remote_addr: 127.0.0.1:49693, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.889 TRCE incoming request, uri: /newextent/53/db, method: GET, req_id: 0cb534ce-2ded-4bb8-84d1-33ab70990a7d, remote_addr: 127.0.0.1:49693, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.889 INFO request completed, latency_us: 298, response_code: 200, uri: /newextent/53/db, method: GET, req_id: 0cb534ce-2ded-4bb8-84d1-33ab70990a7d, remote_addr: 127.0.0.1:49693, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.890 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/035.copy" to "/tmp/downstairs-vrx8aK6L/00/000/035.replace"
Sep 22 23:15:11.890 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:11.891 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/035.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:11.892 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/035"
Sep 22 23:15:11.892 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/035.db"
Sep 22 23:15:11.892 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:11.892 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/035.replace" to "/tmp/downstairs-vrx8aK6L/00/000/035.completed"
Sep 22 23:15:11.892 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:11.892 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:11.892 DEBG [0] It's time to notify for 422
Sep 22 23:15:11.892 INFO Completion from [0] id:422 status:true
Sep 22 23:15:11.892 INFO [423/752] Repair commands completed
Sep 22 23:15:11.892 INFO Pop front: ReconcileIO { id: ReconciliationId(423), op: ExtentReopen { repair_id: ReconciliationId(423), extent_id: 53 }, state: ClientData([New, New, New]) }
Sep 22 23:15:11.892 INFO Sent repair work, now wait for resp
Sep 22 23:15:11.892 INFO [0] received reconcile message
Sep 22 23:15:11.892 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(423), op: ExtentReopen { repair_id: ReconciliationId(423), extent_id: 53 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:11.892 INFO [0] client ExtentReopen { repair_id: ReconciliationId(423), extent_id: 53 }
Sep 22 23:15:11.892 INFO [1] received reconcile message
Sep 22 23:15:11.892 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(423), op: ExtentReopen { repair_id: ReconciliationId(423), extent_id: 53 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:11.892 INFO [1] client ExtentReopen { repair_id: ReconciliationId(423), extent_id: 53 }
Sep 22 23:15:11.892 INFO [2] received reconcile message
Sep 22 23:15:11.892 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(423), op: ExtentReopen { repair_id: ReconciliationId(423), extent_id: 53 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:11.892 INFO [2] client ExtentReopen { repair_id: ReconciliationId(423), extent_id: 53 }
Sep 22 23:15:11.893 DEBG 423 Reopen extent 53
Sep 22 23:15:11.893 DEBG 423 Reopen extent 53
Sep 22 23:15:11.894 DEBG 423 Reopen extent 53
Sep 22 23:15:11.894 DEBG [2] It's time to notify for 423
Sep 22 23:15:11.894 INFO Completion from [2] id:423 status:true
Sep 22 23:15:11.894 INFO [424/752] Repair commands completed
Sep 22 23:15:11.895 INFO Pop front: ReconcileIO { id: ReconciliationId(424), op: ExtentFlush { repair_id: ReconciliationId(424), extent_id: 175, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:11.895 INFO Sent repair work, now wait for resp
Sep 22 23:15:11.895 INFO [0] received reconcile message
Sep 22 23:15:11.895 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(424), op: ExtentFlush { repair_id: ReconciliationId(424), extent_id: 175, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:11.895 INFO [0] client ExtentFlush { repair_id: ReconciliationId(424), extent_id: 175, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:11.895 INFO [1] received reconcile message
Sep 22 23:15:11.895 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(424), op: ExtentFlush { repair_id: ReconciliationId(424), extent_id: 175, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:11.895 INFO [1] client ExtentFlush { repair_id: ReconciliationId(424), extent_id: 175, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:11.895 INFO [2] received reconcile message
Sep 22 23:15:11.895 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(424), op: ExtentFlush { repair_id: ReconciliationId(424), extent_id: 175, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:11.895 INFO [2] client ExtentFlush { repair_id: ReconciliationId(424), extent_id: 175, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:11.895 DEBG 424 Flush extent 175 with f:2 g:2
Sep 22 23:15:11.895 DEBG Flush just extent 175 with f:2 and g:2
Sep 22 23:15:11.895 DEBG [1] It's time to notify for 424
Sep 22 23:15:11.895 INFO Completion from [1] id:424 status:true
Sep 22 23:15:11.895 INFO [425/752] Repair commands completed
Sep 22 23:15:11.895 INFO Pop front: ReconcileIO { id: ReconciliationId(425), op: ExtentClose { repair_id: ReconciliationId(425), extent_id: 175 }, state: ClientData([New, New, New]) }
Sep 22 23:15:11.895 INFO Sent repair work, now wait for resp
Sep 22 23:15:11.895 INFO [0] received reconcile message
Sep 22 23:15:11.895 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(425), op: ExtentClose { repair_id: ReconciliationId(425), extent_id: 175 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:11.895 INFO [0] client ExtentClose { repair_id: ReconciliationId(425), extent_id: 175 }
Sep 22 23:15:11.895 INFO [1] received reconcile message
Sep 22 23:15:11.895 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(425), op: ExtentClose { repair_id: ReconciliationId(425), extent_id: 175 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:11.895 INFO [1] client ExtentClose { repair_id: ReconciliationId(425), extent_id: 175 }
Sep 22 23:15:11.895 INFO [2] received reconcile message
Sep 22 23:15:11.895 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(425), op: ExtentClose { repair_id: ReconciliationId(425), extent_id: 175 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:11.895 INFO [2] client ExtentClose { repair_id: ReconciliationId(425), extent_id: 175 }
Sep 22 23:15:11.895 DEBG 425 Close extent 175
Sep 22 23:15:11.896 DEBG 425 Close extent 175
Sep 22 23:15:11.896 DEBG 425 Close extent 175
Sep 22 23:15:11.896 DEBG [2] It's time to notify for 425
Sep 22 23:15:11.896 INFO Completion from [2] id:425 status:true
Sep 22 23:15:11.896 INFO [426/752] Repair commands completed
Sep 22 23:15:11.896 INFO Pop front: ReconcileIO { id: ReconciliationId(426), op: ExtentRepair { repair_id: ReconciliationId(426), extent_id: 175, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:11.897 INFO Sent repair work, now wait for resp
Sep 22 23:15:11.897 INFO [0] received reconcile message
Sep 22 23:15:11.897 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(426), op: ExtentRepair { repair_id: ReconciliationId(426), extent_id: 175, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:11.897 INFO [0] client ExtentRepair { repair_id: ReconciliationId(426), extent_id: 175, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:11.897 INFO [0] Sending repair request ReconciliationId(426)
Sep 22 23:15:11.897 INFO [1] received reconcile message
Sep 22 23:15:11.897 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(426), op: ExtentRepair { repair_id: ReconciliationId(426), extent_id: 175, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:11.897 INFO [1] client ExtentRepair { repair_id: ReconciliationId(426), extent_id: 175, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:11.897 INFO [1] No action required ReconciliationId(426)
Sep 22 23:15:11.897 INFO [2] received reconcile message
Sep 22 23:15:11.897 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(426), op: ExtentRepair { repair_id: ReconciliationId(426), extent_id: 175, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:11.897 INFO [2] client ExtentRepair { repair_id: ReconciliationId(426), extent_id: 175, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:11.897 INFO [2] No action required ReconciliationId(426)
Sep 22 23:15:11.897 DEBG 426 Repair extent 175 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:11.897 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/0AF.copy"
Sep 22 23:15:11.932 INFO [lossy] sleeping 1 second
Sep 22 23:15:11.960 INFO accepted connection, remote_addr: 127.0.0.1:39881, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.960 TRCE incoming request, uri: /extent/175/files, method: GET, req_id: f4bf0792-7ea4-4ec7-b29a-516bd932aef2, remote_addr: 127.0.0.1:39881, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.960 INFO request completed, latency_us: 195, response_code: 200, uri: /extent/175/files, method: GET, req_id: f4bf0792-7ea4-4ec7-b29a-516bd932aef2, remote_addr: 127.0.0.1:39881, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.961 INFO eid:175 Found repair files: ["0AF", "0AF.db"]
Sep 22 23:15:11.961 TRCE incoming request, uri: /newextent/175/data, method: GET, req_id: a908de21-a3fc-47ae-a6a1-958435575886, remote_addr: 127.0.0.1:39881, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.961 INFO request completed, latency_us: 335, response_code: 200, uri: /newextent/175/data, method: GET, req_id: a908de21-a3fc-47ae-a6a1-958435575886, remote_addr: 127.0.0.1:39881, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.966 TRCE incoming request, uri: /newextent/175/db, method: GET, req_id: ab42246d-b87d-4fac-ad4b-447e11bdd0b5, remote_addr: 127.0.0.1:39881, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.967 INFO request completed, latency_us: 295, response_code: 200, uri: /newextent/175/db, method: GET, req_id: ab42246d-b87d-4fac-ad4b-447e11bdd0b5, remote_addr: 127.0.0.1:39881, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:11.968 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/0AF.copy" to "/tmp/downstairs-vrx8aK6L/00/000/0AF.replace"
Sep 22 23:15:11.968 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:11.969 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/0AF.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:11.969 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/0AF"
Sep 22 23:15:11.969 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/0AF.db"
Sep 22 23:15:11.969 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:11.969 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/0AF.replace" to "/tmp/downstairs-vrx8aK6L/00/000/0AF.completed"
Sep 22 23:15:11.969 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:11.969 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:11.969 DEBG [0] It's time to notify for 426
Sep 22 23:15:11.970 INFO Completion from [0] id:426 status:true
Sep 22 23:15:11.970 INFO [427/752] Repair commands completed
Sep 22 23:15:11.970 INFO Pop front: ReconcileIO { id: ReconciliationId(427), op: ExtentReopen { repair_id: ReconciliationId(427), extent_id: 175 }, state: ClientData([New, New, New]) }
Sep 22 23:15:11.970 INFO Sent repair work, now wait for resp
Sep 22 23:15:11.970 INFO [0] received reconcile message
Sep 22 23:15:11.970 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(427), op: ExtentReopen { repair_id: ReconciliationId(427), extent_id: 175 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:11.970 INFO [0] client ExtentReopen { repair_id: ReconciliationId(427), extent_id: 175 }
Sep 22 23:15:11.970 INFO [1] received reconcile message
Sep 22 23:15:11.970 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(427), op: ExtentReopen { repair_id: ReconciliationId(427), extent_id: 175 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:11.970 INFO [1] client ExtentReopen { repair_id: ReconciliationId(427), extent_id: 175 }
Sep 22 23:15:11.970 INFO [2] received reconcile message
Sep 22 23:15:11.970 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(427), op: ExtentReopen { repair_id: ReconciliationId(427), extent_id: 175 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:11.970 INFO [2] client ExtentReopen { repair_id: ReconciliationId(427), extent_id: 175 }
Sep 22 23:15:11.970 DEBG 427 Reopen extent 175
Sep 22 23:15:11.971 DEBG 427 Reopen extent 175
Sep 22 23:15:11.971 DEBG 427 Reopen extent 175
Sep 22 23:15:11.972 DEBG [2] It's time to notify for 427
Sep 22 23:15:11.972 INFO Completion from [2] id:427 status:true
Sep 22 23:15:11.972 INFO [428/752] Repair commands completed
Sep 22 23:15:11.972 INFO Pop front: ReconcileIO { id: ReconciliationId(428), op: ExtentFlush { repair_id: ReconciliationId(428), extent_id: 140, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:11.972 INFO Sent repair work, now wait for resp
Sep 22 23:15:11.972 INFO [0] received reconcile message
Sep 22 23:15:11.972 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(428), op: ExtentFlush { repair_id: ReconciliationId(428), extent_id: 140, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:11.972 INFO [0] client ExtentFlush { repair_id: ReconciliationId(428), extent_id: 140, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:11.972 INFO [1] received reconcile message
Sep 22 23:15:11.972 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(428), op: ExtentFlush { repair_id: ReconciliationId(428), extent_id: 140, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:11.972 INFO [1] client ExtentFlush { repair_id: ReconciliationId(428), extent_id: 140, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:11.972 INFO [2] received reconcile message
Sep 22 23:15:11.972 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(428), op: ExtentFlush { repair_id: ReconciliationId(428), extent_id: 140, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:11.972 INFO [2] client ExtentFlush { repair_id: ReconciliationId(428), extent_id: 140, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:11.972 DEBG 428 Flush extent 140 with f:2 g:2
Sep 22 23:15:11.972 DEBG Flush just extent 140 with f:2 and g:2
Sep 22 23:15:11.972 DEBG [1] It's time to notify for 428
Sep 22 23:15:11.972 INFO Completion from [1] id:428 status:true
Sep 22 23:15:11.972 INFO [429/752] Repair commands completed
Sep 22 23:15:11.972 INFO Pop front: ReconcileIO { id: ReconciliationId(429), op: ExtentClose { repair_id: ReconciliationId(429), extent_id: 140 }, state: ClientData([New, New, New]) }
Sep 22 23:15:11.972 INFO Sent repair work, now wait for resp
Sep 22 23:15:11.972 INFO [0] received reconcile message
Sep 22 23:15:11.972 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(429), op: ExtentClose { repair_id: ReconciliationId(429), extent_id: 140 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:11.972 INFO [0] client ExtentClose { repair_id: ReconciliationId(429), extent_id: 140 }
Sep 22 23:15:11.972 INFO [1] received reconcile message
Sep 22 23:15:11.972 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(429), op: ExtentClose { repair_id: ReconciliationId(429), extent_id: 140 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:11.972 INFO [1] client ExtentClose { repair_id: ReconciliationId(429), extent_id: 140 }
Sep 22 23:15:11.973 INFO [2] received reconcile message
Sep 22 23:15:11.973 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(429), op: ExtentClose { repair_id: ReconciliationId(429), extent_id: 140 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:11.973 INFO [2] client ExtentClose { repair_id: ReconciliationId(429), extent_id: 140 }
Sep 22 23:15:11.973 DEBG 429 Close extent 140
Sep 22 23:15:11.973 DEBG 429 Close extent 140
Sep 22 23:15:11.973 DEBG 429 Close extent 140
Sep 22 23:15:11.974 DEBG [2] It's time to notify for 429
Sep 22 23:15:11.974 INFO Completion from [2] id:429 status:true
Sep 22 23:15:11.974 INFO [430/752] Repair commands completed
Sep 22 23:15:11.974 INFO Pop front: ReconcileIO { id: ReconciliationId(430), op: ExtentRepair { repair_id: ReconciliationId(430), extent_id: 140, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:11.974 INFO Sent repair work, now wait for resp
Sep 22 23:15:11.974 INFO [0] received reconcile message
Sep 22 23:15:11.974 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(430), op: ExtentRepair { repair_id: ReconciliationId(430), extent_id: 140, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:11.974 INFO [0] client ExtentRepair { repair_id: ReconciliationId(430), extent_id: 140, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:11.974 INFO [0] Sending repair request ReconciliationId(430)
Sep 22 23:15:11.974 INFO [1] received reconcile message
Sep 22 23:15:11.974 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(430), op: ExtentRepair { repair_id: ReconciliationId(430), extent_id: 140, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:11.974 INFO [1] client ExtentRepair { repair_id: ReconciliationId(430), extent_id: 140, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:11.974 INFO [1] No action required ReconciliationId(430)
Sep 22 23:15:11.974 INFO [2] received reconcile message
Sep 22 23:15:11.974 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(430), op: ExtentRepair { repair_id: ReconciliationId(430), extent_id: 140, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:11.974 INFO [2] client ExtentRepair { repair_id: ReconciliationId(430), extent_id: 140, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:11.974 INFO [2] No action required ReconciliationId(430)
Sep 22 23:15:11.974 DEBG 430 Repair extent 140 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:11.974 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/08C.copy"
Sep 22 23:15:12.037 INFO accepted connection, remote_addr: 127.0.0.1:52379, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.037 TRCE incoming request, uri: /extent/140/files, method: GET, req_id: c9d4aaa1-796b-40ba-8a27-2fa3aacc8b8f, remote_addr: 127.0.0.1:52379, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.038 INFO request completed, latency_us: 240, response_code: 200, uri: /extent/140/files, method: GET, req_id: c9d4aaa1-796b-40ba-8a27-2fa3aacc8b8f, remote_addr: 127.0.0.1:52379, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.038 INFO eid:140 Found repair files: ["08C", "08C.db"]
Sep 22 23:15:12.038 TRCE incoming request, uri: /newextent/140/data, method: GET, req_id: 20261ec3-4d50-49e9-87a4-c75652cfcfed, remote_addr: 127.0.0.1:52379, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.039 INFO request completed, latency_us: 331, response_code: 200, uri: /newextent/140/data, method: GET, req_id: 20261ec3-4d50-49e9-87a4-c75652cfcfed, remote_addr: 127.0.0.1:52379, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.044 TRCE incoming request, uri: /newextent/140/db, method: GET, req_id: 9792fdb7-1d39-4c86-843d-c56796939b30, remote_addr: 127.0.0.1:52379, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.044 INFO request completed, latency_us: 306, response_code: 200, uri: /newextent/140/db, method: GET, req_id: 9792fdb7-1d39-4c86-843d-c56796939b30, remote_addr: 127.0.0.1:52379, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.045 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/08C.copy" to "/tmp/downstairs-vrx8aK6L/00/000/08C.replace"
Sep 22 23:15:12.045 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:12.046 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/08C.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:12.047 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/08C"
Sep 22 23:15:12.047 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/08C.db"
Sep 22 23:15:12.047 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:12.047 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/08C.replace" to "/tmp/downstairs-vrx8aK6L/00/000/08C.completed"
Sep 22 23:15:12.047 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:12.047 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:12.047 DEBG [0] It's time to notify for 430
Sep 22 23:15:12.047 INFO Completion from [0] id:430 status:true
Sep 22 23:15:12.047 INFO [431/752] Repair commands completed
Sep 22 23:15:12.047 INFO Pop front: ReconcileIO { id: ReconciliationId(431), op: ExtentReopen { repair_id: ReconciliationId(431), extent_id: 140 }, state: ClientData([New, New, New]) }
Sep 22 23:15:12.047 INFO Sent repair work, now wait for resp
Sep 22 23:15:12.047 INFO [0] received reconcile message
Sep 22 23:15:12.047 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(431), op: ExtentReopen { repair_id: ReconciliationId(431), extent_id: 140 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:12.047 INFO [0] client ExtentReopen { repair_id: ReconciliationId(431), extent_id: 140 }
Sep 22 23:15:12.047 INFO [1] received reconcile message
Sep 22 23:15:12.047 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(431), op: ExtentReopen { repair_id: ReconciliationId(431), extent_id: 140 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:12.047 INFO [1] client ExtentReopen { repair_id: ReconciliationId(431), extent_id: 140 }
Sep 22 23:15:12.048 INFO [2] received reconcile message
Sep 22 23:15:12.048 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(431), op: ExtentReopen { repair_id: ReconciliationId(431), extent_id: 140 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:12.048 INFO [2] client ExtentReopen { repair_id: ReconciliationId(431), extent_id: 140 }
Sep 22 23:15:12.048 DEBG 431 Reopen extent 140
Sep 22 23:15:12.048 DEBG 431 Reopen extent 140
Sep 22 23:15:12.049 DEBG 431 Reopen extent 140
Sep 22 23:15:12.050 DEBG [2] It's time to notify for 431
Sep 22 23:15:12.050 INFO Completion from [2] id:431 status:true
Sep 22 23:15:12.050 INFO [432/752] Repair commands completed
Sep 22 23:15:12.050 INFO Pop front: ReconcileIO { id: ReconciliationId(432), op: ExtentFlush { repair_id: ReconciliationId(432), extent_id: 167, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:12.050 INFO Sent repair work, now wait for resp
Sep 22 23:15:12.050 INFO [0] received reconcile message
Sep 22 23:15:12.050 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(432), op: ExtentFlush { repair_id: ReconciliationId(432), extent_id: 167, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:12.050 INFO [0] client ExtentFlush { repair_id: ReconciliationId(432), extent_id: 167, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:12.050 INFO [1] received reconcile message
Sep 22 23:15:12.050 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(432), op: ExtentFlush { repair_id: ReconciliationId(432), extent_id: 167, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:12.050 INFO [1] client ExtentFlush { repair_id: ReconciliationId(432), extent_id: 167, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:12.050 INFO [2] received reconcile message
Sep 22 23:15:12.050 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(432), op: ExtentFlush { repair_id: ReconciliationId(432), extent_id: 167, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:12.050 INFO [2] client ExtentFlush { repair_id: ReconciliationId(432), extent_id: 167, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:12.050 DEBG 432 Flush extent 167 with f:2 g:2
Sep 22 23:15:12.050 DEBG Flush just extent 167 with f:2 and g:2
Sep 22 23:15:12.050 DEBG [1] It's time to notify for 432
Sep 22 23:15:12.050 INFO Completion from [1] id:432 status:true
Sep 22 23:15:12.050 INFO [433/752] Repair commands completed
Sep 22 23:15:12.050 INFO Pop front: ReconcileIO { id: ReconciliationId(433), op: ExtentClose { repair_id: ReconciliationId(433), extent_id: 167 }, state: ClientData([New, New, New]) }
Sep 22 23:15:12.050 INFO Sent repair work, now wait for resp
Sep 22 23:15:12.050 INFO [0] received reconcile message
Sep 22 23:15:12.050 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(433), op: ExtentClose { repair_id: ReconciliationId(433), extent_id: 167 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:12.050 INFO [0] client ExtentClose { repair_id: ReconciliationId(433), extent_id: 167 }
Sep 22 23:15:12.050 INFO [1] received reconcile message
Sep 22 23:15:12.050 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(433), op: ExtentClose { repair_id: ReconciliationId(433), extent_id: 167 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:12.050 INFO [1] client ExtentClose { repair_id: ReconciliationId(433), extent_id: 167 }
Sep 22 23:15:12.050 INFO [2] received reconcile message
Sep 22 23:15:12.050 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(433), op: ExtentClose { repair_id: ReconciliationId(433), extent_id: 167 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:12.050 INFO [2] client ExtentClose { repair_id: ReconciliationId(433), extent_id: 167 }
Sep 22 23:15:12.051 DEBG 433 Close extent 167
Sep 22 23:15:12.051 DEBG 433 Close extent 167
Sep 22 23:15:12.051 DEBG 433 Close extent 167
Sep 22 23:15:12.052 DEBG [2] It's time to notify for 433
Sep 22 23:15:12.052 INFO Completion from [2] id:433 status:true
Sep 22 23:15:12.052 INFO [434/752] Repair commands completed
Sep 22 23:15:12.052 INFO Pop front: ReconcileIO { id: ReconciliationId(434), op: ExtentRepair { repair_id: ReconciliationId(434), extent_id: 167, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:12.052 INFO Sent repair work, now wait for resp
Sep 22 23:15:12.052 INFO [0] received reconcile message
Sep 22 23:15:12.052 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(434), op: ExtentRepair { repair_id: ReconciliationId(434), extent_id: 167, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:12.052 INFO [0] client ExtentRepair { repair_id: ReconciliationId(434), extent_id: 167, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:12.052 INFO [0] Sending repair request ReconciliationId(434)
Sep 22 23:15:12.052 INFO [1] received reconcile message
Sep 22 23:15:12.052 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(434), op: ExtentRepair { repair_id: ReconciliationId(434), extent_id: 167, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:12.052 INFO [1] client ExtentRepair { repair_id: ReconciliationId(434), extent_id: 167, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:12.052 INFO [1] No action required ReconciliationId(434)
Sep 22 23:15:12.052 INFO [2] received reconcile message
Sep 22 23:15:12.052 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(434), op: ExtentRepair { repair_id: ReconciliationId(434), extent_id: 167, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:12.052 INFO [2] client ExtentRepair { repair_id: ReconciliationId(434), extent_id: 167, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:12.052 INFO [2] No action required ReconciliationId(434)
Sep 22 23:15:12.052 DEBG 434 Repair extent 167 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:12.052 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/0A7.copy"
Sep 22 23:15:12.117 INFO accepted connection, remote_addr: 127.0.0.1:62058, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.117 TRCE incoming request, uri: /extent/167/files, method: GET, req_id: 5541d023-016e-4024-9ed1-0ee81abc8b66, remote_addr: 127.0.0.1:62058, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.117 INFO request completed, latency_us: 191, response_code: 200, uri: /extent/167/files, method: GET, req_id: 5541d023-016e-4024-9ed1-0ee81abc8b66, remote_addr: 127.0.0.1:62058, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.118 INFO eid:167 Found repair files: ["0A7", "0A7.db"]
Sep 22 23:15:12.118 TRCE incoming request, uri: /newextent/167/data, method: GET, req_id: f71f775f-fbaa-4297-824c-9cb66eeae8dd, remote_addr: 127.0.0.1:62058, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.118 INFO request completed, latency_us: 344, response_code: 200, uri: /newextent/167/data, method: GET, req_id: f71f775f-fbaa-4297-824c-9cb66eeae8dd, remote_addr: 127.0.0.1:62058, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.124 TRCE incoming request, uri: /newextent/167/db, method: GET, req_id: d723036b-0083-4e5e-96f8-5164bf9ea101, remote_addr: 127.0.0.1:62058, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.124 INFO request completed, latency_us: 292, response_code: 200, uri: /newextent/167/db, method: GET, req_id: d723036b-0083-4e5e-96f8-5164bf9ea101, remote_addr: 127.0.0.1:62058, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.125 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/0A7.copy" to "/tmp/downstairs-vrx8aK6L/00/000/0A7.replace"
Sep 22 23:15:12.125 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:12.126 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/0A7.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:12.126 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/0A7"
Sep 22 23:15:12.126 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/0A7.db"
Sep 22 23:15:12.126 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:12.126 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/0A7.replace" to "/tmp/downstairs-vrx8aK6L/00/000/0A7.completed"
Sep 22 23:15:12.126 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:12.126 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:12.127 DEBG [0] It's time to notify for 434
Sep 22 23:15:12.127 INFO Completion from [0] id:434 status:true
Sep 22 23:15:12.127 INFO [435/752] Repair commands completed
Sep 22 23:15:12.127 INFO Pop front: ReconcileIO { id: ReconciliationId(435), op: ExtentReopen { repair_id: ReconciliationId(435), extent_id: 167 }, state: ClientData([New, New, New]) }
Sep 22 23:15:12.127 INFO Sent repair work, now wait for resp
Sep 22 23:15:12.127 INFO [0] received reconcile message
Sep 22 23:15:12.127 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(435), op: ExtentReopen { repair_id: ReconciliationId(435), extent_id: 167 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:12.127 INFO [0] client ExtentReopen { repair_id: ReconciliationId(435), extent_id: 167 }
Sep 22 23:15:12.127 INFO [1] received reconcile message
Sep 22 23:15:12.127 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(435), op: ExtentReopen { repair_id: ReconciliationId(435), extent_id: 167 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:12.127 INFO [1] client ExtentReopen { repair_id: ReconciliationId(435), extent_id: 167 }
Sep 22 23:15:12.127 INFO [2] received reconcile message
Sep 22 23:15:12.127 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(435), op: ExtentReopen { repair_id: ReconciliationId(435), extent_id: 167 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:12.127 INFO [2] client ExtentReopen { repair_id: ReconciliationId(435), extent_id: 167 }
Sep 22 23:15:12.127 DEBG 435 Reopen extent 167
Sep 22 23:15:12.128 DEBG 435 Reopen extent 167
Sep 22 23:15:12.128 DEBG 435 Reopen extent 167
Sep 22 23:15:12.129 DEBG [2] It's time to notify for 435
Sep 22 23:15:12.129 INFO Completion from [2] id:435 status:true
Sep 22 23:15:12.129 INFO [436/752] Repair commands completed
Sep 22 23:15:12.129 INFO Pop front: ReconcileIO { id: ReconciliationId(436), op: ExtentFlush { repair_id: ReconciliationId(436), extent_id: 142, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:12.129 INFO Sent repair work, now wait for resp
Sep 22 23:15:12.129 INFO [0] received reconcile message
Sep 22 23:15:12.129 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(436), op: ExtentFlush { repair_id: ReconciliationId(436), extent_id: 142, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:12.129 INFO [0] client ExtentFlush { repair_id: ReconciliationId(436), extent_id: 142, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:12.129 INFO [1] received reconcile message
Sep 22 23:15:12.129 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(436), op: ExtentFlush { repair_id: ReconciliationId(436), extent_id: 142, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:12.129 INFO [1] client ExtentFlush { repair_id: ReconciliationId(436), extent_id: 142, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:12.129 INFO [2] received reconcile message
Sep 22 23:15:12.129 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(436), op: ExtentFlush { repair_id: ReconciliationId(436), extent_id: 142, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:12.129 INFO [2] client ExtentFlush { repair_id: ReconciliationId(436), extent_id: 142, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:12.129 DEBG 436 Flush extent 142 with f:2 g:2
Sep 22 23:15:12.129 DEBG Flush just extent 142 with f:2 and g:2
Sep 22 23:15:12.129 DEBG [1] It's time to notify for 436
Sep 22 23:15:12.129 INFO Completion from [1] id:436 status:true
Sep 22 23:15:12.129 INFO [437/752] Repair commands completed
Sep 22 23:15:12.129 INFO Pop front: ReconcileIO { id: ReconciliationId(437), op: ExtentClose { repair_id: ReconciliationId(437), extent_id: 142 }, state: ClientData([New, New, New]) }
Sep 22 23:15:12.130 INFO Sent repair work, now wait for resp
Sep 22 23:15:12.130 INFO [0] received reconcile message
Sep 22 23:15:12.130 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(437), op: ExtentClose { repair_id: ReconciliationId(437), extent_id: 142 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:12.130 INFO [0] client ExtentClose { repair_id: ReconciliationId(437), extent_id: 142 }
Sep 22 23:15:12.130 INFO [1] received reconcile message
Sep 22 23:15:12.130 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(437), op: ExtentClose { repair_id: ReconciliationId(437), extent_id: 142 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:12.130 INFO [1] client ExtentClose { repair_id: ReconciliationId(437), extent_id: 142 }
Sep 22 23:15:12.130 INFO [2] received reconcile message
Sep 22 23:15:12.130 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(437), op: ExtentClose { repair_id: ReconciliationId(437), extent_id: 142 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:12.130 INFO [2] client ExtentClose { repair_id: ReconciliationId(437), extent_id: 142 }
Sep 22 23:15:12.130 DEBG 437 Close extent 142
Sep 22 23:15:12.130 DEBG 437 Close extent 142
Sep 22 23:15:12.130 DEBG 437 Close extent 142
Sep 22 23:15:12.131 DEBG [2] It's time to notify for 437
Sep 22 23:15:12.131 INFO Completion from [2] id:437 status:true
Sep 22 23:15:12.131 INFO [438/752] Repair commands completed
Sep 22 23:15:12.131 INFO Pop front: ReconcileIO { id: ReconciliationId(438), op: ExtentRepair { repair_id: ReconciliationId(438), extent_id: 142, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:12.131 INFO Sent repair work, now wait for resp
Sep 22 23:15:12.131 INFO [0] received reconcile message
Sep 22 23:15:12.131 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(438), op: ExtentRepair { repair_id: ReconciliationId(438), extent_id: 142, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:12.131 INFO [0] client ExtentRepair { repair_id: ReconciliationId(438), extent_id: 142, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:12.131 INFO [0] Sending repair request ReconciliationId(438)
Sep 22 23:15:12.131 INFO [1] received reconcile message
Sep 22 23:15:12.131 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(438), op: ExtentRepair { repair_id: ReconciliationId(438), extent_id: 142, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:12.131 INFO [1] client ExtentRepair { repair_id: ReconciliationId(438), extent_id: 142, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:12.131 INFO [1] No action required ReconciliationId(438)
Sep 22 23:15:12.131 INFO [2] received reconcile message
Sep 22 23:15:12.131 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(438), op: ExtentRepair { repair_id: ReconciliationId(438), extent_id: 142, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:12.131 INFO [2] client ExtentRepair { repair_id: ReconciliationId(438), extent_id: 142, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:12.131 INFO [2] No action required ReconciliationId(438)
Sep 22 23:15:12.131 DEBG 438 Repair extent 142 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:12.131 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/08E.copy"
Sep 22 23:15:12.194 INFO accepted connection, remote_addr: 127.0.0.1:50202, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.194 TRCE incoming request, uri: /extent/142/files, method: GET, req_id: 35da7fed-ae8c-4d5a-8cc6-47040391d177, remote_addr: 127.0.0.1:50202, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.195 INFO request completed, latency_us: 197, response_code: 200, uri: /extent/142/files, method: GET, req_id: 35da7fed-ae8c-4d5a-8cc6-47040391d177, remote_addr: 127.0.0.1:50202, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.195 INFO eid:142 Found repair files: ["08E", "08E.db"]
Sep 22 23:15:12.195 TRCE incoming request, uri: /newextent/142/data, method: GET, req_id: 5e192a6a-1d91-44c3-a632-1db637d7e88f, remote_addr: 127.0.0.1:50202, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.195 INFO request completed, latency_us: 325, response_code: 200, uri: /newextent/142/data, method: GET, req_id: 5e192a6a-1d91-44c3-a632-1db637d7e88f, remote_addr: 127.0.0.1:50202, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.201 TRCE incoming request, uri: /newextent/142/db, method: GET, req_id: 83dc4f83-ab55-4e3b-9c16-13819b0b5634, remote_addr: 127.0.0.1:50202, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.201 INFO request completed, latency_us: 296, response_code: 200, uri: /newextent/142/db, method: GET, req_id: 83dc4f83-ab55-4e3b-9c16-13819b0b5634, remote_addr: 127.0.0.1:50202, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.202 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/08E.copy" to "/tmp/downstairs-vrx8aK6L/00/000/08E.replace"
Sep 22 23:15:12.202 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:12.203 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/08E.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:12.203 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/08E"
Sep 22 23:15:12.203 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/08E.db"
Sep 22 23:15:12.203 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:12.203 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/08E.replace" to "/tmp/downstairs-vrx8aK6L/00/000/08E.completed"
Sep 22 23:15:12.203 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:12.203 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:12.204 DEBG [0] It's time to notify for 438
Sep 22 23:15:12.204 INFO Completion from [0] id:438 status:true
Sep 22 23:15:12.204 INFO [439/752] Repair commands completed
Sep 22 23:15:12.204 INFO Pop front: ReconcileIO { id: ReconciliationId(439), op: ExtentReopen { repair_id: ReconciliationId(439), extent_id: 142 }, state: ClientData([New, New, New]) }
Sep 22 23:15:12.204 INFO Sent repair work, now wait for resp
Sep 22 23:15:12.204 INFO [0] received reconcile message
Sep 22 23:15:12.204 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(439), op: ExtentReopen { repair_id: ReconciliationId(439), extent_id: 142 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:12.204 INFO [0] client ExtentReopen { repair_id: ReconciliationId(439), extent_id: 142 }
Sep 22 23:15:12.204 INFO [1] received reconcile message
Sep 22 23:15:12.204 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(439), op: ExtentReopen { repair_id: ReconciliationId(439), extent_id: 142 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:12.204 INFO [1] client ExtentReopen { repair_id: ReconciliationId(439), extent_id: 142 }
Sep 22 23:15:12.204 INFO [2] received reconcile message
Sep 22 23:15:12.204 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(439), op: ExtentReopen { repair_id: ReconciliationId(439), extent_id: 142 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:12.204 INFO [2] client ExtentReopen { repair_id: ReconciliationId(439), extent_id: 142 }
Sep 22 23:15:12.204 DEBG 439 Reopen extent 142
Sep 22 23:15:12.205 DEBG 439 Reopen extent 142
Sep 22 23:15:12.205 DEBG 439 Reopen extent 142
Sep 22 23:15:12.206 DEBG [2] It's time to notify for 439
Sep 22 23:15:12.206 INFO Completion from [2] id:439 status:true
Sep 22 23:15:12.206 INFO [440/752] Repair commands completed
Sep 22 23:15:12.206 INFO Pop front: ReconcileIO { id: ReconciliationId(440), op: ExtentFlush { repair_id: ReconciliationId(440), extent_id: 171, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:12.206 INFO Sent repair work, now wait for resp
Sep 22 23:15:12.206 INFO [0] received reconcile message
Sep 22 23:15:12.206 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(440), op: ExtentFlush { repair_id: ReconciliationId(440), extent_id: 171, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:12.206 INFO [0] client ExtentFlush { repair_id: ReconciliationId(440), extent_id: 171, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:12.206 INFO [1] received reconcile message
Sep 22 23:15:12.206 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(440), op: ExtentFlush { repair_id: ReconciliationId(440), extent_id: 171, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:12.206 INFO [1] client ExtentFlush { repair_id: ReconciliationId(440), extent_id: 171, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:12.206 INFO [2] received reconcile message
Sep 22 23:15:12.206 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(440), op: ExtentFlush { repair_id: ReconciliationId(440), extent_id: 171, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:12.206 INFO [2] client ExtentFlush { repair_id: ReconciliationId(440), extent_id: 171, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:12.206 DEBG 440 Flush extent 171 with f:2 g:2
Sep 22 23:15:12.206 DEBG Flush just extent 171 with f:2 and g:2
Sep 22 23:15:12.206 DEBG [1] It's time to notify for 440
Sep 22 23:15:12.207 INFO Completion from [1] id:440 status:true
Sep 22 23:15:12.207 INFO [441/752] Repair commands completed
Sep 22 23:15:12.207 INFO Pop front: ReconcileIO { id: ReconciliationId(441), op: ExtentClose { repair_id: ReconciliationId(441), extent_id: 171 }, state: ClientData([New, New, New]) }
Sep 22 23:15:12.207 INFO Sent repair work, now wait for resp
Sep 22 23:15:12.207 INFO [0] received reconcile message
Sep 22 23:15:12.207 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(441), op: ExtentClose { repair_id: ReconciliationId(441), extent_id: 171 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:12.207 INFO [0] client ExtentClose { repair_id: ReconciliationId(441), extent_id: 171 }
Sep 22 23:15:12.207 INFO [1] received reconcile message
Sep 22 23:15:12.207 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(441), op: ExtentClose { repair_id: ReconciliationId(441), extent_id: 171 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:12.207 INFO [1] client ExtentClose { repair_id: ReconciliationId(441), extent_id: 171 }
Sep 22 23:15:12.207 INFO [2] received reconcile message
Sep 22 23:15:12.207 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(441), op: ExtentClose { repair_id: ReconciliationId(441), extent_id: 171 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:12.207 INFO [2] client ExtentClose { repair_id: ReconciliationId(441), extent_id: 171 }
Sep 22 23:15:12.207 DEBG 441 Close extent 171
Sep 22 23:15:12.207 DEBG 441 Close extent 171
Sep 22 23:15:12.207 DEBG 441 Close extent 171
Sep 22 23:15:12.208 DEBG [2] It's time to notify for 441
Sep 22 23:15:12.208 INFO Completion from [2] id:441 status:true
Sep 22 23:15:12.208 INFO [442/752] Repair commands completed
Sep 22 23:15:12.208 INFO Pop front: ReconcileIO { id: ReconciliationId(442), op: ExtentRepair { repair_id: ReconciliationId(442), extent_id: 171, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:12.208 INFO Sent repair work, now wait for resp
Sep 22 23:15:12.208 INFO [0] received reconcile message
Sep 22 23:15:12.208 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(442), op: ExtentRepair { repair_id: ReconciliationId(442), extent_id: 171, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:12.208 INFO [0] client ExtentRepair { repair_id: ReconciliationId(442), extent_id: 171, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:12.208 INFO [0] Sending repair request ReconciliationId(442)
Sep 22 23:15:12.208 INFO [1] received reconcile message
Sep 22 23:15:12.208 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(442), op: ExtentRepair { repair_id: ReconciliationId(442), extent_id: 171, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:12.208 INFO [1] client ExtentRepair { repair_id: ReconciliationId(442), extent_id: 171, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:12.208 INFO [1] No action required ReconciliationId(442)
Sep 22 23:15:12.208 INFO [2] received reconcile message
Sep 22 23:15:12.208 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(442), op: ExtentRepair { repair_id: ReconciliationId(442), extent_id: 171, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:12.208 INFO [2] client ExtentRepair { repair_id: ReconciliationId(442), extent_id: 171, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:12.208 INFO [2] No action required ReconciliationId(442)
Sep 22 23:15:12.208 DEBG 442 Repair extent 171 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:12.208 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/0AB.copy"
Sep 22 23:15:12.261 DEBG [2] Read AckReady 1084, : downstairs
Sep 22 23:15:12.262 DEBG up_ds_listen was notified
Sep 22 23:15:12.262 DEBG up_ds_listen process 1084
Sep 22 23:15:12.262 DEBG [A] ack job 1084:85, : downstairs
Sep 22 23:15:12.272 INFO accepted connection, remote_addr: 127.0.0.1:36145, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.272 TRCE incoming request, uri: /extent/171/files, method: GET, req_id: 0384f40a-e41b-4027-9f1c-d85df518581e, remote_addr: 127.0.0.1:36145, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.273 INFO request completed, latency_us: 193, response_code: 200, uri: /extent/171/files, method: GET, req_id: 0384f40a-e41b-4027-9f1c-d85df518581e, remote_addr: 127.0.0.1:36145, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.273 INFO eid:171 Found repair files: ["0AB", "0AB.db"]
Sep 22 23:15:12.273 TRCE incoming request, uri: /newextent/171/data, method: GET, req_id: 77719b3b-f1ed-42a5-9a14-6e190e6edf99, remote_addr: 127.0.0.1:36145, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.273 INFO request completed, latency_us: 319, response_code: 200, uri: /newextent/171/data, method: GET, req_id: 77719b3b-f1ed-42a5-9a14-6e190e6edf99, remote_addr: 127.0.0.1:36145, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.279 TRCE incoming request, uri: /newextent/171/db, method: GET, req_id: 16d4f5ca-9b40-42f1-b429-46f94f21af4c, remote_addr: 127.0.0.1:36145, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.279 INFO request completed, latency_us: 295, response_code: 200, uri: /newextent/171/db, method: GET, req_id: 16d4f5ca-9b40-42f1-b429-46f94f21af4c, remote_addr: 127.0.0.1:36145, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.280 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/0AB.copy" to "/tmp/downstairs-vrx8aK6L/00/000/0AB.replace"
Sep 22 23:15:12.280 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:12.281 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/0AB.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:12.281 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/0AB"
Sep 22 23:15:12.281 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/0AB.db"
Sep 22 23:15:12.281 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:12.281 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/0AB.replace" to "/tmp/downstairs-vrx8aK6L/00/000/0AB.completed"
Sep 22 23:15:12.281 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:12.281 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:12.282 DEBG [0] It's time to notify for 442
Sep 22 23:15:12.282 INFO Completion from [0] id:442 status:true
Sep 22 23:15:12.282 INFO [443/752] Repair commands completed
Sep 22 23:15:12.282 INFO Pop front: ReconcileIO { id: ReconciliationId(443), op: ExtentReopen { repair_id: ReconciliationId(443), extent_id: 171 }, state: ClientData([New, New, New]) }
Sep 22 23:15:12.282 INFO Sent repair work, now wait for resp
Sep 22 23:15:12.282 INFO [0] received reconcile message
Sep 22 23:15:12.282 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(443), op: ExtentReopen { repair_id: ReconciliationId(443), extent_id: 171 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:12.282 INFO [0] client ExtentReopen { repair_id: ReconciliationId(443), extent_id: 171 }
Sep 22 23:15:12.282 INFO [1] received reconcile message
Sep 22 23:15:12.282 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(443), op: ExtentReopen { repair_id: ReconciliationId(443), extent_id: 171 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:12.282 INFO [1] client ExtentReopen { repair_id: ReconciliationId(443), extent_id: 171 }
Sep 22 23:15:12.282 INFO [2] received reconcile message
Sep 22 23:15:12.282 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(443), op: ExtentReopen { repair_id: ReconciliationId(443), extent_id: 171 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:12.282 INFO [2] client ExtentReopen { repair_id: ReconciliationId(443), extent_id: 171 }
Sep 22 23:15:12.282 DEBG 443 Reopen extent 171
Sep 22 23:15:12.283 DEBG 443 Reopen extent 171
Sep 22 23:15:12.283 DEBG 443 Reopen extent 171
Sep 22 23:15:12.284 DEBG [2] It's time to notify for 443
Sep 22 23:15:12.284 INFO Completion from [2] id:443 status:true
Sep 22 23:15:12.284 INFO [444/752] Repair commands completed
Sep 22 23:15:12.284 INFO Pop front: ReconcileIO { id: ReconciliationId(444), op: ExtentFlush { repair_id: ReconciliationId(444), extent_id: 42, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:12.284 INFO Sent repair work, now wait for resp
Sep 22 23:15:12.284 INFO [0] received reconcile message
Sep 22 23:15:12.284 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(444), op: ExtentFlush { repair_id: ReconciliationId(444), extent_id: 42, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:12.284 INFO [0] client ExtentFlush { repair_id: ReconciliationId(444), extent_id: 42, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:12.284 INFO [1] received reconcile message
Sep 22 23:15:12.284 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(444), op: ExtentFlush { repair_id: ReconciliationId(444), extent_id: 42, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:12.284 INFO [1] client ExtentFlush { repair_id: ReconciliationId(444), extent_id: 42, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:12.284 INFO [2] received reconcile message
Sep 22 23:15:12.284 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(444), op: ExtentFlush { repair_id: ReconciliationId(444), extent_id: 42, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:12.284 INFO [2] client ExtentFlush { repair_id: ReconciliationId(444), extent_id: 42, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:12.284 DEBG 444 Flush extent 42 with f:2 g:2
Sep 22 23:15:12.284 DEBG Flush just extent 42 with f:2 and g:2
Sep 22 23:15:12.284 DEBG [1] It's time to notify for 444
Sep 22 23:15:12.284 INFO Completion from [1] id:444 status:true
Sep 22 23:15:12.284 INFO [445/752] Repair commands completed
Sep 22 23:15:12.284 INFO Pop front: ReconcileIO { id: ReconciliationId(445), op: ExtentClose { repair_id: ReconciliationId(445), extent_id: 42 }, state: ClientData([New, New, New]) }
Sep 22 23:15:12.284 INFO Sent repair work, now wait for resp
Sep 22 23:15:12.285 INFO [0] received reconcile message
Sep 22 23:15:12.285 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(445), op: ExtentClose { repair_id: ReconciliationId(445), extent_id: 42 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:12.285 INFO [0] client ExtentClose { repair_id: ReconciliationId(445), extent_id: 42 }
Sep 22 23:15:12.285 INFO [1] received reconcile message
Sep 22 23:15:12.285 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(445), op: ExtentClose { repair_id: ReconciliationId(445), extent_id: 42 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:12.285 INFO [1] client ExtentClose { repair_id: ReconciliationId(445), extent_id: 42 }
Sep 22 23:15:12.285 INFO [2] received reconcile message
Sep 22 23:15:12.285 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(445), op: ExtentClose { repair_id: ReconciliationId(445), extent_id: 42 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:12.285 INFO [2] client ExtentClose { repair_id: ReconciliationId(445), extent_id: 42 }
Sep 22 23:15:12.285 DEBG 445 Close extent 42
Sep 22 23:15:12.285 DEBG 445 Close extent 42
Sep 22 23:15:12.285 DEBG 445 Close extent 42
Sep 22 23:15:12.286 DEBG [2] It's time to notify for 445
Sep 22 23:15:12.286 INFO Completion from [2] id:445 status:true
Sep 22 23:15:12.286 INFO [446/752] Repair commands completed
Sep 22 23:15:12.286 INFO Pop front: ReconcileIO { id: ReconciliationId(446), op: ExtentRepair { repair_id: ReconciliationId(446), extent_id: 42, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:12.286 INFO Sent repair work, now wait for resp
Sep 22 23:15:12.286 INFO [0] received reconcile message
Sep 22 23:15:12.286 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(446), op: ExtentRepair { repair_id: ReconciliationId(446), extent_id: 42, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:12.286 INFO [0] client ExtentRepair { repair_id: ReconciliationId(446), extent_id: 42, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:12.286 INFO [0] Sending repair request ReconciliationId(446)
Sep 22 23:15:12.286 INFO [1] received reconcile message
Sep 22 23:15:12.286 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(446), op: ExtentRepair { repair_id: ReconciliationId(446), extent_id: 42, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:12.286 INFO [1] client ExtentRepair { repair_id: ReconciliationId(446), extent_id: 42, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:12.286 INFO [1] No action required ReconciliationId(446)
Sep 22 23:15:12.286 INFO [2] received reconcile message
Sep 22 23:15:12.286 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(446), op: ExtentRepair { repair_id: ReconciliationId(446), extent_id: 42, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:12.286 INFO [2] client ExtentRepair { repair_id: ReconciliationId(446), extent_id: 42, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:12.286 INFO [2] No action required ReconciliationId(446)
Sep 22 23:15:12.286 DEBG 446 Repair extent 42 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:12.286 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/02A.copy"
Sep 22 23:15:12.315 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:12.352 INFO accepted connection, remote_addr: 127.0.0.1:45360, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.352 TRCE incoming request, uri: /extent/42/files, method: GET, req_id: c2ffd996-3e6f-4578-bf41-dd130016f786, remote_addr: 127.0.0.1:45360, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.352 INFO request completed, latency_us: 275, response_code: 200, uri: /extent/42/files, method: GET, req_id: c2ffd996-3e6f-4578-bf41-dd130016f786, remote_addr: 127.0.0.1:45360, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.353 INFO eid:42 Found repair files: ["02A", "02A.db"]
Sep 22 23:15:12.353 TRCE incoming request, uri: /newextent/42/data, method: GET, req_id: 3e49bf1a-0e17-49a6-9022-dfbc80a4c546, remote_addr: 127.0.0.1:45360, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.353 INFO request completed, latency_us: 368, response_code: 200, uri: /newextent/42/data, method: GET, req_id: 3e49bf1a-0e17-49a6-9022-dfbc80a4c546, remote_addr: 127.0.0.1:45360, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.359 TRCE incoming request, uri: /newextent/42/db, method: GET, req_id: 82b06844-40ca-46f1-9a79-af7401d87714, remote_addr: 127.0.0.1:45360, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.359 INFO request completed, latency_us: 306, response_code: 200, uri: /newextent/42/db, method: GET, req_id: 82b06844-40ca-46f1-9a79-af7401d87714, remote_addr: 127.0.0.1:45360, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.360 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/02A.copy" to "/tmp/downstairs-vrx8aK6L/00/000/02A.replace"
Sep 22 23:15:12.360 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:12.362 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/02A.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:12.362 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/02A"
Sep 22 23:15:12.362 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/02A.db"
Sep 22 23:15:12.362 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:12.362 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/02A.replace" to "/tmp/downstairs-vrx8aK6L/00/000/02A.completed"
Sep 22 23:15:12.362 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:12.362 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:12.362 DEBG [0] It's time to notify for 446
Sep 22 23:15:12.362 INFO Completion from [0] id:446 status:true
Sep 22 23:15:12.362 INFO [447/752] Repair commands completed
Sep 22 23:15:12.362 INFO Pop front: ReconcileIO { id: ReconciliationId(447), op: ExtentReopen { repair_id: ReconciliationId(447), extent_id: 42 }, state: ClientData([New, New, New]) }
Sep 22 23:15:12.363 INFO Sent repair work, now wait for resp
Sep 22 23:15:12.363 INFO [0] received reconcile message
Sep 22 23:15:12.363 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(447), op: ExtentReopen { repair_id: ReconciliationId(447), extent_id: 42 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:12.363 INFO [0] client ExtentReopen { repair_id: ReconciliationId(447), extent_id: 42 }
Sep 22 23:15:12.363 INFO [1] received reconcile message
Sep 22 23:15:12.363 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(447), op: ExtentReopen { repair_id: ReconciliationId(447), extent_id: 42 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:12.363 INFO [1] client ExtentReopen { repair_id: ReconciliationId(447), extent_id: 42 }
Sep 22 23:15:12.363 INFO [2] received reconcile message
Sep 22 23:15:12.363 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(447), op: ExtentReopen { repair_id: ReconciliationId(447), extent_id: 42 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:12.363 INFO [2] client ExtentReopen { repair_id: ReconciliationId(447), extent_id: 42 }
Sep 22 23:15:12.363 DEBG 447 Reopen extent 42
Sep 22 23:15:12.364 DEBG 447 Reopen extent 42
Sep 22 23:15:12.364 DEBG 447 Reopen extent 42
Sep 22 23:15:12.365 DEBG [2] It's time to notify for 447
Sep 22 23:15:12.365 INFO Completion from [2] id:447 status:true
Sep 22 23:15:12.365 INFO [448/752] Repair commands completed
Sep 22 23:15:12.365 INFO Pop front: ReconcileIO { id: ReconciliationId(448), op: ExtentFlush { repair_id: ReconciliationId(448), extent_id: 43, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:12.365 INFO Sent repair work, now wait for resp
Sep 22 23:15:12.365 INFO [0] received reconcile message
Sep 22 23:15:12.365 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(448), op: ExtentFlush { repair_id: ReconciliationId(448), extent_id: 43, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:12.365 INFO [0] client ExtentFlush { repair_id: ReconciliationId(448), extent_id: 43, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:12.365 INFO [1] received reconcile message
Sep 22 23:15:12.365 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(448), op: ExtentFlush { repair_id: ReconciliationId(448), extent_id: 43, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:12.365 INFO [1] client ExtentFlush { repair_id: ReconciliationId(448), extent_id: 43, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:12.365 INFO [2] received reconcile message
Sep 22 23:15:12.365 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(448), op: ExtentFlush { repair_id: ReconciliationId(448), extent_id: 43, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:12.365 INFO [2] client ExtentFlush { repair_id: ReconciliationId(448), extent_id: 43, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:12.365 DEBG 448 Flush extent 43 with f:2 g:2
Sep 22 23:15:12.365 DEBG Flush just extent 43 with f:2 and g:2
Sep 22 23:15:12.366 DEBG [1] It's time to notify for 448
Sep 22 23:15:12.366 INFO Completion from [1] id:448 status:true
Sep 22 23:15:12.366 INFO [449/752] Repair commands completed
Sep 22 23:15:12.366 INFO Pop front: ReconcileIO { id: ReconciliationId(449), op: ExtentClose { repair_id: ReconciliationId(449), extent_id: 43 }, state: ClientData([New, New, New]) }
Sep 22 23:15:12.366 INFO Sent repair work, now wait for resp
Sep 22 23:15:12.366 INFO [0] received reconcile message
Sep 22 23:15:12.366 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(449), op: ExtentClose { repair_id: ReconciliationId(449), extent_id: 43 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:12.366 INFO [0] client ExtentClose { repair_id: ReconciliationId(449), extent_id: 43 }
Sep 22 23:15:12.366 INFO [lossy] sleeping 1 second
Sep 22 23:15:12.366 INFO [1] received reconcile message
Sep 22 23:15:12.366 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(449), op: ExtentClose { repair_id: ReconciliationId(449), extent_id: 43 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:12.366 INFO [1] client ExtentClose { repair_id: ReconciliationId(449), extent_id: 43 }
Sep 22 23:15:12.366 INFO [2] received reconcile message
Sep 22 23:15:12.366 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(449), op: ExtentClose { repair_id: ReconciliationId(449), extent_id: 43 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:12.366 INFO [2] client ExtentClose { repair_id: ReconciliationId(449), extent_id: 43 }
Sep 22 23:15:12.366 DEBG 449 Close extent 43
Sep 22 23:15:12.366 DEBG 449 Close extent 43
Sep 22 23:15:12.367 DEBG 449 Close extent 43
Sep 22 23:15:12.367 DEBG [2] It's time to notify for 449
Sep 22 23:15:12.367 INFO Completion from [2] id:449 status:true
Sep 22 23:15:12.367 INFO [450/752] Repair commands completed
Sep 22 23:15:12.367 INFO Pop front: ReconcileIO { id: ReconciliationId(450), op: ExtentRepair { repair_id: ReconciliationId(450), extent_id: 43, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:12.367 INFO Sent repair work, now wait for resp
Sep 22 23:15:12.367 INFO [0] received reconcile message
Sep 22 23:15:12.367 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(450), op: ExtentRepair { repair_id: ReconciliationId(450), extent_id: 43, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:12.367 INFO [0] client ExtentRepair { repair_id: ReconciliationId(450), extent_id: 43, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:12.367 INFO [0] Sending repair request ReconciliationId(450)
Sep 22 23:15:12.367 INFO [1] received reconcile message
Sep 22 23:15:12.367 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(450), op: ExtentRepair { repair_id: ReconciliationId(450), extent_id: 43, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:12.367 INFO [1] client ExtentRepair { repair_id: ReconciliationId(450), extent_id: 43, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:12.367 INFO [1] No action required ReconciliationId(450)
Sep 22 23:15:12.367 INFO [2] received reconcile message
Sep 22 23:15:12.367 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(450), op: ExtentRepair { repair_id: ReconciliationId(450), extent_id: 43, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:12.367 INFO [2] client ExtentRepair { repair_id: ReconciliationId(450), extent_id: 43, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:12.367 INFO [2] No action required ReconciliationId(450)
Sep 22 23:15:12.367 DEBG 450 Repair extent 43 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:12.368 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/02B.copy"
Sep 22 23:15:12.430 INFO accepted connection, remote_addr: 127.0.0.1:60105, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.431 TRCE incoming request, uri: /extent/43/files, method: GET, req_id: 0879bb4d-8f86-4f18-ad7a-4e7cde96d56d, remote_addr: 127.0.0.1:60105, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.431 INFO request completed, latency_us: 253, response_code: 200, uri: /extent/43/files, method: GET, req_id: 0879bb4d-8f86-4f18-ad7a-4e7cde96d56d, remote_addr: 127.0.0.1:60105, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.431 INFO eid:43 Found repair files: ["02B", "02B.db"]
Sep 22 23:15:12.432 TRCE incoming request, uri: /newextent/43/data, method: GET, req_id: 49fcb992-9aa5-493b-b23c-500294aa746d, remote_addr: 127.0.0.1:60105, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.432 INFO request completed, latency_us: 360, response_code: 200, uri: /newextent/43/data, method: GET, req_id: 49fcb992-9aa5-493b-b23c-500294aa746d, remote_addr: 127.0.0.1:60105, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.437 TRCE incoming request, uri: /newextent/43/db, method: GET, req_id: caa411d5-6ee2-453e-b17a-79e7764b89ad, remote_addr: 127.0.0.1:60105, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.437 INFO request completed, latency_us: 294, response_code: 200, uri: /newextent/43/db, method: GET, req_id: caa411d5-6ee2-453e-b17a-79e7764b89ad, remote_addr: 127.0.0.1:60105, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.439 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/02B.copy" to "/tmp/downstairs-vrx8aK6L/00/000/02B.replace"
Sep 22 23:15:12.439 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:12.440 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/02B.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:12.440 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/02B"
Sep 22 23:15:12.440 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/02B.db"
Sep 22 23:15:12.440 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:12.440 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/02B.replace" to "/tmp/downstairs-vrx8aK6L/00/000/02B.completed"
Sep 22 23:15:12.440 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:12.440 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:12.440 DEBG [0] It's time to notify for 450
Sep 22 23:15:12.441 INFO Completion from [0] id:450 status:true
Sep 22 23:15:12.441 INFO [451/752] Repair commands completed
Sep 22 23:15:12.441 INFO Pop front: ReconcileIO { id: ReconciliationId(451), op: ExtentReopen { repair_id: ReconciliationId(451), extent_id: 43 }, state: ClientData([New, New, New]) }
Sep 22 23:15:12.441 INFO Sent repair work, now wait for resp
Sep 22 23:15:12.441 INFO [0] received reconcile message
Sep 22 23:15:12.441 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(451), op: ExtentReopen { repair_id: ReconciliationId(451), extent_id: 43 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:12.441 INFO [0] client ExtentReopen { repair_id: ReconciliationId(451), extent_id: 43 }
Sep 22 23:15:12.441 INFO [1] received reconcile message
Sep 22 23:15:12.441 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(451), op: ExtentReopen { repair_id: ReconciliationId(451), extent_id: 43 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:12.441 INFO [1] client ExtentReopen { repair_id: ReconciliationId(451), extent_id: 43 }
Sep 22 23:15:12.441 INFO [2] received reconcile message
Sep 22 23:15:12.441 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(451), op: ExtentReopen { repair_id: ReconciliationId(451), extent_id: 43 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:12.441 INFO [2] client ExtentReopen { repair_id: ReconciliationId(451), extent_id: 43 }
Sep 22 23:15:12.441 DEBG 451 Reopen extent 43
Sep 22 23:15:12.442 DEBG 451 Reopen extent 43
Sep 22 23:15:12.442 DEBG 451 Reopen extent 43
Sep 22 23:15:12.443 DEBG [2] It's time to notify for 451
Sep 22 23:15:12.443 INFO Completion from [2] id:451 status:true
Sep 22 23:15:12.443 INFO [452/752] Repair commands completed
Sep 22 23:15:12.443 INFO Pop front: ReconcileIO { id: ReconciliationId(452), op: ExtentFlush { repair_id: ReconciliationId(452), extent_id: 179, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:12.443 INFO Sent repair work, now wait for resp
Sep 22 23:15:12.443 INFO [0] received reconcile message
Sep 22 23:15:12.443 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(452), op: ExtentFlush { repair_id: ReconciliationId(452), extent_id: 179, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:12.443 INFO [0] client ExtentFlush { repair_id: ReconciliationId(452), extent_id: 179, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:12.443 INFO [1] received reconcile message
Sep 22 23:15:12.443 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(452), op: ExtentFlush { repair_id: ReconciliationId(452), extent_id: 179, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:12.443 INFO [1] client ExtentFlush { repair_id: ReconciliationId(452), extent_id: 179, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:12.443 INFO [2] received reconcile message
Sep 22 23:15:12.443 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(452), op: ExtentFlush { repair_id: ReconciliationId(452), extent_id: 179, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:12.443 INFO [2] client ExtentFlush { repair_id: ReconciliationId(452), extent_id: 179, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:12.443 DEBG 452 Flush extent 179 with f:2 g:2
Sep 22 23:15:12.443 DEBG Flush just extent 179 with f:2 and g:2
Sep 22 23:15:12.444 DEBG [1] It's time to notify for 452
Sep 22 23:15:12.444 INFO Completion from [1] id:452 status:true
Sep 22 23:15:12.444 INFO [453/752] Repair commands completed
Sep 22 23:15:12.444 INFO Pop front: ReconcileIO { id: ReconciliationId(453), op: ExtentClose { repair_id: ReconciliationId(453), extent_id: 179 }, state: ClientData([New, New, New]) }
Sep 22 23:15:12.444 INFO Sent repair work, now wait for resp
Sep 22 23:15:12.444 INFO [0] received reconcile message
Sep 22 23:15:12.444 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(453), op: ExtentClose { repair_id: ReconciliationId(453), extent_id: 179 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:12.444 INFO [0] client ExtentClose { repair_id: ReconciliationId(453), extent_id: 179 }
Sep 22 23:15:12.444 INFO [1] received reconcile message
Sep 22 23:15:12.444 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(453), op: ExtentClose { repair_id: ReconciliationId(453), extent_id: 179 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:12.444 INFO [1] client ExtentClose { repair_id: ReconciliationId(453), extent_id: 179 }
Sep 22 23:15:12.444 INFO [2] received reconcile message
Sep 22 23:15:12.444 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(453), op: ExtentClose { repair_id: ReconciliationId(453), extent_id: 179 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:12.444 INFO [2] client ExtentClose { repair_id: ReconciliationId(453), extent_id: 179 }
Sep 22 23:15:12.444 DEBG 453 Close extent 179
Sep 22 23:15:12.444 DEBG 453 Close extent 179
Sep 22 23:15:12.445 DEBG 453 Close extent 179
Sep 22 23:15:12.445 DEBG [2] It's time to notify for 453
Sep 22 23:15:12.445 INFO Completion from [2] id:453 status:true
Sep 22 23:15:12.445 INFO [454/752] Repair commands completed
Sep 22 23:15:12.445 INFO Pop front: ReconcileIO { id: ReconciliationId(454), op: ExtentRepair { repair_id: ReconciliationId(454), extent_id: 179, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:12.445 INFO Sent repair work, now wait for resp
Sep 22 23:15:12.445 INFO [0] received reconcile message
Sep 22 23:15:12.445 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(454), op: ExtentRepair { repair_id: ReconciliationId(454), extent_id: 179, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:12.445 INFO [0] client ExtentRepair { repair_id: ReconciliationId(454), extent_id: 179, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:12.445 INFO [0] Sending repair request ReconciliationId(454)
Sep 22 23:15:12.445 INFO [1] received reconcile message
Sep 22 23:15:12.445 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(454), op: ExtentRepair { repair_id: ReconciliationId(454), extent_id: 179, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:12.445 INFO [1] client ExtentRepair { repair_id: ReconciliationId(454), extent_id: 179, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:12.445 INFO [1] No action required ReconciliationId(454)
Sep 22 23:15:12.445 INFO [2] received reconcile message
Sep 22 23:15:12.445 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(454), op: ExtentRepair { repair_id: ReconciliationId(454), extent_id: 179, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:12.445 INFO [2] client ExtentRepair { repair_id: ReconciliationId(454), extent_id: 179, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:12.445 INFO [2] No action required ReconciliationId(454)
Sep 22 23:15:12.446 DEBG 454 Repair extent 179 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:12.446 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/0B3.copy"
Sep 22 23:15:12.508 INFO accepted connection, remote_addr: 127.0.0.1:60795, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.508 TRCE incoming request, uri: /extent/179/files, method: GET, req_id: 878cbee6-1c05-44da-8360-a056c841fce5, remote_addr: 127.0.0.1:60795, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.509 INFO request completed, latency_us: 242, response_code: 200, uri: /extent/179/files, method: GET, req_id: 878cbee6-1c05-44da-8360-a056c841fce5, remote_addr: 127.0.0.1:60795, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.509 INFO eid:179 Found repair files: ["0B3", "0B3.db"]
Sep 22 23:15:12.509 TRCE incoming request, uri: /newextent/179/data, method: GET, req_id: ae584d82-9e6c-4193-b01d-0959a1a66636, remote_addr: 127.0.0.1:60795, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.510 INFO request completed, latency_us: 333, response_code: 200, uri: /newextent/179/data, method: GET, req_id: ae584d82-9e6c-4193-b01d-0959a1a66636, remote_addr: 127.0.0.1:60795, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.515 TRCE incoming request, uri: /newextent/179/db, method: GET, req_id: e42a6c40-27c6-40a3-b6ac-30f694ae7c3d, remote_addr: 127.0.0.1:60795, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.515 INFO request completed, latency_us: 307, response_code: 200, uri: /newextent/179/db, method: GET, req_id: e42a6c40-27c6-40a3-b6ac-30f694ae7c3d, remote_addr: 127.0.0.1:60795, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.516 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/0B3.copy" to "/tmp/downstairs-vrx8aK6L/00/000/0B3.replace"
Sep 22 23:15:12.516 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:12.517 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/0B3.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:12.518 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/0B3"
Sep 22 23:15:12.518 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/0B3.db"
Sep 22 23:15:12.518 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:12.518 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/0B3.replace" to "/tmp/downstairs-vrx8aK6L/00/000/0B3.completed"
Sep 22 23:15:12.518 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:12.518 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:12.518 DEBG [0] It's time to notify for 454
Sep 22 23:15:12.518 INFO Completion from [0] id:454 status:true
Sep 22 23:15:12.518 INFO [455/752] Repair commands completed
Sep 22 23:15:12.518 INFO Pop front: ReconcileIO { id: ReconciliationId(455), op: ExtentReopen { repair_id: ReconciliationId(455), extent_id: 179 }, state: ClientData([New, New, New]) }
Sep 22 23:15:12.518 INFO Sent repair work, now wait for resp
Sep 22 23:15:12.518 INFO [0] received reconcile message
Sep 22 23:15:12.518 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(455), op: ExtentReopen { repair_id: ReconciliationId(455), extent_id: 179 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:12.518 INFO [0] client ExtentReopen { repair_id: ReconciliationId(455), extent_id: 179 }
Sep 22 23:15:12.519 INFO [1] received reconcile message
Sep 22 23:15:12.519 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(455), op: ExtentReopen { repair_id: ReconciliationId(455), extent_id: 179 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:12.519 INFO [1] client ExtentReopen { repair_id: ReconciliationId(455), extent_id: 179 }
Sep 22 23:15:12.519 INFO [2] received reconcile message
Sep 22 23:15:12.519 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(455), op: ExtentReopen { repair_id: ReconciliationId(455), extent_id: 179 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:12.519 INFO [2] client ExtentReopen { repair_id: ReconciliationId(455), extent_id: 179 }
Sep 22 23:15:12.519 DEBG 455 Reopen extent 179
Sep 22 23:15:12.520 DEBG 455 Reopen extent 179
Sep 22 23:15:12.520 DEBG 455 Reopen extent 179
Sep 22 23:15:12.521 DEBG [2] It's time to notify for 455
Sep 22 23:15:12.521 INFO Completion from [2] id:455 status:true
Sep 22 23:15:12.521 INFO [456/752] Repair commands completed
Sep 22 23:15:12.521 INFO Pop front: ReconcileIO { id: ReconciliationId(456), op: ExtentFlush { repair_id: ReconciliationId(456), extent_id: 120, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:12.521 INFO Sent repair work, now wait for resp
Sep 22 23:15:12.521 INFO [0] received reconcile message
Sep 22 23:15:12.521 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(456), op: ExtentFlush { repair_id: ReconciliationId(456), extent_id: 120, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:12.521 INFO [0] client ExtentFlush { repair_id: ReconciliationId(456), extent_id: 120, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:12.521 INFO [1] received reconcile message
Sep 22 23:15:12.521 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(456), op: ExtentFlush { repair_id: ReconciliationId(456), extent_id: 120, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:12.521 INFO [1] client ExtentFlush { repair_id: ReconciliationId(456), extent_id: 120, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:12.521 INFO [2] received reconcile message
Sep 22 23:15:12.521 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(456), op: ExtentFlush { repair_id: ReconciliationId(456), extent_id: 120, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:12.521 INFO [2] client ExtentFlush { repair_id: ReconciliationId(456), extent_id: 120, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:12.521 DEBG 456 Flush extent 120 with f:2 g:2
Sep 22 23:15:12.521 DEBG Flush just extent 120 with f:2 and g:2
Sep 22 23:15:12.521 DEBG [1] It's time to notify for 456
Sep 22 23:15:12.521 INFO Completion from [1] id:456 status:true
Sep 22 23:15:12.521 INFO [457/752] Repair commands completed
Sep 22 23:15:12.521 INFO Pop front: ReconcileIO { id: ReconciliationId(457), op: ExtentClose { repair_id: ReconciliationId(457), extent_id: 120 }, state: ClientData([New, New, New]) }
Sep 22 23:15:12.521 INFO Sent repair work, now wait for resp
Sep 22 23:15:12.521 INFO [0] received reconcile message
Sep 22 23:15:12.521 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(457), op: ExtentClose { repair_id: ReconciliationId(457), extent_id: 120 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:12.521 INFO [0] client ExtentClose { repair_id: ReconciliationId(457), extent_id: 120 }
Sep 22 23:15:12.521 INFO [1] received reconcile message
Sep 22 23:15:12.521 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(457), op: ExtentClose { repair_id: ReconciliationId(457), extent_id: 120 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:12.521 INFO [1] client ExtentClose { repair_id: ReconciliationId(457), extent_id: 120 }
Sep 22 23:15:12.522 INFO [2] received reconcile message
Sep 22 23:15:12.522 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(457), op: ExtentClose { repair_id: ReconciliationId(457), extent_id: 120 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:12.522 INFO [2] client ExtentClose { repair_id: ReconciliationId(457), extent_id: 120 }
Sep 22 23:15:12.522 DEBG 457 Close extent 120
Sep 22 23:15:12.522 DEBG 457 Close extent 120
Sep 22 23:15:12.522 DEBG 457 Close extent 120
Sep 22 23:15:12.523 DEBG [2] It's time to notify for 457
Sep 22 23:15:12.523 INFO Completion from [2] id:457 status:true
Sep 22 23:15:12.523 INFO [458/752] Repair commands completed
Sep 22 23:15:12.523 INFO Pop front: ReconcileIO { id: ReconciliationId(458), op: ExtentRepair { repair_id: ReconciliationId(458), extent_id: 120, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:12.523 INFO Sent repair work, now wait for resp
Sep 22 23:15:12.523 INFO [0] received reconcile message
Sep 22 23:15:12.523 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(458), op: ExtentRepair { repair_id: ReconciliationId(458), extent_id: 120, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:12.523 INFO [0] client ExtentRepair { repair_id: ReconciliationId(458), extent_id: 120, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:12.523 INFO [0] Sending repair request ReconciliationId(458)
Sep 22 23:15:12.523 INFO [1] received reconcile message
Sep 22 23:15:12.523 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(458), op: ExtentRepair { repair_id: ReconciliationId(458), extent_id: 120, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:12.523 INFO [1] client ExtentRepair { repair_id: ReconciliationId(458), extent_id: 120, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:12.523 INFO [1] No action required ReconciliationId(458)
Sep 22 23:15:12.523 INFO [2] received reconcile message
Sep 22 23:15:12.523 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(458), op: ExtentRepair { repair_id: ReconciliationId(458), extent_id: 120, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:12.523 INFO [2] client ExtentRepair { repair_id: ReconciliationId(458), extent_id: 120, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:12.523 INFO [2] No action required ReconciliationId(458)
Sep 22 23:15:12.523 DEBG 458 Repair extent 120 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:12.523 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/078.copy"
Sep 22 23:15:12.589 INFO accepted connection, remote_addr: 127.0.0.1:36560, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.589 TRCE incoming request, uri: /extent/120/files, method: GET, req_id: 07f7762c-3d9a-4087-b476-155d883c58fe, remote_addr: 127.0.0.1:36560, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.589 INFO request completed, latency_us: 190, response_code: 200, uri: /extent/120/files, method: GET, req_id: 07f7762c-3d9a-4087-b476-155d883c58fe, remote_addr: 127.0.0.1:36560, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.590 INFO eid:120 Found repair files: ["078", "078.db"]
Sep 22 23:15:12.590 TRCE incoming request, uri: /newextent/120/data, method: GET, req_id: 8fcf22ec-a2e2-43e9-9b1a-784b135be829, remote_addr: 127.0.0.1:36560, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.590 INFO request completed, latency_us: 361, response_code: 200, uri: /newextent/120/data, method: GET, req_id: 8fcf22ec-a2e2-43e9-9b1a-784b135be829, remote_addr: 127.0.0.1:36560, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.595 TRCE incoming request, uri: /newextent/120/db, method: GET, req_id: 3554c51b-4b10-4323-b0d8-d66bd66eebf3, remote_addr: 127.0.0.1:36560, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.596 INFO request completed, latency_us: 294, response_code: 200, uri: /newextent/120/db, method: GET, req_id: 3554c51b-4b10-4323-b0d8-d66bd66eebf3, remote_addr: 127.0.0.1:36560, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.597 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/078.copy" to "/tmp/downstairs-vrx8aK6L/00/000/078.replace"
Sep 22 23:15:12.597 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:12.598 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/078.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:12.598 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/078"
Sep 22 23:15:12.598 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/078.db"
Sep 22 23:15:12.598 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:12.598 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/078.replace" to "/tmp/downstairs-vrx8aK6L/00/000/078.completed"
Sep 22 23:15:12.598 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:12.598 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:12.598 DEBG [0] It's time to notify for 458
Sep 22 23:15:12.599 INFO Completion from [0] id:458 status:true
Sep 22 23:15:12.599 INFO [459/752] Repair commands completed
Sep 22 23:15:12.599 INFO Pop front: ReconcileIO { id: ReconciliationId(459), op: ExtentReopen { repair_id: ReconciliationId(459), extent_id: 120 }, state: ClientData([New, New, New]) }
Sep 22 23:15:12.599 INFO Sent repair work, now wait for resp
Sep 22 23:15:12.599 INFO [0] received reconcile message
Sep 22 23:15:12.599 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(459), op: ExtentReopen { repair_id: ReconciliationId(459), extent_id: 120 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:12.599 INFO [0] client ExtentReopen { repair_id: ReconciliationId(459), extent_id: 120 }
Sep 22 23:15:12.599 INFO [1] received reconcile message
Sep 22 23:15:12.599 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(459), op: ExtentReopen { repair_id: ReconciliationId(459), extent_id: 120 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:12.599 INFO [1] client ExtentReopen { repair_id: ReconciliationId(459), extent_id: 120 }
Sep 22 23:15:12.599 INFO [2] received reconcile message
Sep 22 23:15:12.599 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(459), op: ExtentReopen { repair_id: ReconciliationId(459), extent_id: 120 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:12.599 INFO [2] client ExtentReopen { repair_id: ReconciliationId(459), extent_id: 120 }
Sep 22 23:15:12.599 DEBG 459 Reopen extent 120
Sep 22 23:15:12.600 DEBG 459 Reopen extent 120
Sep 22 23:15:12.600 DEBG 459 Reopen extent 120
Sep 22 23:15:12.601 DEBG [2] It's time to notify for 459
Sep 22 23:15:12.601 INFO Completion from [2] id:459 status:true
Sep 22 23:15:12.601 INFO [460/752] Repair commands completed
Sep 22 23:15:12.601 INFO Pop front: ReconcileIO { id: ReconciliationId(460), op: ExtentFlush { repair_id: ReconciliationId(460), extent_id: 66, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:12.601 INFO Sent repair work, now wait for resp
Sep 22 23:15:12.601 INFO [0] received reconcile message
Sep 22 23:15:12.601 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(460), op: ExtentFlush { repair_id: ReconciliationId(460), extent_id: 66, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:12.601 INFO [0] client ExtentFlush { repair_id: ReconciliationId(460), extent_id: 66, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:12.601 INFO [1] received reconcile message
Sep 22 23:15:12.601 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(460), op: ExtentFlush { repair_id: ReconciliationId(460), extent_id: 66, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:12.601 INFO [1] client ExtentFlush { repair_id: ReconciliationId(460), extent_id: 66, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:12.601 INFO [2] received reconcile message
Sep 22 23:15:12.601 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(460), op: ExtentFlush { repair_id: ReconciliationId(460), extent_id: 66, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:12.601 INFO [2] client ExtentFlush { repair_id: ReconciliationId(460), extent_id: 66, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:12.601 DEBG 460 Flush extent 66 with f:2 g:2
Sep 22 23:15:12.601 DEBG Flush just extent 66 with f:2 and g:2
Sep 22 23:15:12.601 DEBG [1] It's time to notify for 460
Sep 22 23:15:12.601 INFO Completion from [1] id:460 status:true
Sep 22 23:15:12.601 INFO [461/752] Repair commands completed
Sep 22 23:15:12.601 INFO Pop front: ReconcileIO { id: ReconciliationId(461), op: ExtentClose { repair_id: ReconciliationId(461), extent_id: 66 }, state: ClientData([New, New, New]) }
Sep 22 23:15:12.601 INFO Sent repair work, now wait for resp
Sep 22 23:15:12.601 INFO [0] received reconcile message
Sep 22 23:15:12.601 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(461), op: ExtentClose { repair_id: ReconciliationId(461), extent_id: 66 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:12.601 INFO [0] client ExtentClose { repair_id: ReconciliationId(461), extent_id: 66 }
Sep 22 23:15:12.601 INFO [1] received reconcile message
Sep 22 23:15:12.601 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(461), op: ExtentClose { repair_id: ReconciliationId(461), extent_id: 66 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:12.601 INFO [1] client ExtentClose { repair_id: ReconciliationId(461), extent_id: 66 }
Sep 22 23:15:12.601 INFO [2] received reconcile message
Sep 22 23:15:12.602 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(461), op: ExtentClose { repair_id: ReconciliationId(461), extent_id: 66 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:12.602 INFO [2] client ExtentClose { repair_id: ReconciliationId(461), extent_id: 66 }
Sep 22 23:15:12.602 DEBG 461 Close extent 66
Sep 22 23:15:12.602 DEBG 461 Close extent 66
Sep 22 23:15:12.602 DEBG 461 Close extent 66
Sep 22 23:15:12.603 DEBG [2] It's time to notify for 461
Sep 22 23:15:12.603 INFO Completion from [2] id:461 status:true
Sep 22 23:15:12.603 INFO [462/752] Repair commands completed
Sep 22 23:15:12.603 INFO Pop front: ReconcileIO { id: ReconciliationId(462), op: ExtentRepair { repair_id: ReconciliationId(462), extent_id: 66, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:12.603 INFO Sent repair work, now wait for resp
Sep 22 23:15:12.603 INFO [0] received reconcile message
Sep 22 23:15:12.603 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(462), op: ExtentRepair { repair_id: ReconciliationId(462), extent_id: 66, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:12.603 INFO [0] client ExtentRepair { repair_id: ReconciliationId(462), extent_id: 66, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:12.603 INFO [0] Sending repair request ReconciliationId(462)
Sep 22 23:15:12.603 INFO [1] received reconcile message
Sep 22 23:15:12.603 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(462), op: ExtentRepair { repair_id: ReconciliationId(462), extent_id: 66, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:12.603 INFO [1] client ExtentRepair { repair_id: ReconciliationId(462), extent_id: 66, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:12.603 INFO [1] No action required ReconciliationId(462)
Sep 22 23:15:12.603 INFO [2] received reconcile message
Sep 22 23:15:12.603 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(462), op: ExtentRepair { repair_id: ReconciliationId(462), extent_id: 66, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:12.603 INFO [2] client ExtentRepair { repair_id: ReconciliationId(462), extent_id: 66, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:12.603 INFO [2] No action required ReconciliationId(462)
Sep 22 23:15:12.603 DEBG 462 Repair extent 66 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:12.603 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/042.copy"
Sep 22 23:15:12.668 INFO accepted connection, remote_addr: 127.0.0.1:49244, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.668 TRCE incoming request, uri: /extent/66/files, method: GET, req_id: bb53e584-c52f-4d56-bea5-b60d1d498be4, remote_addr: 127.0.0.1:49244, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.669 INFO request completed, latency_us: 212, response_code: 200, uri: /extent/66/files, method: GET, req_id: bb53e584-c52f-4d56-bea5-b60d1d498be4, remote_addr: 127.0.0.1:49244, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.669 INFO eid:66 Found repair files: ["042", "042.db"]
Sep 22 23:15:12.669 TRCE incoming request, uri: /newextent/66/data, method: GET, req_id: 3975a6d3-6f9e-407b-80dc-e05c7208bb73, remote_addr: 127.0.0.1:49244, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.670 INFO request completed, latency_us: 312, response_code: 200, uri: /newextent/66/data, method: GET, req_id: 3975a6d3-6f9e-407b-80dc-e05c7208bb73, remote_addr: 127.0.0.1:49244, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.675 TRCE incoming request, uri: /newextent/66/db, method: GET, req_id: d6fb1749-8f02-43b8-97a8-e702e3da6454, remote_addr: 127.0.0.1:49244, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.675 INFO request completed, latency_us: 307, response_code: 200, uri: /newextent/66/db, method: GET, req_id: d6fb1749-8f02-43b8-97a8-e702e3da6454, remote_addr: 127.0.0.1:49244, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.676 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/042.copy" to "/tmp/downstairs-vrx8aK6L/00/000/042.replace"
Sep 22 23:15:12.676 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:12.677 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/042.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:12.678 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/042"
Sep 22 23:15:12.678 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/042.db"
Sep 22 23:15:12.678 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:12.678 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/042.replace" to "/tmp/downstairs-vrx8aK6L/00/000/042.completed"
Sep 22 23:15:12.678 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:12.678 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:12.678 DEBG [0] It's time to notify for 462
Sep 22 23:15:12.678 INFO Completion from [0] id:462 status:true
Sep 22 23:15:12.678 INFO [463/752] Repair commands completed
Sep 22 23:15:12.678 INFO Pop front: ReconcileIO { id: ReconciliationId(463), op: ExtentReopen { repair_id: ReconciliationId(463), extent_id: 66 }, state: ClientData([New, New, New]) }
Sep 22 23:15:12.678 INFO Sent repair work, now wait for resp
Sep 22 23:15:12.678 INFO [0] received reconcile message
Sep 22 23:15:12.678 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(463), op: ExtentReopen { repair_id: ReconciliationId(463), extent_id: 66 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:12.678 INFO [0] client ExtentReopen { repair_id: ReconciliationId(463), extent_id: 66 }
Sep 22 23:15:12.678 INFO [1] received reconcile message
Sep 22 23:15:12.678 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(463), op: ExtentReopen { repair_id: ReconciliationId(463), extent_id: 66 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:12.678 INFO [1] client ExtentReopen { repair_id: ReconciliationId(463), extent_id: 66 }
Sep 22 23:15:12.678 INFO [2] received reconcile message
Sep 22 23:15:12.678 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(463), op: ExtentReopen { repair_id: ReconciliationId(463), extent_id: 66 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:12.678 INFO [2] client ExtentReopen { repair_id: ReconciliationId(463), extent_id: 66 }
Sep 22 23:15:12.679 DEBG 463 Reopen extent 66
Sep 22 23:15:12.679 DEBG 463 Reopen extent 66
Sep 22 23:15:12.680 DEBG 463 Reopen extent 66
Sep 22 23:15:12.680 DEBG [2] It's time to notify for 463
Sep 22 23:15:12.680 INFO Completion from [2] id:463 status:true
Sep 22 23:15:12.680 INFO [464/752] Repair commands completed
Sep 22 23:15:12.680 INFO Pop front: ReconcileIO { id: ReconciliationId(464), op: ExtentFlush { repair_id: ReconciliationId(464), extent_id: 132, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:12.680 INFO Sent repair work, now wait for resp
Sep 22 23:15:12.680 INFO [0] received reconcile message
Sep 22 23:15:12.681 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(464), op: ExtentFlush { repair_id: ReconciliationId(464), extent_id: 132, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:12.681 INFO [0] client ExtentFlush { repair_id: ReconciliationId(464), extent_id: 132, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:12.681 INFO [1] received reconcile message
Sep 22 23:15:12.681 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(464), op: ExtentFlush { repair_id: ReconciliationId(464), extent_id: 132, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:12.681 INFO [1] client ExtentFlush { repair_id: ReconciliationId(464), extent_id: 132, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:12.681 INFO [2] received reconcile message
Sep 22 23:15:12.681 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(464), op: ExtentFlush { repair_id: ReconciliationId(464), extent_id: 132, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:12.681 INFO [2] client ExtentFlush { repair_id: ReconciliationId(464), extent_id: 132, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:12.681 DEBG 464 Flush extent 132 with f:2 g:2
Sep 22 23:15:12.681 DEBG Flush just extent 132 with f:2 and g:2
Sep 22 23:15:12.681 DEBG [1] It's time to notify for 464
Sep 22 23:15:12.681 INFO Completion from [1] id:464 status:true
Sep 22 23:15:12.681 INFO [465/752] Repair commands completed
Sep 22 23:15:12.681 INFO Pop front: ReconcileIO { id: ReconciliationId(465), op: ExtentClose { repair_id: ReconciliationId(465), extent_id: 132 }, state: ClientData([New, New, New]) }
Sep 22 23:15:12.681 INFO Sent repair work, now wait for resp
Sep 22 23:15:12.681 INFO [0] received reconcile message
Sep 22 23:15:12.681 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(465), op: ExtentClose { repair_id: ReconciliationId(465), extent_id: 132 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:12.681 INFO [0] client ExtentClose { repair_id: ReconciliationId(465), extent_id: 132 }
Sep 22 23:15:12.681 INFO [1] received reconcile message
Sep 22 23:15:12.681 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(465), op: ExtentClose { repair_id: ReconciliationId(465), extent_id: 132 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:12.681 INFO [1] client ExtentClose { repair_id: ReconciliationId(465), extent_id: 132 }
Sep 22 23:15:12.681 INFO [2] received reconcile message
Sep 22 23:15:12.681 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(465), op: ExtentClose { repair_id: ReconciliationId(465), extent_id: 132 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:12.681 INFO [2] client ExtentClose { repair_id: ReconciliationId(465), extent_id: 132 }
Sep 22 23:15:12.681 DEBG 465 Close extent 132
Sep 22 23:15:12.682 DEBG 465 Close extent 132
Sep 22 23:15:12.682 DEBG 465 Close extent 132
Sep 22 23:15:12.682 DEBG [2] It's time to notify for 465
Sep 22 23:15:12.682 INFO Completion from [2] id:465 status:true
Sep 22 23:15:12.682 INFO [466/752] Repair commands completed
Sep 22 23:15:12.682 INFO Pop front: ReconcileIO { id: ReconciliationId(466), op: ExtentRepair { repair_id: ReconciliationId(466), extent_id: 132, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:12.682 INFO Sent repair work, now wait for resp
Sep 22 23:15:12.682 INFO [0] received reconcile message
Sep 22 23:15:12.683 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(466), op: ExtentRepair { repair_id: ReconciliationId(466), extent_id: 132, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:12.683 INFO [0] client ExtentRepair { repair_id: ReconciliationId(466), extent_id: 132, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:12.683 INFO [0] Sending repair request ReconciliationId(466)
Sep 22 23:15:12.683 INFO [1] received reconcile message
Sep 22 23:15:12.683 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(466), op: ExtentRepair { repair_id: ReconciliationId(466), extent_id: 132, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:12.683 INFO [1] client ExtentRepair { repair_id: ReconciliationId(466), extent_id: 132, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:12.683 INFO [1] No action required ReconciliationId(466)
Sep 22 23:15:12.683 INFO [2] received reconcile message
Sep 22 23:15:12.683 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(466), op: ExtentRepair { repair_id: ReconciliationId(466), extent_id: 132, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:12.683 INFO [2] client ExtentRepair { repair_id: ReconciliationId(466), extent_id: 132, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:12.683 INFO [2] No action required ReconciliationId(466)
Sep 22 23:15:12.683 DEBG 466 Repair extent 132 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:12.683 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/084.copy"
Sep 22 23:15:12.746 DEBG IO Read  1086 has deps [JobId(1085)]
Sep 22 23:15:12.746 DEBG IO Flush 1087 has deps [JobId(1086), JobId(1085)]
Sep 22 23:15:12.748 INFO accepted connection, remote_addr: 127.0.0.1:48116, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.748 TRCE incoming request, uri: /extent/132/files, method: GET, req_id: 5d0de0fb-7303-4501-8eed-302bd0a83ec6, remote_addr: 127.0.0.1:48116, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.748 INFO request completed, latency_us: 252, response_code: 200, uri: /extent/132/files, method: GET, req_id: 5d0de0fb-7303-4501-8eed-302bd0a83ec6, remote_addr: 127.0.0.1:48116, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.748 INFO eid:132 Found repair files: ["084", "084.db"]
Sep 22 23:15:12.749 TRCE incoming request, uri: /newextent/132/data, method: GET, req_id: 1b6dd1c2-37fd-4ce7-860f-09b98c69f38b, remote_addr: 127.0.0.1:48116, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.749 INFO request completed, latency_us: 262, response_code: 200, uri: /newextent/132/data, method: GET, req_id: 1b6dd1c2-37fd-4ce7-860f-09b98c69f38b, remote_addr: 127.0.0.1:48116, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.754 TRCE incoming request, uri: /newextent/132/db, method: GET, req_id: 080e55f9-196a-49e0-b985-c26fc829cc4e, remote_addr: 127.0.0.1:48116, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.755 INFO request completed, latency_us: 299, response_code: 200, uri: /newextent/132/db, method: GET, req_id: 080e55f9-196a-49e0-b985-c26fc829cc4e, remote_addr: 127.0.0.1:48116, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.756 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/084.copy" to "/tmp/downstairs-vrx8aK6L/00/000/084.replace"
Sep 22 23:15:12.756 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:12.757 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/084.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:12.757 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/084"
Sep 22 23:15:12.757 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/084.db"
Sep 22 23:15:12.757 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:12.757 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/084.replace" to "/tmp/downstairs-vrx8aK6L/00/000/084.completed"
Sep 22 23:15:12.757 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:12.757 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:12.757 DEBG [0] It's time to notify for 466
Sep 22 23:15:12.758 INFO Completion from [0] id:466 status:true
Sep 22 23:15:12.758 INFO [467/752] Repair commands completed
Sep 22 23:15:12.758 INFO Pop front: ReconcileIO { id: ReconciliationId(467), op: ExtentReopen { repair_id: ReconciliationId(467), extent_id: 132 }, state: ClientData([New, New, New]) }
Sep 22 23:15:12.758 INFO Sent repair work, now wait for resp
Sep 22 23:15:12.758 INFO [0] received reconcile message
Sep 22 23:15:12.758 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(467), op: ExtentReopen { repair_id: ReconciliationId(467), extent_id: 132 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:12.758 INFO [0] client ExtentReopen { repair_id: ReconciliationId(467), extent_id: 132 }
Sep 22 23:15:12.758 INFO [1] received reconcile message
Sep 22 23:15:12.758 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(467), op: ExtentReopen { repair_id: ReconciliationId(467), extent_id: 132 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:12.758 INFO [1] client ExtentReopen { repair_id: ReconciliationId(467), extent_id: 132 }
Sep 22 23:15:12.758 INFO [2] received reconcile message
Sep 22 23:15:12.758 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(467), op: ExtentReopen { repair_id: ReconciliationId(467), extent_id: 132 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:12.758 INFO [2] client ExtentReopen { repair_id: ReconciliationId(467), extent_id: 132 }
Sep 22 23:15:12.758 DEBG 467 Reopen extent 132
Sep 22 23:15:12.759 DEBG 467 Reopen extent 132
Sep 22 23:15:12.759 DEBG 467 Reopen extent 132
Sep 22 23:15:12.760 DEBG [2] It's time to notify for 467
Sep 22 23:15:12.760 INFO Completion from [2] id:467 status:true
Sep 22 23:15:12.760 INFO [468/752] Repair commands completed
Sep 22 23:15:12.760 INFO Pop front: ReconcileIO { id: ReconciliationId(468), op: ExtentFlush { repair_id: ReconciliationId(468), extent_id: 173, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:12.760 INFO Sent repair work, now wait for resp
Sep 22 23:15:12.760 INFO [0] received reconcile message
Sep 22 23:15:12.760 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(468), op: ExtentFlush { repair_id: ReconciliationId(468), extent_id: 173, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:12.760 INFO [0] client ExtentFlush { repair_id: ReconciliationId(468), extent_id: 173, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:12.760 INFO [1] received reconcile message
Sep 22 23:15:12.760 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(468), op: ExtentFlush { repair_id: ReconciliationId(468), extent_id: 173, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:12.760 INFO [1] client ExtentFlush { repair_id: ReconciliationId(468), extent_id: 173, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:12.760 INFO [2] received reconcile message
Sep 22 23:15:12.760 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(468), op: ExtentFlush { repair_id: ReconciliationId(468), extent_id: 173, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:12.760 INFO [2] client ExtentFlush { repair_id: ReconciliationId(468), extent_id: 173, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:12.760 DEBG 468 Flush extent 173 with f:2 g:2
Sep 22 23:15:12.760 DEBG Flush just extent 173 with f:2 and g:2
Sep 22 23:15:12.761 DEBG [1] It's time to notify for 468
Sep 22 23:15:12.761 INFO Completion from [1] id:468 status:true
Sep 22 23:15:12.761 INFO [469/752] Repair commands completed
Sep 22 23:15:12.761 INFO Pop front: ReconcileIO { id: ReconciliationId(469), op: ExtentClose { repair_id: ReconciliationId(469), extent_id: 173 }, state: ClientData([New, New, New]) }
Sep 22 23:15:12.761 INFO Sent repair work, now wait for resp
Sep 22 23:15:12.761 INFO [0] received reconcile message
Sep 22 23:15:12.761 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(469), op: ExtentClose { repair_id: ReconciliationId(469), extent_id: 173 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:12.761 INFO [0] client ExtentClose { repair_id: ReconciliationId(469), extent_id: 173 }
Sep 22 23:15:12.761 INFO [1] received reconcile message
Sep 22 23:15:12.761 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(469), op: ExtentClose { repair_id: ReconciliationId(469), extent_id: 173 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:12.761 INFO [1] client ExtentClose { repair_id: ReconciliationId(469), extent_id: 173 }
Sep 22 23:15:12.761 INFO [2] received reconcile message
Sep 22 23:15:12.761 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(469), op: ExtentClose { repair_id: ReconciliationId(469), extent_id: 173 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:12.761 INFO [2] client ExtentClose { repair_id: ReconciliationId(469), extent_id: 173 }
Sep 22 23:15:12.761 DEBG 469 Close extent 173
Sep 22 23:15:12.761 DEBG 469 Close extent 173
Sep 22 23:15:12.762 DEBG 469 Close extent 173
Sep 22 23:15:12.762 DEBG [2] It's time to notify for 469
Sep 22 23:15:12.762 INFO Completion from [2] id:469 status:true
Sep 22 23:15:12.762 INFO [470/752] Repair commands completed
Sep 22 23:15:12.762 INFO Pop front: ReconcileIO { id: ReconciliationId(470), op: ExtentRepair { repair_id: ReconciliationId(470), extent_id: 173, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:12.762 INFO Sent repair work, now wait for resp
Sep 22 23:15:12.762 INFO [0] received reconcile message
Sep 22 23:15:12.762 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(470), op: ExtentRepair { repair_id: ReconciliationId(470), extent_id: 173, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:12.762 INFO [0] client ExtentRepair { repair_id: ReconciliationId(470), extent_id: 173, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:12.762 INFO [0] Sending repair request ReconciliationId(470)
Sep 22 23:15:12.762 INFO [1] received reconcile message
Sep 22 23:15:12.762 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(470), op: ExtentRepair { repair_id: ReconciliationId(470), extent_id: 173, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:12.762 INFO [1] client ExtentRepair { repair_id: ReconciliationId(470), extent_id: 173, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:12.762 INFO [1] No action required ReconciliationId(470)
Sep 22 23:15:12.762 INFO [2] received reconcile message
Sep 22 23:15:12.762 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(470), op: ExtentRepair { repair_id: ReconciliationId(470), extent_id: 173, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:12.762 INFO [2] client ExtentRepair { repair_id: ReconciliationId(470), extent_id: 173, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:12.762 INFO [2] No action required ReconciliationId(470)
Sep 22 23:15:12.762 DEBG 470 Repair extent 173 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:12.763 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/0AD.copy"
Sep 22 23:15:12.829 INFO accepted connection, remote_addr: 127.0.0.1:41892, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.829 TRCE incoming request, uri: /extent/173/files, method: GET, req_id: 08072b74-10d3-4df6-89cd-2a04c8d689fb, remote_addr: 127.0.0.1:41892, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.829 INFO request completed, latency_us: 242, response_code: 200, uri: /extent/173/files, method: GET, req_id: 08072b74-10d3-4df6-89cd-2a04c8d689fb, remote_addr: 127.0.0.1:41892, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.830 INFO eid:173 Found repair files: ["0AD", "0AD.db"]
Sep 22 23:15:12.830 TRCE incoming request, uri: /newextent/173/data, method: GET, req_id: 39ce5814-ae05-42fc-bc95-96e344c61040, remote_addr: 127.0.0.1:41892, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.830 INFO request completed, latency_us: 361, response_code: 200, uri: /newextent/173/data, method: GET, req_id: 39ce5814-ae05-42fc-bc95-96e344c61040, remote_addr: 127.0.0.1:41892, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.836 TRCE incoming request, uri: /newextent/173/db, method: GET, req_id: bd09b9ee-0326-4787-a908-9a6777d92438, remote_addr: 127.0.0.1:41892, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.836 INFO request completed, latency_us: 297, response_code: 200, uri: /newextent/173/db, method: GET, req_id: bd09b9ee-0326-4787-a908-9a6777d92438, remote_addr: 127.0.0.1:41892, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.837 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/0AD.copy" to "/tmp/downstairs-vrx8aK6L/00/000/0AD.replace"
Sep 22 23:15:12.837 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:12.838 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/0AD.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:12.838 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/0AD"
Sep 22 23:15:12.838 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/0AD.db"
Sep 22 23:15:12.838 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:12.838 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/0AD.replace" to "/tmp/downstairs-vrx8aK6L/00/000/0AD.completed"
Sep 22 23:15:12.838 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:12.838 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:12.839 DEBG [0] It's time to notify for 470
Sep 22 23:15:12.839 INFO Completion from [0] id:470 status:true
Sep 22 23:15:12.839 INFO [471/752] Repair commands completed
Sep 22 23:15:12.839 INFO Pop front: ReconcileIO { id: ReconciliationId(471), op: ExtentReopen { repair_id: ReconciliationId(471), extent_id: 173 }, state: ClientData([New, New, New]) }
Sep 22 23:15:12.839 INFO Sent repair work, now wait for resp
Sep 22 23:15:12.839 INFO [0] received reconcile message
Sep 22 23:15:12.839 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(471), op: ExtentReopen { repair_id: ReconciliationId(471), extent_id: 173 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:12.839 INFO [0] client ExtentReopen { repair_id: ReconciliationId(471), extent_id: 173 }
Sep 22 23:15:12.839 INFO [1] received reconcile message
Sep 22 23:15:12.839 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(471), op: ExtentReopen { repair_id: ReconciliationId(471), extent_id: 173 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:12.839 INFO [1] client ExtentReopen { repair_id: ReconciliationId(471), extent_id: 173 }
Sep 22 23:15:12.839 INFO [2] received reconcile message
Sep 22 23:15:12.839 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(471), op: ExtentReopen { repair_id: ReconciliationId(471), extent_id: 173 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:12.839 INFO [2] client ExtentReopen { repair_id: ReconciliationId(471), extent_id: 173 }
Sep 22 23:15:12.839 DEBG 471 Reopen extent 173
Sep 22 23:15:12.840 DEBG 471 Reopen extent 173
Sep 22 23:15:12.841 DEBG 471 Reopen extent 173
Sep 22 23:15:12.841 DEBG [2] It's time to notify for 471
Sep 22 23:15:12.841 INFO Completion from [2] id:471 status:true
Sep 22 23:15:12.841 INFO [472/752] Repair commands completed
Sep 22 23:15:12.841 INFO Pop front: ReconcileIO { id: ReconciliationId(472), op: ExtentFlush { repair_id: ReconciliationId(472), extent_id: 180, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:12.841 INFO Sent repair work, now wait for resp
Sep 22 23:15:12.841 INFO [0] received reconcile message
Sep 22 23:15:12.841 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(472), op: ExtentFlush { repair_id: ReconciliationId(472), extent_id: 180, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:12.841 INFO [0] client ExtentFlush { repair_id: ReconciliationId(472), extent_id: 180, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:12.841 INFO [1] received reconcile message
Sep 22 23:15:12.841 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(472), op: ExtentFlush { repair_id: ReconciliationId(472), extent_id: 180, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:12.841 INFO [1] client ExtentFlush { repair_id: ReconciliationId(472), extent_id: 180, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:12.841 INFO [2] received reconcile message
Sep 22 23:15:12.841 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(472), op: ExtentFlush { repair_id: ReconciliationId(472), extent_id: 180, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:12.841 INFO [2] client ExtentFlush { repair_id: ReconciliationId(472), extent_id: 180, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:12.842 DEBG 472 Flush extent 180 with f:2 g:2
Sep 22 23:15:12.842 DEBG Flush just extent 180 with f:2 and g:2
Sep 22 23:15:12.842 DEBG [1] It's time to notify for 472
Sep 22 23:15:12.842 INFO Completion from [1] id:472 status:true
Sep 22 23:15:12.842 INFO [473/752] Repair commands completed
Sep 22 23:15:12.842 INFO Pop front: ReconcileIO { id: ReconciliationId(473), op: ExtentClose { repair_id: ReconciliationId(473), extent_id: 180 }, state: ClientData([New, New, New]) }
Sep 22 23:15:12.842 INFO Sent repair work, now wait for resp
Sep 22 23:15:12.842 INFO [0] received reconcile message
Sep 22 23:15:12.842 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(473), op: ExtentClose { repair_id: ReconciliationId(473), extent_id: 180 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:12.842 INFO [0] client ExtentClose { repair_id: ReconciliationId(473), extent_id: 180 }
Sep 22 23:15:12.842 INFO [1] received reconcile message
Sep 22 23:15:12.842 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(473), op: ExtentClose { repair_id: ReconciliationId(473), extent_id: 180 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:12.842 INFO [1] client ExtentClose { repair_id: ReconciliationId(473), extent_id: 180 }
Sep 22 23:15:12.842 INFO [2] received reconcile message
Sep 22 23:15:12.842 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(473), op: ExtentClose { repair_id: ReconciliationId(473), extent_id: 180 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:12.842 INFO [2] client ExtentClose { repair_id: ReconciliationId(473), extent_id: 180 }
Sep 22 23:15:12.842 DEBG 473 Close extent 180
Sep 22 23:15:12.842 DEBG 473 Close extent 180
Sep 22 23:15:12.843 DEBG 473 Close extent 180
Sep 22 23:15:12.843 DEBG [2] It's time to notify for 473
Sep 22 23:15:12.843 INFO Completion from [2] id:473 status:true
Sep 22 23:15:12.843 INFO [474/752] Repair commands completed
Sep 22 23:15:12.843 INFO Pop front: ReconcileIO { id: ReconciliationId(474), op: ExtentRepair { repair_id: ReconciliationId(474), extent_id: 180, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:12.843 INFO Sent repair work, now wait for resp
Sep 22 23:15:12.843 INFO [0] received reconcile message
Sep 22 23:15:12.843 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(474), op: ExtentRepair { repair_id: ReconciliationId(474), extent_id: 180, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:12.843 INFO [0] client ExtentRepair { repair_id: ReconciliationId(474), extent_id: 180, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:12.843 INFO [0] Sending repair request ReconciliationId(474)
Sep 22 23:15:12.843 INFO [1] received reconcile message
Sep 22 23:15:12.843 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(474), op: ExtentRepair { repair_id: ReconciliationId(474), extent_id: 180, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:12.843 INFO [1] client ExtentRepair { repair_id: ReconciliationId(474), extent_id: 180, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:12.843 INFO [1] No action required ReconciliationId(474)
Sep 22 23:15:12.843 INFO [2] received reconcile message
Sep 22 23:15:12.843 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(474), op: ExtentRepair { repair_id: ReconciliationId(474), extent_id: 180, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:12.844 INFO [2] client ExtentRepair { repair_id: ReconciliationId(474), extent_id: 180, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:12.844 INFO [2] No action required ReconciliationId(474)
Sep 22 23:15:12.844 DEBG 474 Repair extent 180 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:12.844 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/0B4.copy"
Sep 22 23:15:12.906 INFO accepted connection, remote_addr: 127.0.0.1:45527, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.907 TRCE incoming request, uri: /extent/180/files, method: GET, req_id: 68f118d7-9574-44dc-be8f-cbdfb6437e23, remote_addr: 127.0.0.1:45527, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.907 INFO request completed, latency_us: 207, response_code: 200, uri: /extent/180/files, method: GET, req_id: 68f118d7-9574-44dc-be8f-cbdfb6437e23, remote_addr: 127.0.0.1:45527, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.907 INFO eid:180 Found repair files: ["0B4", "0B4.db"]
Sep 22 23:15:12.907 TRCE incoming request, uri: /newextent/180/data, method: GET, req_id: f83e0cf7-3e23-4864-a6d5-3c097fe6554e, remote_addr: 127.0.0.1:45527, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.908 INFO request completed, latency_us: 309, response_code: 200, uri: /newextent/180/data, method: GET, req_id: f83e0cf7-3e23-4864-a6d5-3c097fe6554e, remote_addr: 127.0.0.1:45527, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.913 TRCE incoming request, uri: /newextent/180/db, method: GET, req_id: c1fc9cea-1a9d-43c6-9713-dcba7603efd1, remote_addr: 127.0.0.1:45527, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.913 INFO request completed, latency_us: 305, response_code: 200, uri: /newextent/180/db, method: GET, req_id: c1fc9cea-1a9d-43c6-9713-dcba7603efd1, remote_addr: 127.0.0.1:45527, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.914 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/0B4.copy" to "/tmp/downstairs-vrx8aK6L/00/000/0B4.replace"
Sep 22 23:15:12.914 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:12.915 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/0B4.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:12.915 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/0B4"
Sep 22 23:15:12.915 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/0B4.db"
Sep 22 23:15:12.916 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:12.916 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/0B4.replace" to "/tmp/downstairs-vrx8aK6L/00/000/0B4.completed"
Sep 22 23:15:12.916 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:12.916 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:12.916 DEBG [0] It's time to notify for 474
Sep 22 23:15:12.916 INFO Completion from [0] id:474 status:true
Sep 22 23:15:12.916 INFO [475/752] Repair commands completed
Sep 22 23:15:12.916 INFO Pop front: ReconcileIO { id: ReconciliationId(475), op: ExtentReopen { repair_id: ReconciliationId(475), extent_id: 180 }, state: ClientData([New, New, New]) }
Sep 22 23:15:12.916 INFO Sent repair work, now wait for resp
Sep 22 23:15:12.916 INFO [0] received reconcile message
Sep 22 23:15:12.916 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(475), op: ExtentReopen { repair_id: ReconciliationId(475), extent_id: 180 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:12.916 INFO [0] client ExtentReopen { repair_id: ReconciliationId(475), extent_id: 180 }
Sep 22 23:15:12.916 INFO [1] received reconcile message
Sep 22 23:15:12.916 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(475), op: ExtentReopen { repair_id: ReconciliationId(475), extent_id: 180 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:12.916 INFO [1] client ExtentReopen { repair_id: ReconciliationId(475), extent_id: 180 }
Sep 22 23:15:12.916 INFO [2] received reconcile message
Sep 22 23:15:12.916 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(475), op: ExtentReopen { repair_id: ReconciliationId(475), extent_id: 180 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:12.916 INFO [2] client ExtentReopen { repair_id: ReconciliationId(475), extent_id: 180 }
Sep 22 23:15:12.916 DEBG 475 Reopen extent 180
Sep 22 23:15:12.917 DEBG 475 Reopen extent 180
Sep 22 23:15:12.917 DEBG 475 Reopen extent 180
Sep 22 23:15:12.918 DEBG [2] It's time to notify for 475
Sep 22 23:15:12.918 INFO Completion from [2] id:475 status:true
Sep 22 23:15:12.918 INFO [476/752] Repair commands completed
Sep 22 23:15:12.918 INFO Pop front: ReconcileIO { id: ReconciliationId(476), op: ExtentFlush { repair_id: ReconciliationId(476), extent_id: 184, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:12.918 INFO Sent repair work, now wait for resp
Sep 22 23:15:12.918 INFO [0] received reconcile message
Sep 22 23:15:12.918 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(476), op: ExtentFlush { repair_id: ReconciliationId(476), extent_id: 184, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:12.918 INFO [0] client ExtentFlush { repair_id: ReconciliationId(476), extent_id: 184, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:12.918 INFO [1] received reconcile message
Sep 22 23:15:12.918 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(476), op: ExtentFlush { repair_id: ReconciliationId(476), extent_id: 184, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:12.918 INFO [1] client ExtentFlush { repair_id: ReconciliationId(476), extent_id: 184, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:12.918 INFO [2] received reconcile message
Sep 22 23:15:12.918 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(476), op: ExtentFlush { repair_id: ReconciliationId(476), extent_id: 184, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:12.918 INFO [2] client ExtentFlush { repair_id: ReconciliationId(476), extent_id: 184, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:12.919 DEBG 476 Flush extent 184 with f:2 g:2
Sep 22 23:15:12.919 DEBG Flush just extent 184 with f:2 and g:2
Sep 22 23:15:12.919 DEBG [1] It's time to notify for 476
Sep 22 23:15:12.919 INFO Completion from [1] id:476 status:true
Sep 22 23:15:12.919 INFO [477/752] Repair commands completed
Sep 22 23:15:12.919 INFO Pop front: ReconcileIO { id: ReconciliationId(477), op: ExtentClose { repair_id: ReconciliationId(477), extent_id: 184 }, state: ClientData([New, New, New]) }
Sep 22 23:15:12.919 INFO Sent repair work, now wait for resp
Sep 22 23:15:12.919 INFO [0] received reconcile message
Sep 22 23:15:12.919 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(477), op: ExtentClose { repair_id: ReconciliationId(477), extent_id: 184 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:12.919 INFO [0] client ExtentClose { repair_id: ReconciliationId(477), extent_id: 184 }
Sep 22 23:15:12.919 INFO [1] received reconcile message
Sep 22 23:15:12.919 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(477), op: ExtentClose { repair_id: ReconciliationId(477), extent_id: 184 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:12.919 INFO [1] client ExtentClose { repair_id: ReconciliationId(477), extent_id: 184 }
Sep 22 23:15:12.919 INFO [2] received reconcile message
Sep 22 23:15:12.919 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(477), op: ExtentClose { repair_id: ReconciliationId(477), extent_id: 184 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:12.919 INFO [2] client ExtentClose { repair_id: ReconciliationId(477), extent_id: 184 }
Sep 22 23:15:12.919 DEBG 477 Close extent 184
Sep 22 23:15:12.919 DEBG 477 Close extent 184
Sep 22 23:15:12.920 DEBG 477 Close extent 184
Sep 22 23:15:12.920 DEBG [2] It's time to notify for 477
Sep 22 23:15:12.920 INFO Completion from [2] id:477 status:true
Sep 22 23:15:12.920 INFO [478/752] Repair commands completed
Sep 22 23:15:12.920 INFO Pop front: ReconcileIO { id: ReconciliationId(478), op: ExtentRepair { repair_id: ReconciliationId(478), extent_id: 184, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:12.920 INFO Sent repair work, now wait for resp
Sep 22 23:15:12.920 INFO [0] received reconcile message
Sep 22 23:15:12.920 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(478), op: ExtentRepair { repair_id: ReconciliationId(478), extent_id: 184, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:12.920 INFO [0] client ExtentRepair { repair_id: ReconciliationId(478), extent_id: 184, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:12.920 INFO [0] Sending repair request ReconciliationId(478)
Sep 22 23:15:12.920 INFO [1] received reconcile message
Sep 22 23:15:12.920 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(478), op: ExtentRepair { repair_id: ReconciliationId(478), extent_id: 184, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:12.920 INFO [1] client ExtentRepair { repair_id: ReconciliationId(478), extent_id: 184, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:12.920 INFO [1] No action required ReconciliationId(478)
Sep 22 23:15:12.920 INFO [2] received reconcile message
Sep 22 23:15:12.920 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(478), op: ExtentRepair { repair_id: ReconciliationId(478), extent_id: 184, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:12.920 INFO [2] client ExtentRepair { repair_id: ReconciliationId(478), extent_id: 184, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:12.921 INFO [2] No action required ReconciliationId(478)
Sep 22 23:15:12.921 DEBG 478 Repair extent 184 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:12.921 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/0B8.copy"
Sep 22 23:15:12.983 INFO accepted connection, remote_addr: 127.0.0.1:49065, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.983 TRCE incoming request, uri: /extent/184/files, method: GET, req_id: 9e95320a-36f2-4564-bc77-385b136ed18b, remote_addr: 127.0.0.1:49065, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.983 INFO request completed, latency_us: 198, response_code: 200, uri: /extent/184/files, method: GET, req_id: 9e95320a-36f2-4564-bc77-385b136ed18b, remote_addr: 127.0.0.1:49065, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.983 INFO eid:184 Found repair files: ["0B8", "0B8.db"]
Sep 22 23:15:12.984 TRCE incoming request, uri: /newextent/184/data, method: GET, req_id: 304ec717-f476-4fc9-ac08-9f32fea1f32f, remote_addr: 127.0.0.1:49065, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.984 INFO request completed, latency_us: 322, response_code: 200, uri: /newextent/184/data, method: GET, req_id: 304ec717-f476-4fc9-ac08-9f32fea1f32f, remote_addr: 127.0.0.1:49065, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.989 TRCE incoming request, uri: /newextent/184/db, method: GET, req_id: bb731711-2ba2-4595-9ef5-69cd5bc7e36c, remote_addr: 127.0.0.1:49065, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.989 INFO request completed, latency_us: 309, response_code: 200, uri: /newextent/184/db, method: GET, req_id: bb731711-2ba2-4595-9ef5-69cd5bc7e36c, remote_addr: 127.0.0.1:49065, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:12.990 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/0B8.copy" to "/tmp/downstairs-vrx8aK6L/00/000/0B8.replace"
Sep 22 23:15:12.991 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:12.991 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/0B8.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:12.992 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/0B8"
Sep 22 23:15:12.992 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/0B8.db"
Sep 22 23:15:12.992 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:12.992 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/0B8.replace" to "/tmp/downstairs-vrx8aK6L/00/000/0B8.completed"
Sep 22 23:15:12.992 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:12.992 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:12.992 DEBG [0] It's time to notify for 478
Sep 22 23:15:12.992 INFO Completion from [0] id:478 status:true
Sep 22 23:15:12.992 INFO [479/752] Repair commands completed
Sep 22 23:15:12.992 INFO Pop front: ReconcileIO { id: ReconciliationId(479), op: ExtentReopen { repair_id: ReconciliationId(479), extent_id: 184 }, state: ClientData([New, New, New]) }
Sep 22 23:15:12.992 INFO Sent repair work, now wait for resp
Sep 22 23:15:12.992 INFO [0] received reconcile message
Sep 22 23:15:12.992 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(479), op: ExtentReopen { repair_id: ReconciliationId(479), extent_id: 184 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:12.992 INFO [0] client ExtentReopen { repair_id: ReconciliationId(479), extent_id: 184 }
Sep 22 23:15:12.992 INFO [1] received reconcile message
Sep 22 23:15:12.992 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(479), op: ExtentReopen { repair_id: ReconciliationId(479), extent_id: 184 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:12.992 INFO [1] client ExtentReopen { repair_id: ReconciliationId(479), extent_id: 184 }
Sep 22 23:15:12.992 INFO [2] received reconcile message
Sep 22 23:15:12.992 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(479), op: ExtentReopen { repair_id: ReconciliationId(479), extent_id: 184 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:12.992 INFO [2] client ExtentReopen { repair_id: ReconciliationId(479), extent_id: 184 }
Sep 22 23:15:12.993 DEBG 479 Reopen extent 184
Sep 22 23:15:12.993 DEBG 479 Reopen extent 184
Sep 22 23:15:12.994 DEBG 479 Reopen extent 184
Sep 22 23:15:12.994 DEBG [2] It's time to notify for 479
Sep 22 23:15:12.994 INFO Completion from [2] id:479 status:true
Sep 22 23:15:12.994 INFO [480/752] Repair commands completed
Sep 22 23:15:12.994 INFO Pop front: ReconcileIO { id: ReconciliationId(480), op: ExtentFlush { repair_id: ReconciliationId(480), extent_id: 24, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:12.994 INFO Sent repair work, now wait for resp
Sep 22 23:15:12.994 INFO [0] received reconcile message
Sep 22 23:15:12.995 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(480), op: ExtentFlush { repair_id: ReconciliationId(480), extent_id: 24, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:12.995 INFO [0] client ExtentFlush { repair_id: ReconciliationId(480), extent_id: 24, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:12.995 INFO [1] received reconcile message
Sep 22 23:15:12.995 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(480), op: ExtentFlush { repair_id: ReconciliationId(480), extent_id: 24, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:12.995 INFO [1] client ExtentFlush { repair_id: ReconciliationId(480), extent_id: 24, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:12.995 INFO [2] received reconcile message
Sep 22 23:15:12.995 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(480), op: ExtentFlush { repair_id: ReconciliationId(480), extent_id: 24, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:12.995 INFO [2] client ExtentFlush { repair_id: ReconciliationId(480), extent_id: 24, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:12.995 DEBG 480 Flush extent 24 with f:2 g:2
Sep 22 23:15:12.995 DEBG Flush just extent 24 with f:2 and g:2
Sep 22 23:15:12.995 DEBG [1] It's time to notify for 480
Sep 22 23:15:12.995 INFO Completion from [1] id:480 status:true
Sep 22 23:15:12.995 INFO [481/752] Repair commands completed
Sep 22 23:15:12.995 INFO Pop front: ReconcileIO { id: ReconciliationId(481), op: ExtentClose { repair_id: ReconciliationId(481), extent_id: 24 }, state: ClientData([New, New, New]) }
Sep 22 23:15:12.995 INFO Sent repair work, now wait for resp
Sep 22 23:15:12.995 INFO [0] received reconcile message
Sep 22 23:15:12.995 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(481), op: ExtentClose { repair_id: ReconciliationId(481), extent_id: 24 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:12.995 INFO [0] client ExtentClose { repair_id: ReconciliationId(481), extent_id: 24 }
Sep 22 23:15:12.995 INFO [1] received reconcile message
Sep 22 23:15:12.995 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(481), op: ExtentClose { repair_id: ReconciliationId(481), extent_id: 24 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:12.995 INFO [1] client ExtentClose { repair_id: ReconciliationId(481), extent_id: 24 }
Sep 22 23:15:12.995 INFO [2] received reconcile message
Sep 22 23:15:12.995 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(481), op: ExtentClose { repair_id: ReconciliationId(481), extent_id: 24 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:12.995 INFO [2] client ExtentClose { repair_id: ReconciliationId(481), extent_id: 24 }
Sep 22 23:15:12.995 DEBG 481 Close extent 24
Sep 22 23:15:12.996 DEBG 481 Close extent 24
Sep 22 23:15:12.996 DEBG 481 Close extent 24
Sep 22 23:15:12.996 DEBG [2] It's time to notify for 481
Sep 22 23:15:12.996 INFO Completion from [2] id:481 status:true
Sep 22 23:15:12.996 INFO [482/752] Repair commands completed
Sep 22 23:15:12.996 INFO Pop front: ReconcileIO { id: ReconciliationId(482), op: ExtentRepair { repair_id: ReconciliationId(482), extent_id: 24, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:12.996 INFO Sent repair work, now wait for resp
Sep 22 23:15:12.996 INFO [0] received reconcile message
Sep 22 23:15:12.997 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(482), op: ExtentRepair { repair_id: ReconciliationId(482), extent_id: 24, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:12.997 INFO [0] client ExtentRepair { repair_id: ReconciliationId(482), extent_id: 24, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:12.997 INFO [0] Sending repair request ReconciliationId(482)
Sep 22 23:15:12.997 INFO [1] received reconcile message
Sep 22 23:15:12.997 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(482), op: ExtentRepair { repair_id: ReconciliationId(482), extent_id: 24, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:12.997 INFO [1] client ExtentRepair { repair_id: ReconciliationId(482), extent_id: 24, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:12.997 INFO [1] No action required ReconciliationId(482)
Sep 22 23:15:12.997 INFO [2] received reconcile message
Sep 22 23:15:12.997 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(482), op: ExtentRepair { repair_id: ReconciliationId(482), extent_id: 24, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:12.997 INFO [2] client ExtentRepair { repair_id: ReconciliationId(482), extent_id: 24, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:12.997 INFO [2] No action required ReconciliationId(482)
Sep 22 23:15:12.997 DEBG 482 Repair extent 24 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:12.997 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/018.copy"
Sep 22 23:15:13.061 INFO accepted connection, remote_addr: 127.0.0.1:59076, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.061 TRCE incoming request, uri: /extent/24/files, method: GET, req_id: 03f43fcc-c5a7-48f3-a713-af38666b2052, remote_addr: 127.0.0.1:59076, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.061 INFO request completed, latency_us: 276, response_code: 200, uri: /extent/24/files, method: GET, req_id: 03f43fcc-c5a7-48f3-a713-af38666b2052, remote_addr: 127.0.0.1:59076, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.061 INFO eid:24 Found repair files: ["018", "018.db"]
Sep 22 23:15:13.062 TRCE incoming request, uri: /newextent/24/data, method: GET, req_id: 3380397b-cc84-4cb8-b99c-1e3a7a53dac2, remote_addr: 127.0.0.1:59076, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.062 INFO request completed, latency_us: 308, response_code: 200, uri: /newextent/24/data, method: GET, req_id: 3380397b-cc84-4cb8-b99c-1e3a7a53dac2, remote_addr: 127.0.0.1:59076, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.067 TRCE incoming request, uri: /newextent/24/db, method: GET, req_id: 767ff906-7ab6-461e-bfd8-60e97219d14d, remote_addr: 127.0.0.1:59076, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.068 INFO request completed, latency_us: 297, response_code: 200, uri: /newextent/24/db, method: GET, req_id: 767ff906-7ab6-461e-bfd8-60e97219d14d, remote_addr: 127.0.0.1:59076, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.069 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/018.copy" to "/tmp/downstairs-vrx8aK6L/00/000/018.replace"
Sep 22 23:15:13.069 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:13.070 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/018.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:13.070 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/018"
Sep 22 23:15:13.070 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/018.db"
Sep 22 23:15:13.070 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:13.070 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/018.replace" to "/tmp/downstairs-vrx8aK6L/00/000/018.completed"
Sep 22 23:15:13.070 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:13.070 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:13.070 DEBG [0] It's time to notify for 482
Sep 22 23:15:13.070 INFO Completion from [0] id:482 status:true
Sep 22 23:15:13.070 INFO [483/752] Repair commands completed
Sep 22 23:15:13.070 INFO Pop front: ReconcileIO { id: ReconciliationId(483), op: ExtentReopen { repair_id: ReconciliationId(483), extent_id: 24 }, state: ClientData([New, New, New]) }
Sep 22 23:15:13.071 INFO Sent repair work, now wait for resp
Sep 22 23:15:13.071 INFO [0] received reconcile message
Sep 22 23:15:13.071 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(483), op: ExtentReopen { repair_id: ReconciliationId(483), extent_id: 24 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:13.071 INFO [0] client ExtentReopen { repair_id: ReconciliationId(483), extent_id: 24 }
Sep 22 23:15:13.071 INFO [1] received reconcile message
Sep 22 23:15:13.071 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(483), op: ExtentReopen { repair_id: ReconciliationId(483), extent_id: 24 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:13.071 INFO [1] client ExtentReopen { repair_id: ReconciliationId(483), extent_id: 24 }
Sep 22 23:15:13.071 INFO [2] received reconcile message
Sep 22 23:15:13.071 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(483), op: ExtentReopen { repair_id: ReconciliationId(483), extent_id: 24 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:13.071 INFO [2] client ExtentReopen { repair_id: ReconciliationId(483), extent_id: 24 }
Sep 22 23:15:13.071 DEBG 483 Reopen extent 24
Sep 22 23:15:13.072 DEBG 483 Reopen extent 24
Sep 22 23:15:13.072 DEBG 483 Reopen extent 24
Sep 22 23:15:13.073 DEBG [2] It's time to notify for 483
Sep 22 23:15:13.073 INFO Completion from [2] id:483 status:true
Sep 22 23:15:13.073 INFO [484/752] Repair commands completed
Sep 22 23:15:13.073 INFO Pop front: ReconcileIO { id: ReconciliationId(484), op: ExtentFlush { repair_id: ReconciliationId(484), extent_id: 50, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:13.073 INFO Sent repair work, now wait for resp
Sep 22 23:15:13.073 INFO [0] received reconcile message
Sep 22 23:15:13.073 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(484), op: ExtentFlush { repair_id: ReconciliationId(484), extent_id: 50, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:13.073 INFO [0] client ExtentFlush { repair_id: ReconciliationId(484), extent_id: 50, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:13.073 INFO [1] received reconcile message
Sep 22 23:15:13.073 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(484), op: ExtentFlush { repair_id: ReconciliationId(484), extent_id: 50, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:13.073 INFO [1] client ExtentFlush { repair_id: ReconciliationId(484), extent_id: 50, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:13.073 INFO [2] received reconcile message
Sep 22 23:15:13.073 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(484), op: ExtentFlush { repair_id: ReconciliationId(484), extent_id: 50, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:13.073 INFO [2] client ExtentFlush { repair_id: ReconciliationId(484), extent_id: 50, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:13.073 DEBG 484 Flush extent 50 with f:2 g:2
Sep 22 23:15:13.073 DEBG Flush just extent 50 with f:2 and g:2
Sep 22 23:15:13.073 DEBG [1] It's time to notify for 484
Sep 22 23:15:13.074 INFO Completion from [1] id:484 status:true
Sep 22 23:15:13.074 INFO [485/752] Repair commands completed
Sep 22 23:15:13.074 INFO Pop front: ReconcileIO { id: ReconciliationId(485), op: ExtentClose { repair_id: ReconciliationId(485), extent_id: 50 }, state: ClientData([New, New, New]) }
Sep 22 23:15:13.074 INFO Sent repair work, now wait for resp
Sep 22 23:15:13.074 INFO [0] received reconcile message
Sep 22 23:15:13.074 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(485), op: ExtentClose { repair_id: ReconciliationId(485), extent_id: 50 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:13.074 INFO [0] client ExtentClose { repair_id: ReconciliationId(485), extent_id: 50 }
Sep 22 23:15:13.074 INFO [1] received reconcile message
Sep 22 23:15:13.074 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(485), op: ExtentClose { repair_id: ReconciliationId(485), extent_id: 50 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:13.074 INFO [1] client ExtentClose { repair_id: ReconciliationId(485), extent_id: 50 }
Sep 22 23:15:13.074 INFO [2] received reconcile message
Sep 22 23:15:13.074 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(485), op: ExtentClose { repair_id: ReconciliationId(485), extent_id: 50 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:13.074 INFO [2] client ExtentClose { repair_id: ReconciliationId(485), extent_id: 50 }
Sep 22 23:15:13.074 DEBG 485 Close extent 50
Sep 22 23:15:13.074 DEBG 485 Close extent 50
Sep 22 23:15:13.075 DEBG 485 Close extent 50
Sep 22 23:15:13.075 DEBG [2] It's time to notify for 485
Sep 22 23:15:13.075 INFO Completion from [2] id:485 status:true
Sep 22 23:15:13.075 INFO [486/752] Repair commands completed
Sep 22 23:15:13.075 INFO Pop front: ReconcileIO { id: ReconciliationId(486), op: ExtentRepair { repair_id: ReconciliationId(486), extent_id: 50, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:13.075 INFO Sent repair work, now wait for resp
Sep 22 23:15:13.075 INFO [0] received reconcile message
Sep 22 23:15:13.075 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(486), op: ExtentRepair { repair_id: ReconciliationId(486), extent_id: 50, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:13.075 INFO [0] client ExtentRepair { repair_id: ReconciliationId(486), extent_id: 50, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:13.075 INFO [0] Sending repair request ReconciliationId(486)
Sep 22 23:15:13.075 INFO [1] received reconcile message
Sep 22 23:15:13.075 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(486), op: ExtentRepair { repair_id: ReconciliationId(486), extent_id: 50, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:13.075 INFO [1] client ExtentRepair { repair_id: ReconciliationId(486), extent_id: 50, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:13.075 INFO [1] No action required ReconciliationId(486)
Sep 22 23:15:13.075 INFO [2] received reconcile message
Sep 22 23:15:13.075 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(486), op: ExtentRepair { repair_id: ReconciliationId(486), extent_id: 50, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:13.075 INFO [2] client ExtentRepair { repair_id: ReconciliationId(486), extent_id: 50, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:13.075 INFO [2] No action required ReconciliationId(486)
Sep 22 23:15:13.075 DEBG 486 Repair extent 50 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:13.075 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/032.copy"
Sep 22 23:15:13.091 INFO [lossy] sleeping 1 second
Sep 22 23:15:13.091 WARN returning error on flush!
Sep 22 23:15:13.091 DEBG Flush     :1085 extent_limit None deps:[JobId(1084)] res:false f:32 g:1
Sep 22 23:15:13.091 DEBG Flush     :1085 extent_limit None deps:[JobId(1084)] res:true f:32 g:1
Sep 22 23:15:13.091 INFO [lossy] sleeping 1 second
Sep 22 23:15:13.091 ERRO [0] job id 1085 saw error GenericError("test error")
Sep 22 23:15:13.091 DEBG up_ds_listen was notified
Sep 22 23:15:13.091 DEBG up_ds_listen process 1085
Sep 22 23:15:13.091 DEBG [A] ack job 1085:86, : downstairs
Sep 22 23:15:13.092 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:13.140 INFO accepted connection, remote_addr: 127.0.0.1:43150, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.141 TRCE incoming request, uri: /extent/50/files, method: GET, req_id: 1ce7639f-da64-4f3c-b9bb-24c6bd72d2bc, remote_addr: 127.0.0.1:43150, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.141 INFO request completed, latency_us: 209, response_code: 200, uri: /extent/50/files, method: GET, req_id: 1ce7639f-da64-4f3c-b9bb-24c6bd72d2bc, remote_addr: 127.0.0.1:43150, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.141 INFO eid:50 Found repair files: ["032", "032.db"]
Sep 22 23:15:13.141 TRCE incoming request, uri: /newextent/50/data, method: GET, req_id: 3a65417e-9019-4ca3-90b3-93681d4ea9af, remote_addr: 127.0.0.1:43150, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.142 INFO request completed, latency_us: 317, response_code: 200, uri: /newextent/50/data, method: GET, req_id: 3a65417e-9019-4ca3-90b3-93681d4ea9af, remote_addr: 127.0.0.1:43150, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.147 TRCE incoming request, uri: /newextent/50/db, method: GET, req_id: de912fa8-ad69-4f7c-95e4-62e9a35567e7, remote_addr: 127.0.0.1:43150, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.147 INFO request completed, latency_us: 310, response_code: 200, uri: /newextent/50/db, method: GET, req_id: de912fa8-ad69-4f7c-95e4-62e9a35567e7, remote_addr: 127.0.0.1:43150, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.148 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/032.copy" to "/tmp/downstairs-vrx8aK6L/00/000/032.replace"
Sep 22 23:15:13.148 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:13.149 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/032.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:13.150 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/032"
Sep 22 23:15:13.150 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/032.db"
Sep 22 23:15:13.150 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:13.150 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/032.replace" to "/tmp/downstairs-vrx8aK6L/00/000/032.completed"
Sep 22 23:15:13.150 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:13.150 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:13.150 DEBG [0] It's time to notify for 486
Sep 22 23:15:13.150 INFO Completion from [0] id:486 status:true
Sep 22 23:15:13.150 INFO [487/752] Repair commands completed
Sep 22 23:15:13.150 INFO Pop front: ReconcileIO { id: ReconciliationId(487), op: ExtentReopen { repair_id: ReconciliationId(487), extent_id: 50 }, state: ClientData([New, New, New]) }
Sep 22 23:15:13.150 INFO Sent repair work, now wait for resp
Sep 22 23:15:13.150 INFO [0] received reconcile message
Sep 22 23:15:13.150 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(487), op: ExtentReopen { repair_id: ReconciliationId(487), extent_id: 50 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:13.150 INFO [0] client ExtentReopen { repair_id: ReconciliationId(487), extent_id: 50 }
Sep 22 23:15:13.150 INFO [1] received reconcile message
Sep 22 23:15:13.150 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(487), op: ExtentReopen { repair_id: ReconciliationId(487), extent_id: 50 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:13.150 INFO [1] client ExtentReopen { repair_id: ReconciliationId(487), extent_id: 50 }
Sep 22 23:15:13.150 INFO [2] received reconcile message
Sep 22 23:15:13.150 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(487), op: ExtentReopen { repair_id: ReconciliationId(487), extent_id: 50 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:13.150 INFO [2] client ExtentReopen { repair_id: ReconciliationId(487), extent_id: 50 }
Sep 22 23:15:13.151 DEBG 487 Reopen extent 50
Sep 22 23:15:13.151 DEBG 487 Reopen extent 50
Sep 22 23:15:13.152 DEBG 487 Reopen extent 50
Sep 22 23:15:13.152 DEBG [2] It's time to notify for 487
Sep 22 23:15:13.152 INFO Completion from [2] id:487 status:true
Sep 22 23:15:13.152 INFO [488/752] Repair commands completed
Sep 22 23:15:13.152 INFO Pop front: ReconcileIO { id: ReconciliationId(488), op: ExtentFlush { repair_id: ReconciliationId(488), extent_id: 117, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:13.152 INFO Sent repair work, now wait for resp
Sep 22 23:15:13.152 INFO [0] received reconcile message
Sep 22 23:15:13.152 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(488), op: ExtentFlush { repair_id: ReconciliationId(488), extent_id: 117, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:13.153 INFO [0] client ExtentFlush { repair_id: ReconciliationId(488), extent_id: 117, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:13.153 INFO [1] received reconcile message
Sep 22 23:15:13.153 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(488), op: ExtentFlush { repair_id: ReconciliationId(488), extent_id: 117, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:13.153 INFO [1] client ExtentFlush { repair_id: ReconciliationId(488), extent_id: 117, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:13.153 INFO [2] received reconcile message
Sep 22 23:15:13.153 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(488), op: ExtentFlush { repair_id: ReconciliationId(488), extent_id: 117, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:13.153 INFO [2] client ExtentFlush { repair_id: ReconciliationId(488), extent_id: 117, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:13.153 DEBG 488 Flush extent 117 with f:2 g:2
Sep 22 23:15:13.153 DEBG Flush just extent 117 with f:2 and g:2
Sep 22 23:15:13.153 DEBG [1] It's time to notify for 488
Sep 22 23:15:13.153 INFO Completion from [1] id:488 status:true
Sep 22 23:15:13.153 INFO [489/752] Repair commands completed
Sep 22 23:15:13.153 INFO Pop front: ReconcileIO { id: ReconciliationId(489), op: ExtentClose { repair_id: ReconciliationId(489), extent_id: 117 }, state: ClientData([New, New, New]) }
Sep 22 23:15:13.153 INFO Sent repair work, now wait for resp
Sep 22 23:15:13.153 INFO [0] received reconcile message
Sep 22 23:15:13.153 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(489), op: ExtentClose { repair_id: ReconciliationId(489), extent_id: 117 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:13.153 INFO [0] client ExtentClose { repair_id: ReconciliationId(489), extent_id: 117 }
Sep 22 23:15:13.153 INFO [1] received reconcile message
Sep 22 23:15:13.153 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(489), op: ExtentClose { repair_id: ReconciliationId(489), extent_id: 117 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:13.153 INFO [1] client ExtentClose { repair_id: ReconciliationId(489), extent_id: 117 }
Sep 22 23:15:13.153 INFO [2] received reconcile message
Sep 22 23:15:13.153 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(489), op: ExtentClose { repair_id: ReconciliationId(489), extent_id: 117 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:13.153 INFO [2] client ExtentClose { repair_id: ReconciliationId(489), extent_id: 117 }
Sep 22 23:15:13.153 DEBG 489 Close extent 117
Sep 22 23:15:13.154 DEBG 489 Close extent 117
Sep 22 23:15:13.154 DEBG 489 Close extent 117
Sep 22 23:15:13.154 DEBG [2] It's time to notify for 489
Sep 22 23:15:13.154 INFO Completion from [2] id:489 status:true
Sep 22 23:15:13.154 INFO [490/752] Repair commands completed
Sep 22 23:15:13.154 INFO Pop front: ReconcileIO { id: ReconciliationId(490), op: ExtentRepair { repair_id: ReconciliationId(490), extent_id: 117, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:13.154 INFO Sent repair work, now wait for resp
Sep 22 23:15:13.154 INFO [0] received reconcile message
Sep 22 23:15:13.154 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(490), op: ExtentRepair { repair_id: ReconciliationId(490), extent_id: 117, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:13.154 INFO [0] client ExtentRepair { repair_id: ReconciliationId(490), extent_id: 117, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:13.154 INFO [0] Sending repair request ReconciliationId(490)
Sep 22 23:15:13.155 INFO [1] received reconcile message
Sep 22 23:15:13.155 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(490), op: ExtentRepair { repair_id: ReconciliationId(490), extent_id: 117, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:13.155 INFO [1] client ExtentRepair { repair_id: ReconciliationId(490), extent_id: 117, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:13.155 INFO [1] No action required ReconciliationId(490)
Sep 22 23:15:13.155 INFO [2] received reconcile message
Sep 22 23:15:13.155 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(490), op: ExtentRepair { repair_id: ReconciliationId(490), extent_id: 117, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:13.155 INFO [2] client ExtentRepair { repair_id: ReconciliationId(490), extent_id: 117, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:13.155 INFO [2] No action required ReconciliationId(490)
Sep 22 23:15:13.155 DEBG 490 Repair extent 117 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:13.155 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/075.copy"
Sep 22 23:15:13.219 INFO accepted connection, remote_addr: 127.0.0.1:60516, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.219 TRCE incoming request, uri: /extent/117/files, method: GET, req_id: 6dda8120-96a8-4a28-8a35-6ee12d48f80e, remote_addr: 127.0.0.1:60516, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.219 INFO request completed, latency_us: 233, response_code: 200, uri: /extent/117/files, method: GET, req_id: 6dda8120-96a8-4a28-8a35-6ee12d48f80e, remote_addr: 127.0.0.1:60516, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.219 INFO eid:117 Found repair files: ["075", "075.db"]
Sep 22 23:15:13.220 TRCE incoming request, uri: /newextent/117/data, method: GET, req_id: 1c111a77-052e-4e0c-8b21-43683b2cbb20, remote_addr: 127.0.0.1:60516, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.220 INFO request completed, latency_us: 327, response_code: 200, uri: /newextent/117/data, method: GET, req_id: 1c111a77-052e-4e0c-8b21-43683b2cbb20, remote_addr: 127.0.0.1:60516, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.225 TRCE incoming request, uri: /newextent/117/db, method: GET, req_id: 855253cc-e39a-4bb5-bfc3-3558f0dd07f9, remote_addr: 127.0.0.1:60516, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.225 INFO request completed, latency_us: 291, response_code: 200, uri: /newextent/117/db, method: GET, req_id: 855253cc-e39a-4bb5-bfc3-3558f0dd07f9, remote_addr: 127.0.0.1:60516, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.227 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/075.copy" to "/tmp/downstairs-vrx8aK6L/00/000/075.replace"
Sep 22 23:15:13.227 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:13.228 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/075.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:13.228 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/075"
Sep 22 23:15:13.228 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/075.db"
Sep 22 23:15:13.228 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:13.228 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/075.replace" to "/tmp/downstairs-vrx8aK6L/00/000/075.completed"
Sep 22 23:15:13.228 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:13.228 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:13.228 DEBG [0] It's time to notify for 490
Sep 22 23:15:13.228 INFO Completion from [0] id:490 status:true
Sep 22 23:15:13.228 INFO [491/752] Repair commands completed
Sep 22 23:15:13.229 INFO Pop front: ReconcileIO { id: ReconciliationId(491), op: ExtentReopen { repair_id: ReconciliationId(491), extent_id: 117 }, state: ClientData([New, New, New]) }
Sep 22 23:15:13.229 INFO Sent repair work, now wait for resp
Sep 22 23:15:13.229 INFO [0] received reconcile message
Sep 22 23:15:13.229 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(491), op: ExtentReopen { repair_id: ReconciliationId(491), extent_id: 117 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:13.229 INFO [0] client ExtentReopen { repair_id: ReconciliationId(491), extent_id: 117 }
Sep 22 23:15:13.229 INFO [1] received reconcile message
Sep 22 23:15:13.229 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(491), op: ExtentReopen { repair_id: ReconciliationId(491), extent_id: 117 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:13.229 INFO [1] client ExtentReopen { repair_id: ReconciliationId(491), extent_id: 117 }
Sep 22 23:15:13.229 INFO [2] received reconcile message
Sep 22 23:15:13.229 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(491), op: ExtentReopen { repair_id: ReconciliationId(491), extent_id: 117 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:13.229 INFO [2] client ExtentReopen { repair_id: ReconciliationId(491), extent_id: 117 }
Sep 22 23:15:13.229 DEBG 491 Reopen extent 117
Sep 22 23:15:13.230 DEBG 491 Reopen extent 117
Sep 22 23:15:13.230 DEBG 491 Reopen extent 117
Sep 22 23:15:13.231 DEBG [2] It's time to notify for 491
Sep 22 23:15:13.231 INFO Completion from [2] id:491 status:true
Sep 22 23:15:13.231 INFO [492/752] Repair commands completed
Sep 22 23:15:13.231 INFO Pop front: ReconcileIO { id: ReconciliationId(492), op: ExtentFlush { repair_id: ReconciliationId(492), extent_id: 38, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:13.231 INFO Sent repair work, now wait for resp
Sep 22 23:15:13.231 INFO [0] received reconcile message
Sep 22 23:15:13.231 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(492), op: ExtentFlush { repair_id: ReconciliationId(492), extent_id: 38, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:13.231 INFO [0] client ExtentFlush { repair_id: ReconciliationId(492), extent_id: 38, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:13.231 INFO [1] received reconcile message
Sep 22 23:15:13.231 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(492), op: ExtentFlush { repair_id: ReconciliationId(492), extent_id: 38, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:13.231 INFO [1] client ExtentFlush { repair_id: ReconciliationId(492), extent_id: 38, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:13.231 INFO [2] received reconcile message
Sep 22 23:15:13.231 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(492), op: ExtentFlush { repair_id: ReconciliationId(492), extent_id: 38, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:13.231 INFO [2] client ExtentFlush { repair_id: ReconciliationId(492), extent_id: 38, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:13.231 DEBG 492 Flush extent 38 with f:2 g:2
Sep 22 23:15:13.231 DEBG Flush just extent 38 with f:2 and g:2
Sep 22 23:15:13.231 DEBG [1] It's time to notify for 492
Sep 22 23:15:13.231 INFO Completion from [1] id:492 status:true
Sep 22 23:15:13.231 INFO [493/752] Repair commands completed
Sep 22 23:15:13.231 INFO Pop front: ReconcileIO { id: ReconciliationId(493), op: ExtentClose { repair_id: ReconciliationId(493), extent_id: 38 }, state: ClientData([New, New, New]) }
Sep 22 23:15:13.231 INFO Sent repair work, now wait for resp
Sep 22 23:15:13.231 INFO [0] received reconcile message
Sep 22 23:15:13.231 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(493), op: ExtentClose { repair_id: ReconciliationId(493), extent_id: 38 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:13.231 INFO [0] client ExtentClose { repair_id: ReconciliationId(493), extent_id: 38 }
Sep 22 23:15:13.232 INFO [1] received reconcile message
Sep 22 23:15:13.232 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(493), op: ExtentClose { repair_id: ReconciliationId(493), extent_id: 38 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:13.232 INFO [1] client ExtentClose { repair_id: ReconciliationId(493), extent_id: 38 }
Sep 22 23:15:13.232 INFO [2] received reconcile message
Sep 22 23:15:13.232 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(493), op: ExtentClose { repair_id: ReconciliationId(493), extent_id: 38 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:13.232 INFO [2] client ExtentClose { repair_id: ReconciliationId(493), extent_id: 38 }
Sep 22 23:15:13.232 DEBG 493 Close extent 38
Sep 22 23:15:13.232 DEBG 493 Close extent 38
Sep 22 23:15:13.232 DEBG 493 Close extent 38
Sep 22 23:15:13.233 DEBG [2] It's time to notify for 493
Sep 22 23:15:13.233 INFO Completion from [2] id:493 status:true
Sep 22 23:15:13.233 INFO [494/752] Repair commands completed
Sep 22 23:15:13.233 INFO Pop front: ReconcileIO { id: ReconciliationId(494), op: ExtentRepair { repair_id: ReconciliationId(494), extent_id: 38, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:13.233 INFO Sent repair work, now wait for resp
Sep 22 23:15:13.233 INFO [0] received reconcile message
Sep 22 23:15:13.233 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(494), op: ExtentRepair { repair_id: ReconciliationId(494), extent_id: 38, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:13.233 INFO [0] client ExtentRepair { repair_id: ReconciliationId(494), extent_id: 38, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:13.233 INFO [0] Sending repair request ReconciliationId(494)
Sep 22 23:15:13.233 INFO [1] received reconcile message
Sep 22 23:15:13.233 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(494), op: ExtentRepair { repair_id: ReconciliationId(494), extent_id: 38, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:13.233 INFO [1] client ExtentRepair { repair_id: ReconciliationId(494), extent_id: 38, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:13.233 INFO [1] No action required ReconciliationId(494)
Sep 22 23:15:13.233 INFO [2] received reconcile message
Sep 22 23:15:13.233 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(494), op: ExtentRepair { repair_id: ReconciliationId(494), extent_id: 38, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:13.233 INFO [2] client ExtentRepair { repair_id: ReconciliationId(494), extent_id: 38, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:13.233 INFO [2] No action required ReconciliationId(494)
Sep 22 23:15:13.233 DEBG 494 Repair extent 38 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:13.233 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/026.copy"
Sep 22 23:15:13.297 INFO accepted connection, remote_addr: 127.0.0.1:64364, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.297 TRCE incoming request, uri: /extent/38/files, method: GET, req_id: 0af04966-b43f-4b58-b87f-758c3d46058c, remote_addr: 127.0.0.1:64364, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.298 INFO request completed, latency_us: 216, response_code: 200, uri: /extent/38/files, method: GET, req_id: 0af04966-b43f-4b58-b87f-758c3d46058c, remote_addr: 127.0.0.1:64364, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.298 INFO eid:38 Found repair files: ["026", "026.db"]
Sep 22 23:15:13.298 TRCE incoming request, uri: /newextent/38/data, method: GET, req_id: 3533f4a0-4e02-4bce-a324-d48a9d2112a1, remote_addr: 127.0.0.1:64364, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.298 INFO request completed, latency_us: 310, response_code: 200, uri: /newextent/38/data, method: GET, req_id: 3533f4a0-4e02-4bce-a324-d48a9d2112a1, remote_addr: 127.0.0.1:64364, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.304 TRCE incoming request, uri: /newextent/38/db, method: GET, req_id: fa4e5ab6-6114-44ce-bee3-d9518c97a760, remote_addr: 127.0.0.1:64364, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.304 INFO request completed, latency_us: 293, response_code: 200, uri: /newextent/38/db, method: GET, req_id: fa4e5ab6-6114-44ce-bee3-d9518c97a760, remote_addr: 127.0.0.1:64364, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.305 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/026.copy" to "/tmp/downstairs-vrx8aK6L/00/000/026.replace"
Sep 22 23:15:13.305 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:13.306 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/026.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:13.306 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/026"
Sep 22 23:15:13.306 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/026.db"
Sep 22 23:15:13.306 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:13.306 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/026.replace" to "/tmp/downstairs-vrx8aK6L/00/000/026.completed"
Sep 22 23:15:13.306 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:13.306 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:13.307 DEBG [0] It's time to notify for 494
Sep 22 23:15:13.307 INFO Completion from [0] id:494 status:true
Sep 22 23:15:13.307 INFO [495/752] Repair commands completed
Sep 22 23:15:13.307 INFO Pop front: ReconcileIO { id: ReconciliationId(495), op: ExtentReopen { repair_id: ReconciliationId(495), extent_id: 38 }, state: ClientData([New, New, New]) }
Sep 22 23:15:13.307 INFO Sent repair work, now wait for resp
Sep 22 23:15:13.307 INFO [0] received reconcile message
Sep 22 23:15:13.307 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(495), op: ExtentReopen { repair_id: ReconciliationId(495), extent_id: 38 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:13.307 INFO [0] client ExtentReopen { repair_id: ReconciliationId(495), extent_id: 38 }
Sep 22 23:15:13.307 INFO [1] received reconcile message
Sep 22 23:15:13.307 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(495), op: ExtentReopen { repair_id: ReconciliationId(495), extent_id: 38 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:13.307 INFO [1] client ExtentReopen { repair_id: ReconciliationId(495), extent_id: 38 }
Sep 22 23:15:13.307 INFO [2] received reconcile message
Sep 22 23:15:13.307 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(495), op: ExtentReopen { repair_id: ReconciliationId(495), extent_id: 38 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:13.307 INFO [2] client ExtentReopen { repair_id: ReconciliationId(495), extent_id: 38 }
Sep 22 23:15:13.307 DEBG 495 Reopen extent 38
Sep 22 23:15:13.308 DEBG 495 Reopen extent 38
Sep 22 23:15:13.308 DEBG 495 Reopen extent 38
Sep 22 23:15:13.309 DEBG [2] It's time to notify for 495
Sep 22 23:15:13.309 INFO Completion from [2] id:495 status:true
Sep 22 23:15:13.309 INFO [496/752] Repair commands completed
Sep 22 23:15:13.309 INFO Pop front: ReconcileIO { id: ReconciliationId(496), op: ExtentFlush { repair_id: ReconciliationId(496), extent_id: 129, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:13.309 INFO Sent repair work, now wait for resp
Sep 22 23:15:13.309 INFO [0] received reconcile message
Sep 22 23:15:13.309 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(496), op: ExtentFlush { repair_id: ReconciliationId(496), extent_id: 129, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:13.309 INFO [0] client ExtentFlush { repair_id: ReconciliationId(496), extent_id: 129, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:13.309 INFO [1] received reconcile message
Sep 22 23:15:13.309 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(496), op: ExtentFlush { repair_id: ReconciliationId(496), extent_id: 129, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:13.309 INFO [1] client ExtentFlush { repair_id: ReconciliationId(496), extent_id: 129, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:13.309 INFO [2] received reconcile message
Sep 22 23:15:13.309 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(496), op: ExtentFlush { repair_id: ReconciliationId(496), extent_id: 129, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:13.309 INFO [2] client ExtentFlush { repair_id: ReconciliationId(496), extent_id: 129, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:13.309 DEBG 496 Flush extent 129 with f:2 g:2
Sep 22 23:15:13.309 DEBG Flush just extent 129 with f:2 and g:2
Sep 22 23:15:13.310 DEBG [1] It's time to notify for 496
Sep 22 23:15:13.310 INFO Completion from [1] id:496 status:true
Sep 22 23:15:13.310 INFO [497/752] Repair commands completed
Sep 22 23:15:13.310 INFO Pop front: ReconcileIO { id: ReconciliationId(497), op: ExtentClose { repair_id: ReconciliationId(497), extent_id: 129 }, state: ClientData([New, New, New]) }
Sep 22 23:15:13.310 INFO Sent repair work, now wait for resp
Sep 22 23:15:13.310 INFO [0] received reconcile message
Sep 22 23:15:13.310 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(497), op: ExtentClose { repair_id: ReconciliationId(497), extent_id: 129 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:13.310 INFO [0] client ExtentClose { repair_id: ReconciliationId(497), extent_id: 129 }
Sep 22 23:15:13.310 INFO [1] received reconcile message
Sep 22 23:15:13.310 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(497), op: ExtentClose { repair_id: ReconciliationId(497), extent_id: 129 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:13.310 INFO [1] client ExtentClose { repair_id: ReconciliationId(497), extent_id: 129 }
Sep 22 23:15:13.310 INFO [2] received reconcile message
Sep 22 23:15:13.310 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(497), op: ExtentClose { repair_id: ReconciliationId(497), extent_id: 129 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:13.310 INFO [2] client ExtentClose { repair_id: ReconciliationId(497), extent_id: 129 }
Sep 22 23:15:13.310 DEBG 497 Close extent 129
Sep 22 23:15:13.310 DEBG 497 Close extent 129
Sep 22 23:15:13.310 DEBG 497 Close extent 129
Sep 22 23:15:13.311 DEBG [2] It's time to notify for 497
Sep 22 23:15:13.311 INFO Completion from [2] id:497 status:true
Sep 22 23:15:13.311 INFO [498/752] Repair commands completed
Sep 22 23:15:13.311 INFO Pop front: ReconcileIO { id: ReconciliationId(498), op: ExtentRepair { repair_id: ReconciliationId(498), extent_id: 129, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:13.311 INFO Sent repair work, now wait for resp
Sep 22 23:15:13.311 INFO [0] received reconcile message
Sep 22 23:15:13.311 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(498), op: ExtentRepair { repair_id: ReconciliationId(498), extent_id: 129, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:13.311 INFO [0] client ExtentRepair { repair_id: ReconciliationId(498), extent_id: 129, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:13.311 INFO [0] Sending repair request ReconciliationId(498)
Sep 22 23:15:13.311 INFO [1] received reconcile message
Sep 22 23:15:13.311 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(498), op: ExtentRepair { repair_id: ReconciliationId(498), extent_id: 129, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:13.311 INFO [1] client ExtentRepair { repair_id: ReconciliationId(498), extent_id: 129, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:13.311 INFO [1] No action required ReconciliationId(498)
Sep 22 23:15:13.311 INFO [2] received reconcile message
Sep 22 23:15:13.311 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(498), op: ExtentRepair { repair_id: ReconciliationId(498), extent_id: 129, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:13.311 INFO [2] client ExtentRepair { repair_id: ReconciliationId(498), extent_id: 129, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:13.311 INFO [2] No action required ReconciliationId(498)
Sep 22 23:15:13.311 DEBG 498 Repair extent 129 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:13.311 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/081.copy"
Sep 22 23:15:13.367 INFO [lossy] skipping 1085
Sep 22 23:15:13.367 INFO [lossy] skipping 1086
Sep 22 23:15:13.367 WARN returning error on flush!
Sep 22 23:15:13.367 DEBG Flush     :1085 extent_limit None deps:[JobId(1084)] res:false f:32 g:1
Sep 22 23:15:13.367 INFO [lossy] skipping 1086
Sep 22 23:15:13.367 DEBG Flush     :1085 extent_limit None deps:[JobId(1084)] res:true f:32 g:1
Sep 22 23:15:13.367 INFO [lossy] skipping 1086
Sep 22 23:15:13.374 DEBG Read      :1086 deps:[JobId(1085)] res:true
Sep 22 23:15:13.376 INFO accepted connection, remote_addr: 127.0.0.1:49015, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.376 TRCE incoming request, uri: /extent/129/files, method: GET, req_id: aa7072d1-85e4-4181-9432-67e49799d3ff, remote_addr: 127.0.0.1:49015, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.376 INFO request completed, latency_us: 228, response_code: 200, uri: /extent/129/files, method: GET, req_id: aa7072d1-85e4-4181-9432-67e49799d3ff, remote_addr: 127.0.0.1:49015, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.376 INFO eid:129 Found repair files: ["081", "081.db"]
Sep 22 23:15:13.377 TRCE incoming request, uri: /newextent/129/data, method: GET, req_id: d965f5d0-75fb-41b6-98d3-37ca17fe6325, remote_addr: 127.0.0.1:49015, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.377 INFO request completed, latency_us: 280, response_code: 200, uri: /newextent/129/data, method: GET, req_id: d965f5d0-75fb-41b6-98d3-37ca17fe6325, remote_addr: 127.0.0.1:49015, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.382 TRCE incoming request, uri: /newextent/129/db, method: GET, req_id: 1d0abd9b-8daf-40e6-b6e5-96c439273276, remote_addr: 127.0.0.1:49015, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.383 INFO request completed, latency_us: 298, response_code: 200, uri: /newextent/129/db, method: GET, req_id: 1d0abd9b-8daf-40e6-b6e5-96c439273276, remote_addr: 127.0.0.1:49015, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.384 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/081.copy" to "/tmp/downstairs-vrx8aK6L/00/000/081.replace"
Sep 22 23:15:13.384 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:13.385 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/081.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:13.385 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/081"
Sep 22 23:15:13.385 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/081.db"
Sep 22 23:15:13.385 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:13.385 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/081.replace" to "/tmp/downstairs-vrx8aK6L/00/000/081.completed"
Sep 22 23:15:13.385 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:13.385 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:13.386 DEBG [0] It's time to notify for 498
Sep 22 23:15:13.386 INFO Completion from [0] id:498 status:true
Sep 22 23:15:13.386 INFO [499/752] Repair commands completed
Sep 22 23:15:13.386 INFO Pop front: ReconcileIO { id: ReconciliationId(499), op: ExtentReopen { repair_id: ReconciliationId(499), extent_id: 129 }, state: ClientData([New, New, New]) }
Sep 22 23:15:13.386 INFO Sent repair work, now wait for resp
Sep 22 23:15:13.386 INFO [0] received reconcile message
Sep 22 23:15:13.386 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(499), op: ExtentReopen { repair_id: ReconciliationId(499), extent_id: 129 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:13.386 INFO [0] client ExtentReopen { repair_id: ReconciliationId(499), extent_id: 129 }
Sep 22 23:15:13.386 INFO [1] received reconcile message
Sep 22 23:15:13.386 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(499), op: ExtentReopen { repair_id: ReconciliationId(499), extent_id: 129 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:13.386 INFO [1] client ExtentReopen { repair_id: ReconciliationId(499), extent_id: 129 }
Sep 22 23:15:13.386 INFO [2] received reconcile message
Sep 22 23:15:13.386 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(499), op: ExtentReopen { repair_id: ReconciliationId(499), extent_id: 129 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:13.386 INFO [2] client ExtentReopen { repair_id: ReconciliationId(499), extent_id: 129 }
Sep 22 23:15:13.386 DEBG 499 Reopen extent 129
Sep 22 23:15:13.387 DEBG 499 Reopen extent 129
Sep 22 23:15:13.387 DEBG 499 Reopen extent 129
Sep 22 23:15:13.388 DEBG [2] It's time to notify for 499
Sep 22 23:15:13.388 INFO Completion from [2] id:499 status:true
Sep 22 23:15:13.388 INFO [500/752] Repair commands completed
Sep 22 23:15:13.388 INFO Pop front: ReconcileIO { id: ReconciliationId(500), op: ExtentFlush { repair_id: ReconciliationId(500), extent_id: 153, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:13.388 INFO Sent repair work, now wait for resp
Sep 22 23:15:13.388 INFO [0] received reconcile message
Sep 22 23:15:13.388 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(500), op: ExtentFlush { repair_id: ReconciliationId(500), extent_id: 153, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:13.388 INFO [0] client ExtentFlush { repair_id: ReconciliationId(500), extent_id: 153, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:13.388 INFO [1] received reconcile message
Sep 22 23:15:13.388 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(500), op: ExtentFlush { repair_id: ReconciliationId(500), extent_id: 153, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:13.388 INFO [1] client ExtentFlush { repair_id: ReconciliationId(500), extent_id: 153, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:13.388 INFO [2] received reconcile message
Sep 22 23:15:13.388 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(500), op: ExtentFlush { repair_id: ReconciliationId(500), extent_id: 153, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:13.388 INFO [2] client ExtentFlush { repair_id: ReconciliationId(500), extent_id: 153, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:13.388 DEBG 500 Flush extent 153 with f:2 g:2
Sep 22 23:15:13.388 DEBG Flush just extent 153 with f:2 and g:2
Sep 22 23:15:13.388 DEBG [1] It's time to notify for 500
Sep 22 23:15:13.389 INFO Completion from [1] id:500 status:true
Sep 22 23:15:13.389 INFO [501/752] Repair commands completed
Sep 22 23:15:13.389 INFO Pop front: ReconcileIO { id: ReconciliationId(501), op: ExtentClose { repair_id: ReconciliationId(501), extent_id: 153 }, state: ClientData([New, New, New]) }
Sep 22 23:15:13.389 INFO Sent repair work, now wait for resp
Sep 22 23:15:13.389 INFO [0] received reconcile message
Sep 22 23:15:13.389 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(501), op: ExtentClose { repair_id: ReconciliationId(501), extent_id: 153 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:13.389 INFO [0] client ExtentClose { repair_id: ReconciliationId(501), extent_id: 153 }
Sep 22 23:15:13.389 INFO [1] received reconcile message
Sep 22 23:15:13.389 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(501), op: ExtentClose { repair_id: ReconciliationId(501), extent_id: 153 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:13.389 INFO [1] client ExtentClose { repair_id: ReconciliationId(501), extent_id: 153 }
Sep 22 23:15:13.389 INFO [2] received reconcile message
Sep 22 23:15:13.389 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(501), op: ExtentClose { repair_id: ReconciliationId(501), extent_id: 153 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:13.389 INFO [2] client ExtentClose { repair_id: ReconciliationId(501), extent_id: 153 }
Sep 22 23:15:13.389 DEBG 501 Close extent 153
Sep 22 23:15:13.389 DEBG 501 Close extent 153
Sep 22 23:15:13.389 DEBG 501 Close extent 153
Sep 22 23:15:13.390 DEBG [2] It's time to notify for 501
Sep 22 23:15:13.390 INFO Completion from [2] id:501 status:true
Sep 22 23:15:13.390 INFO [502/752] Repair commands completed
Sep 22 23:15:13.390 INFO Pop front: ReconcileIO { id: ReconciliationId(502), op: ExtentRepair { repair_id: ReconciliationId(502), extent_id: 153, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:13.390 INFO Sent repair work, now wait for resp
Sep 22 23:15:13.390 INFO [0] received reconcile message
Sep 22 23:15:13.390 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(502), op: ExtentRepair { repair_id: ReconciliationId(502), extent_id: 153, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:13.390 INFO [0] client ExtentRepair { repair_id: ReconciliationId(502), extent_id: 153, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:13.390 INFO [0] Sending repair request ReconciliationId(502)
Sep 22 23:15:13.390 INFO [1] received reconcile message
Sep 22 23:15:13.390 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(502), op: ExtentRepair { repair_id: ReconciliationId(502), extent_id: 153, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:13.390 INFO [1] client ExtentRepair { repair_id: ReconciliationId(502), extent_id: 153, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:13.390 INFO [1] No action required ReconciliationId(502)
Sep 22 23:15:13.390 INFO [2] received reconcile message
Sep 22 23:15:13.390 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(502), op: ExtentRepair { repair_id: ReconciliationId(502), extent_id: 153, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:13.390 INFO [2] client ExtentRepair { repair_id: ReconciliationId(502), extent_id: 153, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:13.390 INFO [2] No action required ReconciliationId(502)
Sep 22 23:15:13.390 DEBG 502 Repair extent 153 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:13.390 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/099.copy"
Sep 22 23:15:13.395 ERRO [1] job id 1085 saw error GenericError("test error")
Sep 22 23:15:13.395 DEBG [rc] retire 1085 clears [JobId(1084), JobId(1085)], : downstairs
Sep 22 23:15:13.397 DEBG Flush     :1087 extent_limit None deps:[JobId(1086), JobId(1085)] res:true f:33 g:1
Sep 22 23:15:13.397 INFO [lossy] sleeping 1 second
Sep 22 23:15:13.453 INFO accepted connection, remote_addr: 127.0.0.1:43815, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.453 TRCE incoming request, uri: /extent/153/files, method: GET, req_id: dade48af-3ee9-4024-ab10-da947ced5fe4, remote_addr: 127.0.0.1:43815, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.454 INFO request completed, latency_us: 268, response_code: 200, uri: /extent/153/files, method: GET, req_id: dade48af-3ee9-4024-ab10-da947ced5fe4, remote_addr: 127.0.0.1:43815, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.454 INFO eid:153 Found repair files: ["099", "099.db"]
Sep 22 23:15:13.454 TRCE incoming request, uri: /newextent/153/data, method: GET, req_id: 7f238ae3-c180-42ab-a261-0919986a1706, remote_addr: 127.0.0.1:43815, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.455 INFO request completed, latency_us: 342, response_code: 200, uri: /newextent/153/data, method: GET, req_id: 7f238ae3-c180-42ab-a261-0919986a1706, remote_addr: 127.0.0.1:43815, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.460 TRCE incoming request, uri: /newextent/153/db, method: GET, req_id: 4ac6a6f5-8d96-42fc-a104-f413b6ae8753, remote_addr: 127.0.0.1:43815, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.460 INFO request completed, latency_us: 330, response_code: 200, uri: /newextent/153/db, method: GET, req_id: 4ac6a6f5-8d96-42fc-a104-f413b6ae8753, remote_addr: 127.0.0.1:43815, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.461 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/099.copy" to "/tmp/downstairs-vrx8aK6L/00/000/099.replace"
Sep 22 23:15:13.461 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:13.463 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/099.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:13.463 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/099"
Sep 22 23:15:13.463 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/099.db"
Sep 22 23:15:13.463 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:13.463 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/099.replace" to "/tmp/downstairs-vrx8aK6L/00/000/099.completed"
Sep 22 23:15:13.463 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:13.463 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:13.464 DEBG [0] It's time to notify for 502
Sep 22 23:15:13.464 INFO Completion from [0] id:502 status:true
Sep 22 23:15:13.464 INFO [503/752] Repair commands completed
Sep 22 23:15:13.464 INFO Pop front: ReconcileIO { id: ReconciliationId(503), op: ExtentReopen { repair_id: ReconciliationId(503), extent_id: 153 }, state: ClientData([New, New, New]) }
Sep 22 23:15:13.464 INFO Sent repair work, now wait for resp
Sep 22 23:15:13.464 INFO [0] received reconcile message
Sep 22 23:15:13.464 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(503), op: ExtentReopen { repair_id: ReconciliationId(503), extent_id: 153 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:13.464 INFO [0] client ExtentReopen { repair_id: ReconciliationId(503), extent_id: 153 }
Sep 22 23:15:13.464 INFO [1] received reconcile message
Sep 22 23:15:13.464 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(503), op: ExtentReopen { repair_id: ReconciliationId(503), extent_id: 153 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:13.464 INFO [1] client ExtentReopen { repair_id: ReconciliationId(503), extent_id: 153 }
Sep 22 23:15:13.464 INFO [2] received reconcile message
Sep 22 23:15:13.464 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(503), op: ExtentReopen { repair_id: ReconciliationId(503), extent_id: 153 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:13.464 INFO [2] client ExtentReopen { repair_id: ReconciliationId(503), extent_id: 153 }
Sep 22 23:15:13.464 DEBG 503 Reopen extent 153
Sep 22 23:15:13.465 DEBG 503 Reopen extent 153
Sep 22 23:15:13.466 DEBG 503 Reopen extent 153
Sep 22 23:15:13.466 DEBG [2] It's time to notify for 503
Sep 22 23:15:13.466 INFO Completion from [2] id:503 status:true
Sep 22 23:15:13.466 INFO [504/752] Repair commands completed
Sep 22 23:15:13.466 INFO Pop front: ReconcileIO { id: ReconciliationId(504), op: ExtentFlush { repair_id: ReconciliationId(504), extent_id: 181, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:13.466 INFO Sent repair work, now wait for resp
Sep 22 23:15:13.466 INFO [0] received reconcile message
Sep 22 23:15:13.466 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(504), op: ExtentFlush { repair_id: ReconciliationId(504), extent_id: 181, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:13.466 INFO [0] client ExtentFlush { repair_id: ReconciliationId(504), extent_id: 181, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:13.466 INFO [1] received reconcile message
Sep 22 23:15:13.466 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(504), op: ExtentFlush { repair_id: ReconciliationId(504), extent_id: 181, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:13.466 INFO [1] client ExtentFlush { repair_id: ReconciliationId(504), extent_id: 181, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:13.466 INFO [2] received reconcile message
Sep 22 23:15:13.467 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(504), op: ExtentFlush { repair_id: ReconciliationId(504), extent_id: 181, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:13.467 INFO [2] client ExtentFlush { repair_id: ReconciliationId(504), extent_id: 181, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:13.467 DEBG 504 Flush extent 181 with f:2 g:2
Sep 22 23:15:13.467 DEBG Flush just extent 181 with f:2 and g:2
Sep 22 23:15:13.467 DEBG [1] It's time to notify for 504
Sep 22 23:15:13.467 INFO Completion from [1] id:504 status:true
Sep 22 23:15:13.467 INFO [505/752] Repair commands completed
Sep 22 23:15:13.467 INFO Pop front: ReconcileIO { id: ReconciliationId(505), op: ExtentClose { repair_id: ReconciliationId(505), extent_id: 181 }, state: ClientData([New, New, New]) }
Sep 22 23:15:13.467 INFO Sent repair work, now wait for resp
Sep 22 23:15:13.467 INFO [0] received reconcile message
Sep 22 23:15:13.467 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(505), op: ExtentClose { repair_id: ReconciliationId(505), extent_id: 181 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:13.467 INFO [0] client ExtentClose { repair_id: ReconciliationId(505), extent_id: 181 }
Sep 22 23:15:13.467 INFO [1] received reconcile message
Sep 22 23:15:13.467 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(505), op: ExtentClose { repair_id: ReconciliationId(505), extent_id: 181 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:13.467 INFO [1] client ExtentClose { repair_id: ReconciliationId(505), extent_id: 181 }
Sep 22 23:15:13.467 INFO [2] received reconcile message
Sep 22 23:15:13.467 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(505), op: ExtentClose { repair_id: ReconciliationId(505), extent_id: 181 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:13.467 INFO [2] client ExtentClose { repair_id: ReconciliationId(505), extent_id: 181 }
Sep 22 23:15:13.467 DEBG 505 Close extent 181
Sep 22 23:15:13.467 DEBG 505 Close extent 181
Sep 22 23:15:13.468 DEBG 505 Close extent 181
Sep 22 23:15:13.468 DEBG [2] It's time to notify for 505
Sep 22 23:15:13.468 INFO Completion from [2] id:505 status:true
Sep 22 23:15:13.468 INFO [506/752] Repair commands completed
Sep 22 23:15:13.468 INFO Pop front: ReconcileIO { id: ReconciliationId(506), op: ExtentRepair { repair_id: ReconciliationId(506), extent_id: 181, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:13.468 INFO Sent repair work, now wait for resp
Sep 22 23:15:13.468 INFO [0] received reconcile message
Sep 22 23:15:13.468 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(506), op: ExtentRepair { repair_id: ReconciliationId(506), extent_id: 181, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:13.468 INFO [0] client ExtentRepair { repair_id: ReconciliationId(506), extent_id: 181, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:13.468 INFO [0] Sending repair request ReconciliationId(506)
Sep 22 23:15:13.468 INFO [1] received reconcile message
Sep 22 23:15:13.469 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(506), op: ExtentRepair { repair_id: ReconciliationId(506), extent_id: 181, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:13.469 INFO [1] client ExtentRepair { repair_id: ReconciliationId(506), extent_id: 181, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:13.469 INFO [1] No action required ReconciliationId(506)
Sep 22 23:15:13.469 INFO [2] received reconcile message
Sep 22 23:15:13.469 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(506), op: ExtentRepair { repair_id: ReconciliationId(506), extent_id: 181, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:13.469 INFO [2] client ExtentRepair { repair_id: ReconciliationId(506), extent_id: 181, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:13.469 INFO [2] No action required ReconciliationId(506)
Sep 22 23:15:13.469 DEBG 506 Repair extent 181 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:13.469 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/0B5.copy"
Sep 22 23:15:13.536 INFO accepted connection, remote_addr: 127.0.0.1:49266, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.537 TRCE incoming request, uri: /extent/181/files, method: GET, req_id: 2d7f784b-20a6-4f26-bbdf-41edd85812f2, remote_addr: 127.0.0.1:49266, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.537 INFO request completed, latency_us: 285, response_code: 200, uri: /extent/181/files, method: GET, req_id: 2d7f784b-20a6-4f26-bbdf-41edd85812f2, remote_addr: 127.0.0.1:49266, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.537 INFO eid:181 Found repair files: ["0B5", "0B5.db"]
Sep 22 23:15:13.538 TRCE incoming request, uri: /newextent/181/data, method: GET, req_id: c525f427-4a1d-407b-aca4-998e2f574484, remote_addr: 127.0.0.1:49266, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.538 INFO request completed, latency_us: 312, response_code: 200, uri: /newextent/181/data, method: GET, req_id: c525f427-4a1d-407b-aca4-998e2f574484, remote_addr: 127.0.0.1:49266, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.543 TRCE incoming request, uri: /newextent/181/db, method: GET, req_id: 5863f496-2e94-4220-b8cb-63c2cff379c5, remote_addr: 127.0.0.1:49266, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.543 INFO request completed, latency_us: 299, response_code: 200, uri: /newextent/181/db, method: GET, req_id: 5863f496-2e94-4220-b8cb-63c2cff379c5, remote_addr: 127.0.0.1:49266, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.545 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/0B5.copy" to "/tmp/downstairs-vrx8aK6L/00/000/0B5.replace"
Sep 22 23:15:13.545 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:13.546 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/0B5.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:13.546 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/0B5"
Sep 22 23:15:13.546 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/0B5.db"
Sep 22 23:15:13.546 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:13.546 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/0B5.replace" to "/tmp/downstairs-vrx8aK6L/00/000/0B5.completed"
Sep 22 23:15:13.546 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:13.546 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:13.547 DEBG [0] It's time to notify for 506
Sep 22 23:15:13.547 INFO Completion from [0] id:506 status:true
Sep 22 23:15:13.547 INFO [507/752] Repair commands completed
Sep 22 23:15:13.547 INFO Pop front: ReconcileIO { id: ReconciliationId(507), op: ExtentReopen { repair_id: ReconciliationId(507), extent_id: 181 }, state: ClientData([New, New, New]) }
Sep 22 23:15:13.547 INFO Sent repair work, now wait for resp
Sep 22 23:15:13.547 INFO [0] received reconcile message
Sep 22 23:15:13.547 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(507), op: ExtentReopen { repair_id: ReconciliationId(507), extent_id: 181 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:13.547 INFO [0] client ExtentReopen { repair_id: ReconciliationId(507), extent_id: 181 }
Sep 22 23:15:13.547 INFO [1] received reconcile message
Sep 22 23:15:13.547 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(507), op: ExtentReopen { repair_id: ReconciliationId(507), extent_id: 181 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:13.547 INFO [1] client ExtentReopen { repair_id: ReconciliationId(507), extent_id: 181 }
Sep 22 23:15:13.547 INFO [2] received reconcile message
Sep 22 23:15:13.547 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(507), op: ExtentReopen { repair_id: ReconciliationId(507), extent_id: 181 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:13.547 INFO [2] client ExtentReopen { repair_id: ReconciliationId(507), extent_id: 181 }
Sep 22 23:15:13.547 DEBG 507 Reopen extent 181
Sep 22 23:15:13.548 DEBG 507 Reopen extent 181
Sep 22 23:15:13.549 DEBG 507 Reopen extent 181
Sep 22 23:15:13.549 DEBG [2] It's time to notify for 507
Sep 22 23:15:13.549 INFO Completion from [2] id:507 status:true
Sep 22 23:15:13.549 INFO [508/752] Repair commands completed
Sep 22 23:15:13.549 INFO Pop front: ReconcileIO { id: ReconciliationId(508), op: ExtentFlush { repair_id: ReconciliationId(508), extent_id: 128, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:13.549 INFO Sent repair work, now wait for resp
Sep 22 23:15:13.549 INFO [0] received reconcile message
Sep 22 23:15:13.549 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(508), op: ExtentFlush { repair_id: ReconciliationId(508), extent_id: 128, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:13.549 INFO [0] client ExtentFlush { repair_id: ReconciliationId(508), extent_id: 128, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:13.549 INFO [1] received reconcile message
Sep 22 23:15:13.549 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(508), op: ExtentFlush { repair_id: ReconciliationId(508), extent_id: 128, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:13.549 INFO [1] client ExtentFlush { repair_id: ReconciliationId(508), extent_id: 128, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:13.549 INFO [2] received reconcile message
Sep 22 23:15:13.549 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(508), op: ExtentFlush { repair_id: ReconciliationId(508), extent_id: 128, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:13.549 INFO [2] client ExtentFlush { repair_id: ReconciliationId(508), extent_id: 128, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:13.550 DEBG 508 Flush extent 128 with f:2 g:2
Sep 22 23:15:13.550 DEBG Flush just extent 128 with f:2 and g:2
Sep 22 23:15:13.550 DEBG [1] It's time to notify for 508
Sep 22 23:15:13.550 INFO Completion from [1] id:508 status:true
Sep 22 23:15:13.550 INFO [509/752] Repair commands completed
Sep 22 23:15:13.550 INFO Pop front: ReconcileIO { id: ReconciliationId(509), op: ExtentClose { repair_id: ReconciliationId(509), extent_id: 128 }, state: ClientData([New, New, New]) }
Sep 22 23:15:13.550 INFO Sent repair work, now wait for resp
Sep 22 23:15:13.550 INFO [0] received reconcile message
Sep 22 23:15:13.550 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(509), op: ExtentClose { repair_id: ReconciliationId(509), extent_id: 128 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:13.550 INFO [0] client ExtentClose { repair_id: ReconciliationId(509), extent_id: 128 }
Sep 22 23:15:13.550 INFO [1] received reconcile message
Sep 22 23:15:13.550 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(509), op: ExtentClose { repair_id: ReconciliationId(509), extent_id: 128 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:13.550 INFO [1] client ExtentClose { repair_id: ReconciliationId(509), extent_id: 128 }
Sep 22 23:15:13.550 INFO [2] received reconcile message
Sep 22 23:15:13.550 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(509), op: ExtentClose { repair_id: ReconciliationId(509), extent_id: 128 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:13.550 INFO [2] client ExtentClose { repair_id: ReconciliationId(509), extent_id: 128 }
Sep 22 23:15:13.550 DEBG 509 Close extent 128
Sep 22 23:15:13.550 DEBG 509 Close extent 128
Sep 22 23:15:13.551 DEBG 509 Close extent 128
Sep 22 23:15:13.551 DEBG [2] It's time to notify for 509
Sep 22 23:15:13.551 INFO Completion from [2] id:509 status:true
Sep 22 23:15:13.551 INFO [510/752] Repair commands completed
Sep 22 23:15:13.551 INFO Pop front: ReconcileIO { id: ReconciliationId(510), op: ExtentRepair { repair_id: ReconciliationId(510), extent_id: 128, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:13.551 INFO Sent repair work, now wait for resp
Sep 22 23:15:13.551 INFO [0] received reconcile message
Sep 22 23:15:13.551 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(510), op: ExtentRepair { repair_id: ReconciliationId(510), extent_id: 128, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:13.551 INFO [0] client ExtentRepair { repair_id: ReconciliationId(510), extent_id: 128, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:13.551 INFO [0] Sending repair request ReconciliationId(510)
Sep 22 23:15:13.551 INFO [1] received reconcile message
Sep 22 23:15:13.551 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(510), op: ExtentRepair { repair_id: ReconciliationId(510), extent_id: 128, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:13.551 INFO [1] client ExtentRepair { repair_id: ReconciliationId(510), extent_id: 128, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:13.551 INFO [1] No action required ReconciliationId(510)
Sep 22 23:15:13.552 INFO [2] received reconcile message
Sep 22 23:15:13.552 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(510), op: ExtentRepair { repair_id: ReconciliationId(510), extent_id: 128, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:13.552 INFO [2] client ExtentRepair { repair_id: ReconciliationId(510), extent_id: 128, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:13.552 INFO [2] No action required ReconciliationId(510)
Sep 22 23:15:13.552 DEBG 510 Repair extent 128 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:13.552 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/080.copy"
Sep 22 23:15:13.617 INFO accepted connection, remote_addr: 127.0.0.1:37556, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.617 TRCE incoming request, uri: /extent/128/files, method: GET, req_id: 6f5e5739-adc2-48e9-9416-3f1219d8a501, remote_addr: 127.0.0.1:37556, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.617 INFO request completed, latency_us: 283, response_code: 200, uri: /extent/128/files, method: GET, req_id: 6f5e5739-adc2-48e9-9416-3f1219d8a501, remote_addr: 127.0.0.1:37556, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.618 INFO eid:128 Found repair files: ["080", "080.db"]
Sep 22 23:15:13.618 TRCE incoming request, uri: /newextent/128/data, method: GET, req_id: 3b5af781-8b15-4bc7-8adf-bae66398ca9b, remote_addr: 127.0.0.1:37556, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.618 INFO request completed, latency_us: 370, response_code: 200, uri: /newextent/128/data, method: GET, req_id: 3b5af781-8b15-4bc7-8adf-bae66398ca9b, remote_addr: 127.0.0.1:37556, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.623 TRCE incoming request, uri: /newextent/128/db, method: GET, req_id: f271051d-0b62-4762-a9ec-76ef0cff07fb, remote_addr: 127.0.0.1:37556, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.624 INFO request completed, latency_us: 295, response_code: 200, uri: /newextent/128/db, method: GET, req_id: f271051d-0b62-4762-a9ec-76ef0cff07fb, remote_addr: 127.0.0.1:37556, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.625 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/080.copy" to "/tmp/downstairs-vrx8aK6L/00/000/080.replace"
Sep 22 23:15:13.625 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:13.626 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/080.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:13.626 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/080"
Sep 22 23:15:13.626 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/080.db"
Sep 22 23:15:13.626 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:13.626 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/080.replace" to "/tmp/downstairs-vrx8aK6L/00/000/080.completed"
Sep 22 23:15:13.626 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:13.627 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:13.627 DEBG [0] It's time to notify for 510
Sep 22 23:15:13.627 INFO Completion from [0] id:510 status:true
Sep 22 23:15:13.627 INFO [511/752] Repair commands completed
Sep 22 23:15:13.627 INFO Pop front: ReconcileIO { id: ReconciliationId(511), op: ExtentReopen { repair_id: ReconciliationId(511), extent_id: 128 }, state: ClientData([New, New, New]) }
Sep 22 23:15:13.627 INFO Sent repair work, now wait for resp
Sep 22 23:15:13.627 INFO [0] received reconcile message
Sep 22 23:15:13.627 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(511), op: ExtentReopen { repair_id: ReconciliationId(511), extent_id: 128 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:13.627 INFO [0] client ExtentReopen { repair_id: ReconciliationId(511), extent_id: 128 }
Sep 22 23:15:13.627 INFO [1] received reconcile message
Sep 22 23:15:13.627 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(511), op: ExtentReopen { repair_id: ReconciliationId(511), extent_id: 128 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:13.627 INFO [1] client ExtentReopen { repair_id: ReconciliationId(511), extent_id: 128 }
Sep 22 23:15:13.627 INFO [2] received reconcile message
Sep 22 23:15:13.627 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(511), op: ExtentReopen { repair_id: ReconciliationId(511), extent_id: 128 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:13.627 INFO [2] client ExtentReopen { repair_id: ReconciliationId(511), extent_id: 128 }
Sep 22 23:15:13.627 DEBG 511 Reopen extent 128
Sep 22 23:15:13.628 DEBG 511 Reopen extent 128
Sep 22 23:15:13.629 DEBG 511 Reopen extent 128
Sep 22 23:15:13.629 DEBG [2] It's time to notify for 511
Sep 22 23:15:13.629 INFO Completion from [2] id:511 status:true
Sep 22 23:15:13.629 INFO [512/752] Repair commands completed
Sep 22 23:15:13.629 INFO Pop front: ReconcileIO { id: ReconciliationId(512), op: ExtentFlush { repair_id: ReconciliationId(512), extent_id: 72, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:13.629 INFO Sent repair work, now wait for resp
Sep 22 23:15:13.629 INFO [0] received reconcile message
Sep 22 23:15:13.629 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(512), op: ExtentFlush { repair_id: ReconciliationId(512), extent_id: 72, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:13.629 INFO [0] client ExtentFlush { repair_id: ReconciliationId(512), extent_id: 72, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:13.630 INFO [1] received reconcile message
Sep 22 23:15:13.630 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(512), op: ExtentFlush { repair_id: ReconciliationId(512), extent_id: 72, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:13.630 INFO [1] client ExtentFlush { repair_id: ReconciliationId(512), extent_id: 72, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:13.630 INFO [2] received reconcile message
Sep 22 23:15:13.630 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(512), op: ExtentFlush { repair_id: ReconciliationId(512), extent_id: 72, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:13.630 INFO [2] client ExtentFlush { repair_id: ReconciliationId(512), extent_id: 72, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:13.630 DEBG 512 Flush extent 72 with f:2 g:2
Sep 22 23:15:13.630 DEBG Flush just extent 72 with f:2 and g:2
Sep 22 23:15:13.630 DEBG [1] It's time to notify for 512
Sep 22 23:15:13.630 INFO Completion from [1] id:512 status:true
Sep 22 23:15:13.630 INFO [513/752] Repair commands completed
Sep 22 23:15:13.630 INFO Pop front: ReconcileIO { id: ReconciliationId(513), op: ExtentClose { repair_id: ReconciliationId(513), extent_id: 72 }, state: ClientData([New, New, New]) }
Sep 22 23:15:13.630 INFO Sent repair work, now wait for resp
Sep 22 23:15:13.630 INFO [0] received reconcile message
Sep 22 23:15:13.630 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(513), op: ExtentClose { repair_id: ReconciliationId(513), extent_id: 72 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:13.630 INFO [0] client ExtentClose { repair_id: ReconciliationId(513), extent_id: 72 }
Sep 22 23:15:13.630 INFO [1] received reconcile message
Sep 22 23:15:13.630 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(513), op: ExtentClose { repair_id: ReconciliationId(513), extent_id: 72 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:13.630 INFO [1] client ExtentClose { repair_id: ReconciliationId(513), extent_id: 72 }
Sep 22 23:15:13.630 INFO [2] received reconcile message
Sep 22 23:15:13.630 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(513), op: ExtentClose { repair_id: ReconciliationId(513), extent_id: 72 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:13.630 INFO [2] client ExtentClose { repair_id: ReconciliationId(513), extent_id: 72 }
Sep 22 23:15:13.630 DEBG 513 Close extent 72
Sep 22 23:15:13.631 DEBG 513 Close extent 72
Sep 22 23:15:13.631 DEBG 513 Close extent 72
Sep 22 23:15:13.631 DEBG [2] It's time to notify for 513
Sep 22 23:15:13.631 INFO Completion from [2] id:513 status:true
Sep 22 23:15:13.631 INFO [514/752] Repair commands completed
Sep 22 23:15:13.631 INFO Pop front: ReconcileIO { id: ReconciliationId(514), op: ExtentRepair { repair_id: ReconciliationId(514), extent_id: 72, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:13.631 INFO Sent repair work, now wait for resp
Sep 22 23:15:13.631 INFO [0] received reconcile message
Sep 22 23:15:13.631 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(514), op: ExtentRepair { repair_id: ReconciliationId(514), extent_id: 72, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:13.632 INFO [0] client ExtentRepair { repair_id: ReconciliationId(514), extent_id: 72, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:13.632 INFO [0] Sending repair request ReconciliationId(514)
Sep 22 23:15:13.632 INFO [1] received reconcile message
Sep 22 23:15:13.632 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(514), op: ExtentRepair { repair_id: ReconciliationId(514), extent_id: 72, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:13.632 INFO [1] client ExtentRepair { repair_id: ReconciliationId(514), extent_id: 72, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:13.632 INFO [1] No action required ReconciliationId(514)
Sep 22 23:15:13.632 INFO [2] received reconcile message
Sep 22 23:15:13.632 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(514), op: ExtentRepair { repair_id: ReconciliationId(514), extent_id: 72, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:13.632 INFO [2] client ExtentRepair { repair_id: ReconciliationId(514), extent_id: 72, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:13.632 INFO [2] No action required ReconciliationId(514)
Sep 22 23:15:13.632 DEBG 514 Repair extent 72 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:13.632 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/048.copy"
Sep 22 23:15:13.699 INFO accepted connection, remote_addr: 127.0.0.1:38705, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.699 TRCE incoming request, uri: /extent/72/files, method: GET, req_id: d0524d83-dd0a-48bf-808d-c8666fc1b432, remote_addr: 127.0.0.1:38705, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.699 INFO request completed, latency_us: 295, response_code: 200, uri: /extent/72/files, method: GET, req_id: d0524d83-dd0a-48bf-808d-c8666fc1b432, remote_addr: 127.0.0.1:38705, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.699 INFO eid:72 Found repair files: ["048", "048.db"]
Sep 22 23:15:13.700 TRCE incoming request, uri: /newextent/72/data, method: GET, req_id: d09590e0-97f4-4f86-9acb-9897edc55229, remote_addr: 127.0.0.1:38705, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.700 INFO request completed, latency_us: 374, response_code: 200, uri: /newextent/72/data, method: GET, req_id: d09590e0-97f4-4f86-9acb-9897edc55229, remote_addr: 127.0.0.1:38705, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.705 TRCE incoming request, uri: /newextent/72/db, method: GET, req_id: 8e9658e1-0bb6-4d33-82e3-d4204cd1da41, remote_addr: 127.0.0.1:38705, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.706 INFO request completed, latency_us: 294, response_code: 200, uri: /newextent/72/db, method: GET, req_id: 8e9658e1-0bb6-4d33-82e3-d4204cd1da41, remote_addr: 127.0.0.1:38705, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.707 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/048.copy" to "/tmp/downstairs-vrx8aK6L/00/000/048.replace"
Sep 22 23:15:13.707 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:13.708 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/048.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:13.708 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/048"
Sep 22 23:15:13.709 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/048.db"
Sep 22 23:15:13.709 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:13.709 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/048.replace" to "/tmp/downstairs-vrx8aK6L/00/000/048.completed"
Sep 22 23:15:13.709 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:13.709 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:13.709 DEBG [0] It's time to notify for 514
Sep 22 23:15:13.709 INFO Completion from [0] id:514 status:true
Sep 22 23:15:13.709 INFO [515/752] Repair commands completed
Sep 22 23:15:13.709 INFO Pop front: ReconcileIO { id: ReconciliationId(515), op: ExtentReopen { repair_id: ReconciliationId(515), extent_id: 72 }, state: ClientData([New, New, New]) }
Sep 22 23:15:13.709 INFO Sent repair work, now wait for resp
Sep 22 23:15:13.709 INFO [0] received reconcile message
Sep 22 23:15:13.709 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(515), op: ExtentReopen { repair_id: ReconciliationId(515), extent_id: 72 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:13.709 INFO [0] client ExtentReopen { repair_id: ReconciliationId(515), extent_id: 72 }
Sep 22 23:15:13.709 INFO [1] received reconcile message
Sep 22 23:15:13.709 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(515), op: ExtentReopen { repair_id: ReconciliationId(515), extent_id: 72 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:13.709 INFO [1] client ExtentReopen { repair_id: ReconciliationId(515), extent_id: 72 }
Sep 22 23:15:13.709 INFO [2] received reconcile message
Sep 22 23:15:13.709 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(515), op: ExtentReopen { repair_id: ReconciliationId(515), extent_id: 72 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:13.709 INFO [2] client ExtentReopen { repair_id: ReconciliationId(515), extent_id: 72 }
Sep 22 23:15:13.710 DEBG 515 Reopen extent 72
Sep 22 23:15:13.710 DEBG 515 Reopen extent 72
Sep 22 23:15:13.711 DEBG 515 Reopen extent 72
Sep 22 23:15:13.712 DEBG [2] It's time to notify for 515
Sep 22 23:15:13.712 INFO Completion from [2] id:515 status:true
Sep 22 23:15:13.712 INFO [516/752] Repair commands completed
Sep 22 23:15:13.712 INFO Pop front: ReconcileIO { id: ReconciliationId(516), op: ExtentFlush { repair_id: ReconciliationId(516), extent_id: 21, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:13.712 INFO Sent repair work, now wait for resp
Sep 22 23:15:13.712 INFO [0] received reconcile message
Sep 22 23:15:13.712 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(516), op: ExtentFlush { repair_id: ReconciliationId(516), extent_id: 21, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:13.712 INFO [0] client ExtentFlush { repair_id: ReconciliationId(516), extent_id: 21, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:13.712 INFO [1] received reconcile message
Sep 22 23:15:13.712 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(516), op: ExtentFlush { repair_id: ReconciliationId(516), extent_id: 21, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:13.712 INFO [1] client ExtentFlush { repair_id: ReconciliationId(516), extent_id: 21, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:13.712 INFO [2] received reconcile message
Sep 22 23:15:13.712 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(516), op: ExtentFlush { repair_id: ReconciliationId(516), extent_id: 21, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:13.712 INFO [2] client ExtentFlush { repair_id: ReconciliationId(516), extent_id: 21, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:13.712 DEBG 516 Flush extent 21 with f:2 g:2
Sep 22 23:15:13.712 DEBG Flush just extent 21 with f:2 and g:2
Sep 22 23:15:13.712 DEBG [1] It's time to notify for 516
Sep 22 23:15:13.712 INFO Completion from [1] id:516 status:true
Sep 22 23:15:13.712 INFO [517/752] Repair commands completed
Sep 22 23:15:13.712 INFO Pop front: ReconcileIO { id: ReconciliationId(517), op: ExtentClose { repair_id: ReconciliationId(517), extent_id: 21 }, state: ClientData([New, New, New]) }
Sep 22 23:15:13.712 INFO Sent repair work, now wait for resp
Sep 22 23:15:13.712 INFO [0] received reconcile message
Sep 22 23:15:13.712 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(517), op: ExtentClose { repair_id: ReconciliationId(517), extent_id: 21 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:13.712 INFO [0] client ExtentClose { repair_id: ReconciliationId(517), extent_id: 21 }
Sep 22 23:15:13.712 INFO [1] received reconcile message
Sep 22 23:15:13.712 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(517), op: ExtentClose { repair_id: ReconciliationId(517), extent_id: 21 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:13.712 INFO [1] client ExtentClose { repair_id: ReconciliationId(517), extent_id: 21 }
Sep 22 23:15:13.712 INFO [2] received reconcile message
Sep 22 23:15:13.712 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(517), op: ExtentClose { repair_id: ReconciliationId(517), extent_id: 21 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:13.712 INFO [2] client ExtentClose { repair_id: ReconciliationId(517), extent_id: 21 }
Sep 22 23:15:13.713 DEBG 517 Close extent 21
Sep 22 23:15:13.713 DEBG 517 Close extent 21
Sep 22 23:15:13.713 DEBG 517 Close extent 21
Sep 22 23:15:13.714 DEBG [2] It's time to notify for 517
Sep 22 23:15:13.714 INFO Completion from [2] id:517 status:true
Sep 22 23:15:13.714 INFO [518/752] Repair commands completed
Sep 22 23:15:13.714 INFO Pop front: ReconcileIO { id: ReconciliationId(518), op: ExtentRepair { repair_id: ReconciliationId(518), extent_id: 21, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:13.714 INFO Sent repair work, now wait for resp
Sep 22 23:15:13.714 INFO [0] received reconcile message
Sep 22 23:15:13.714 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(518), op: ExtentRepair { repair_id: ReconciliationId(518), extent_id: 21, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:13.714 INFO [0] client ExtentRepair { repair_id: ReconciliationId(518), extent_id: 21, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:13.714 INFO [0] Sending repair request ReconciliationId(518)
Sep 22 23:15:13.714 INFO [1] received reconcile message
Sep 22 23:15:13.714 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(518), op: ExtentRepair { repair_id: ReconciliationId(518), extent_id: 21, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:13.714 INFO [1] client ExtentRepair { repair_id: ReconciliationId(518), extent_id: 21, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:13.714 INFO [1] No action required ReconciliationId(518)
Sep 22 23:15:13.714 INFO [2] received reconcile message
Sep 22 23:15:13.714 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(518), op: ExtentRepair { repair_id: ReconciliationId(518), extent_id: 21, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:13.714 INFO [2] client ExtentRepair { repair_id: ReconciliationId(518), extent_id: 21, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:13.714 INFO [2] No action required ReconciliationId(518)
Sep 22 23:15:13.714 DEBG 518 Repair extent 21 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:13.714 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/015.copy"
Sep 22 23:15:13.776 DEBG [1] Read AckReady 1086, : downstairs
Sep 22 23:15:13.777 DEBG up_ds_listen was notified
Sep 22 23:15:13.777 DEBG up_ds_listen process 1086
Sep 22 23:15:13.777 DEBG [A] ack job 1086:87, : downstairs
Sep 22 23:15:13.780 INFO accepted connection, remote_addr: 127.0.0.1:63666, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.780 TRCE incoming request, uri: /extent/21/files, method: GET, req_id: f8570990-1b21-46d1-b129-15f4b85d0a14, remote_addr: 127.0.0.1:63666, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.781 INFO request completed, latency_us: 268, response_code: 200, uri: /extent/21/files, method: GET, req_id: f8570990-1b21-46d1-b129-15f4b85d0a14, remote_addr: 127.0.0.1:63666, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.781 INFO eid:21 Found repair files: ["015", "015.db"]
Sep 22 23:15:13.781 TRCE incoming request, uri: /newextent/21/data, method: GET, req_id: 19f942f7-3d81-4d01-aa3c-f0b041a70d00, remote_addr: 127.0.0.1:63666, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.782 INFO request completed, latency_us: 342, response_code: 200, uri: /newextent/21/data, method: GET, req_id: 19f942f7-3d81-4d01-aa3c-f0b041a70d00, remote_addr: 127.0.0.1:63666, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.787 TRCE incoming request, uri: /newextent/21/db, method: GET, req_id: 7ce48312-1415-4b75-86d7-05f425cc7385, remote_addr: 127.0.0.1:63666, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.787 INFO request completed, latency_us: 307, response_code: 200, uri: /newextent/21/db, method: GET, req_id: 7ce48312-1415-4b75-86d7-05f425cc7385, remote_addr: 127.0.0.1:63666, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.788 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/015.copy" to "/tmp/downstairs-vrx8aK6L/00/000/015.replace"
Sep 22 23:15:13.788 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:13.790 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/015.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:13.790 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/015"
Sep 22 23:15:13.790 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/015.db"
Sep 22 23:15:13.790 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:13.790 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/015.replace" to "/tmp/downstairs-vrx8aK6L/00/000/015.completed"
Sep 22 23:15:13.790 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:13.790 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:13.790 DEBG [0] It's time to notify for 518
Sep 22 23:15:13.790 INFO Completion from [0] id:518 status:true
Sep 22 23:15:13.790 INFO [519/752] Repair commands completed
Sep 22 23:15:13.790 INFO Pop front: ReconcileIO { id: ReconciliationId(519), op: ExtentReopen { repair_id: ReconciliationId(519), extent_id: 21 }, state: ClientData([New, New, New]) }
Sep 22 23:15:13.790 INFO Sent repair work, now wait for resp
Sep 22 23:15:13.791 INFO [0] received reconcile message
Sep 22 23:15:13.791 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(519), op: ExtentReopen { repair_id: ReconciliationId(519), extent_id: 21 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:13.791 INFO [0] client ExtentReopen { repair_id: ReconciliationId(519), extent_id: 21 }
Sep 22 23:15:13.791 INFO [1] received reconcile message
Sep 22 23:15:13.791 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(519), op: ExtentReopen { repair_id: ReconciliationId(519), extent_id: 21 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:13.791 INFO [1] client ExtentReopen { repair_id: ReconciliationId(519), extent_id: 21 }
Sep 22 23:15:13.791 INFO [2] received reconcile message
Sep 22 23:15:13.791 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(519), op: ExtentReopen { repair_id: ReconciliationId(519), extent_id: 21 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:13.791 INFO [2] client ExtentReopen { repair_id: ReconciliationId(519), extent_id: 21 }
Sep 22 23:15:13.791 DEBG 519 Reopen extent 21
Sep 22 23:15:13.792 DEBG 519 Reopen extent 21
Sep 22 23:15:13.792 DEBG 519 Reopen extent 21
Sep 22 23:15:13.793 DEBG [2] It's time to notify for 519
Sep 22 23:15:13.793 INFO Completion from [2] id:519 status:true
Sep 22 23:15:13.793 INFO [520/752] Repair commands completed
Sep 22 23:15:13.793 INFO Pop front: ReconcileIO { id: ReconciliationId(520), op: ExtentFlush { repair_id: ReconciliationId(520), extent_id: 61, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:13.793 INFO Sent repair work, now wait for resp
Sep 22 23:15:13.793 INFO [0] received reconcile message
Sep 22 23:15:13.793 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(520), op: ExtentFlush { repair_id: ReconciliationId(520), extent_id: 61, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:13.793 INFO [0] client ExtentFlush { repair_id: ReconciliationId(520), extent_id: 61, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:13.793 INFO [1] received reconcile message
Sep 22 23:15:13.793 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(520), op: ExtentFlush { repair_id: ReconciliationId(520), extent_id: 61, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:13.793 INFO [1] client ExtentFlush { repair_id: ReconciliationId(520), extent_id: 61, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:13.793 INFO [2] received reconcile message
Sep 22 23:15:13.793 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(520), op: ExtentFlush { repair_id: ReconciliationId(520), extent_id: 61, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:13.793 INFO [2] client ExtentFlush { repair_id: ReconciliationId(520), extent_id: 61, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:13.793 DEBG 520 Flush extent 61 with f:2 g:2
Sep 22 23:15:13.793 DEBG Flush just extent 61 with f:2 and g:2
Sep 22 23:15:13.794 DEBG [1] It's time to notify for 520
Sep 22 23:15:13.794 INFO Completion from [1] id:520 status:true
Sep 22 23:15:13.794 INFO [521/752] Repair commands completed
Sep 22 23:15:13.794 INFO Pop front: ReconcileIO { id: ReconciliationId(521), op: ExtentClose { repair_id: ReconciliationId(521), extent_id: 61 }, state: ClientData([New, New, New]) }
Sep 22 23:15:13.794 INFO Sent repair work, now wait for resp
Sep 22 23:15:13.794 INFO [0] received reconcile message
Sep 22 23:15:13.794 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(521), op: ExtentClose { repair_id: ReconciliationId(521), extent_id: 61 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:13.794 INFO [0] client ExtentClose { repair_id: ReconciliationId(521), extent_id: 61 }
Sep 22 23:15:13.794 INFO [1] received reconcile message
Sep 22 23:15:13.794 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(521), op: ExtentClose { repair_id: ReconciliationId(521), extent_id: 61 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:13.794 INFO [1] client ExtentClose { repair_id: ReconciliationId(521), extent_id: 61 }
Sep 22 23:15:13.794 INFO [2] received reconcile message
Sep 22 23:15:13.794 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(521), op: ExtentClose { repair_id: ReconciliationId(521), extent_id: 61 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:13.794 INFO [2] client ExtentClose { repair_id: ReconciliationId(521), extent_id: 61 }
Sep 22 23:15:13.794 DEBG 521 Close extent 61
Sep 22 23:15:13.794 DEBG 521 Close extent 61
Sep 22 23:15:13.795 DEBG 521 Close extent 61
Sep 22 23:15:13.795 DEBG [2] It's time to notify for 521
Sep 22 23:15:13.795 INFO Completion from [2] id:521 status:true
Sep 22 23:15:13.795 INFO [522/752] Repair commands completed
Sep 22 23:15:13.795 INFO Pop front: ReconcileIO { id: ReconciliationId(522), op: ExtentRepair { repair_id: ReconciliationId(522), extent_id: 61, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:13.795 INFO Sent repair work, now wait for resp
Sep 22 23:15:13.795 INFO [0] received reconcile message
Sep 22 23:15:13.795 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(522), op: ExtentRepair { repair_id: ReconciliationId(522), extent_id: 61, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:13.795 INFO [0] client ExtentRepair { repair_id: ReconciliationId(522), extent_id: 61, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:13.795 INFO [0] Sending repair request ReconciliationId(522)
Sep 22 23:15:13.795 INFO [1] received reconcile message
Sep 22 23:15:13.795 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(522), op: ExtentRepair { repair_id: ReconciliationId(522), extent_id: 61, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:13.795 INFO [1] client ExtentRepair { repair_id: ReconciliationId(522), extent_id: 61, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:13.795 INFO [1] No action required ReconciliationId(522)
Sep 22 23:15:13.795 INFO [2] received reconcile message
Sep 22 23:15:13.795 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(522), op: ExtentRepair { repair_id: ReconciliationId(522), extent_id: 61, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:13.795 INFO [2] client ExtentRepair { repair_id: ReconciliationId(522), extent_id: 61, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:13.795 INFO [2] No action required ReconciliationId(522)
Sep 22 23:15:13.795 DEBG 522 Repair extent 61 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:13.796 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/03D.copy"
Sep 22 23:15:13.830 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:13.832 DEBG IO Read  1088 has deps [JobId(1087)]
Sep 22 23:15:13.858 INFO accepted connection, remote_addr: 127.0.0.1:41065, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.858 TRCE incoming request, uri: /extent/61/files, method: GET, req_id: 4d0cf51c-4325-4e6a-ac66-376cd904811e, remote_addr: 127.0.0.1:41065, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.858 INFO request completed, latency_us: 270, response_code: 200, uri: /extent/61/files, method: GET, req_id: 4d0cf51c-4325-4e6a-ac66-376cd904811e, remote_addr: 127.0.0.1:41065, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.858 INFO eid:61 Found repair files: ["03D", "03D.db"]
Sep 22 23:15:13.859 TRCE incoming request, uri: /newextent/61/data, method: GET, req_id: 7a757647-b0b7-4a02-8153-334e7a870d52, remote_addr: 127.0.0.1:41065, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.859 INFO request completed, latency_us: 366, response_code: 200, uri: /newextent/61/data, method: GET, req_id: 7a757647-b0b7-4a02-8153-334e7a870d52, remote_addr: 127.0.0.1:41065, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.864 TRCE incoming request, uri: /newextent/61/db, method: GET, req_id: df57f013-93c1-4fff-adec-dd7d76521b10, remote_addr: 127.0.0.1:41065, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.865 INFO request completed, latency_us: 294, response_code: 200, uri: /newextent/61/db, method: GET, req_id: df57f013-93c1-4fff-adec-dd7d76521b10, remote_addr: 127.0.0.1:41065, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.866 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/03D.copy" to "/tmp/downstairs-vrx8aK6L/00/000/03D.replace"
Sep 22 23:15:13.866 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:13.867 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/03D.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:13.867 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/03D"
Sep 22 23:15:13.867 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/03D.db"
Sep 22 23:15:13.867 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:13.867 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/03D.replace" to "/tmp/downstairs-vrx8aK6L/00/000/03D.completed"
Sep 22 23:15:13.867 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:13.867 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:13.868 DEBG [0] It's time to notify for 522
Sep 22 23:15:13.868 INFO Completion from [0] id:522 status:true
Sep 22 23:15:13.868 INFO [523/752] Repair commands completed
Sep 22 23:15:13.868 INFO Pop front: ReconcileIO { id: ReconciliationId(523), op: ExtentReopen { repair_id: ReconciliationId(523), extent_id: 61 }, state: ClientData([New, New, New]) }
Sep 22 23:15:13.868 INFO Sent repair work, now wait for resp
Sep 22 23:15:13.868 INFO [0] received reconcile message
Sep 22 23:15:13.868 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(523), op: ExtentReopen { repair_id: ReconciliationId(523), extent_id: 61 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:13.868 INFO [0] client ExtentReopen { repair_id: ReconciliationId(523), extent_id: 61 }
Sep 22 23:15:13.868 INFO [1] received reconcile message
Sep 22 23:15:13.868 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(523), op: ExtentReopen { repair_id: ReconciliationId(523), extent_id: 61 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:13.868 INFO [1] client ExtentReopen { repair_id: ReconciliationId(523), extent_id: 61 }
Sep 22 23:15:13.868 INFO [2] received reconcile message
Sep 22 23:15:13.868 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(523), op: ExtentReopen { repair_id: ReconciliationId(523), extent_id: 61 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:13.868 INFO [2] client ExtentReopen { repair_id: ReconciliationId(523), extent_id: 61 }
Sep 22 23:15:13.868 DEBG 523 Reopen extent 61
Sep 22 23:15:13.869 DEBG 523 Reopen extent 61
Sep 22 23:15:13.869 DEBG 523 Reopen extent 61
Sep 22 23:15:13.870 DEBG [2] It's time to notify for 523
Sep 22 23:15:13.870 INFO Completion from [2] id:523 status:true
Sep 22 23:15:13.870 INFO [524/752] Repair commands completed
Sep 22 23:15:13.870 INFO Pop front: ReconcileIO { id: ReconciliationId(524), op: ExtentFlush { repair_id: ReconciliationId(524), extent_id: 134, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:13.870 INFO Sent repair work, now wait for resp
Sep 22 23:15:13.870 INFO [0] received reconcile message
Sep 22 23:15:13.870 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(524), op: ExtentFlush { repair_id: ReconciliationId(524), extent_id: 134, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:13.870 INFO [0] client ExtentFlush { repair_id: ReconciliationId(524), extent_id: 134, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:13.870 INFO [1] received reconcile message
Sep 22 23:15:13.870 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(524), op: ExtentFlush { repair_id: ReconciliationId(524), extent_id: 134, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:13.870 INFO [1] client ExtentFlush { repair_id: ReconciliationId(524), extent_id: 134, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:13.870 INFO [2] received reconcile message
Sep 22 23:15:13.870 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(524), op: ExtentFlush { repair_id: ReconciliationId(524), extent_id: 134, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:13.870 INFO [2] client ExtentFlush { repair_id: ReconciliationId(524), extent_id: 134, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:13.871 DEBG 524 Flush extent 134 with f:2 g:2
Sep 22 23:15:13.871 DEBG Flush just extent 134 with f:2 and g:2
Sep 22 23:15:13.871 DEBG [1] It's time to notify for 524
Sep 22 23:15:13.871 INFO Completion from [1] id:524 status:true
Sep 22 23:15:13.871 INFO [525/752] Repair commands completed
Sep 22 23:15:13.871 INFO Pop front: ReconcileIO { id: ReconciliationId(525), op: ExtentClose { repair_id: ReconciliationId(525), extent_id: 134 }, state: ClientData([New, New, New]) }
Sep 22 23:15:13.871 INFO Sent repair work, now wait for resp
Sep 22 23:15:13.871 INFO [0] received reconcile message
Sep 22 23:15:13.871 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(525), op: ExtentClose { repair_id: ReconciliationId(525), extent_id: 134 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:13.871 INFO [0] client ExtentClose { repair_id: ReconciliationId(525), extent_id: 134 }
Sep 22 23:15:13.871 INFO [1] received reconcile message
Sep 22 23:15:13.871 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(525), op: ExtentClose { repair_id: ReconciliationId(525), extent_id: 134 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:13.871 INFO [1] client ExtentClose { repair_id: ReconciliationId(525), extent_id: 134 }
Sep 22 23:15:13.871 INFO [2] received reconcile message
Sep 22 23:15:13.871 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(525), op: ExtentClose { repair_id: ReconciliationId(525), extent_id: 134 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:13.871 INFO [2] client ExtentClose { repair_id: ReconciliationId(525), extent_id: 134 }
Sep 22 23:15:13.871 DEBG 525 Close extent 134
Sep 22 23:15:13.871 DEBG 525 Close extent 134
Sep 22 23:15:13.872 DEBG 525 Close extent 134
Sep 22 23:15:13.872 DEBG [2] It's time to notify for 525
Sep 22 23:15:13.872 INFO Completion from [2] id:525 status:true
Sep 22 23:15:13.872 INFO [526/752] Repair commands completed
Sep 22 23:15:13.872 INFO Pop front: ReconcileIO { id: ReconciliationId(526), op: ExtentRepair { repair_id: ReconciliationId(526), extent_id: 134, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:13.872 INFO Sent repair work, now wait for resp
Sep 22 23:15:13.872 INFO [0] received reconcile message
Sep 22 23:15:13.872 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(526), op: ExtentRepair { repair_id: ReconciliationId(526), extent_id: 134, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:13.872 INFO [0] client ExtentRepair { repair_id: ReconciliationId(526), extent_id: 134, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:13.872 INFO [0] Sending repair request ReconciliationId(526)
Sep 22 23:15:13.872 INFO [1] received reconcile message
Sep 22 23:15:13.872 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(526), op: ExtentRepair { repair_id: ReconciliationId(526), extent_id: 134, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:13.872 INFO [1] client ExtentRepair { repair_id: ReconciliationId(526), extent_id: 134, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:13.872 INFO [1] No action required ReconciliationId(526)
Sep 22 23:15:13.872 INFO [2] received reconcile message
Sep 22 23:15:13.872 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(526), op: ExtentRepair { repair_id: ReconciliationId(526), extent_id: 134, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:13.872 INFO [2] client ExtentRepair { repair_id: ReconciliationId(526), extent_id: 134, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:13.872 INFO [2] No action required ReconciliationId(526)
Sep 22 23:15:13.873 DEBG 526 Repair extent 134 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:13.873 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/086.copy"
Sep 22 23:15:13.935 INFO accepted connection, remote_addr: 127.0.0.1:54478, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.936 TRCE incoming request, uri: /extent/134/files, method: GET, req_id: bbd822cb-34e9-4365-a104-a4653e2342b2, remote_addr: 127.0.0.1:54478, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.936 INFO request completed, latency_us: 272, response_code: 200, uri: /extent/134/files, method: GET, req_id: bbd822cb-34e9-4365-a104-a4653e2342b2, remote_addr: 127.0.0.1:54478, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.936 INFO eid:134 Found repair files: ["086", "086.db"]
Sep 22 23:15:13.936 TRCE incoming request, uri: /newextent/134/data, method: GET, req_id: f8e649c0-3983-4f1f-86f8-e92d0f1a8e2d, remote_addr: 127.0.0.1:54478, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.937 INFO request completed, latency_us: 368, response_code: 200, uri: /newextent/134/data, method: GET, req_id: f8e649c0-3983-4f1f-86f8-e92d0f1a8e2d, remote_addr: 127.0.0.1:54478, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.942 TRCE incoming request, uri: /newextent/134/db, method: GET, req_id: d670367b-fc4c-4ebe-a643-36ead02ff54c, remote_addr: 127.0.0.1:54478, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.942 INFO request completed, latency_us: 295, response_code: 200, uri: /newextent/134/db, method: GET, req_id: d670367b-fc4c-4ebe-a643-36ead02ff54c, remote_addr: 127.0.0.1:54478, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:13.944 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/086.copy" to "/tmp/downstairs-vrx8aK6L/00/000/086.replace"
Sep 22 23:15:13.944 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:13.945 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/086.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:13.945 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/086"
Sep 22 23:15:13.945 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/086.db"
Sep 22 23:15:13.945 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:13.945 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/086.replace" to "/tmp/downstairs-vrx8aK6L/00/000/086.completed"
Sep 22 23:15:13.945 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:13.945 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:13.946 DEBG [0] It's time to notify for 526
Sep 22 23:15:13.946 INFO Completion from [0] id:526 status:true
Sep 22 23:15:13.946 INFO [527/752] Repair commands completed
Sep 22 23:15:13.946 INFO Pop front: ReconcileIO { id: ReconciliationId(527), op: ExtentReopen { repair_id: ReconciliationId(527), extent_id: 134 }, state: ClientData([New, New, New]) }
Sep 22 23:15:13.946 INFO Sent repair work, now wait for resp
Sep 22 23:15:13.946 INFO [0] received reconcile message
Sep 22 23:15:13.946 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(527), op: ExtentReopen { repair_id: ReconciliationId(527), extent_id: 134 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:13.946 INFO [0] client ExtentReopen { repair_id: ReconciliationId(527), extent_id: 134 }
Sep 22 23:15:13.946 INFO [1] received reconcile message
Sep 22 23:15:13.946 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(527), op: ExtentReopen { repair_id: ReconciliationId(527), extent_id: 134 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:13.946 INFO [1] client ExtentReopen { repair_id: ReconciliationId(527), extent_id: 134 }
Sep 22 23:15:13.946 INFO [2] received reconcile message
Sep 22 23:15:13.946 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(527), op: ExtentReopen { repair_id: ReconciliationId(527), extent_id: 134 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:13.946 INFO [2] client ExtentReopen { repair_id: ReconciliationId(527), extent_id: 134 }
Sep 22 23:15:13.946 DEBG 527 Reopen extent 134
Sep 22 23:15:13.947 DEBG 527 Reopen extent 134
Sep 22 23:15:13.948 DEBG 527 Reopen extent 134
Sep 22 23:15:13.948 DEBG [2] It's time to notify for 527
Sep 22 23:15:13.948 INFO Completion from [2] id:527 status:true
Sep 22 23:15:13.948 INFO [528/752] Repair commands completed
Sep 22 23:15:13.948 INFO Pop front: ReconcileIO { id: ReconciliationId(528), op: ExtentFlush { repair_id: ReconciliationId(528), extent_id: 46, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:13.948 INFO Sent repair work, now wait for resp
Sep 22 23:15:13.948 INFO [0] received reconcile message
Sep 22 23:15:13.948 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(528), op: ExtentFlush { repair_id: ReconciliationId(528), extent_id: 46, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:13.948 INFO [0] client ExtentFlush { repair_id: ReconciliationId(528), extent_id: 46, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:13.948 INFO [1] received reconcile message
Sep 22 23:15:13.948 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(528), op: ExtentFlush { repair_id: ReconciliationId(528), extent_id: 46, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:13.949 INFO [1] client ExtentFlush { repair_id: ReconciliationId(528), extent_id: 46, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:13.949 INFO [2] received reconcile message
Sep 22 23:15:13.949 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(528), op: ExtentFlush { repair_id: ReconciliationId(528), extent_id: 46, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:13.949 INFO [2] client ExtentFlush { repair_id: ReconciliationId(528), extent_id: 46, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:13.949 DEBG 528 Flush extent 46 with f:2 g:2
Sep 22 23:15:13.949 DEBG Flush just extent 46 with f:2 and g:2
Sep 22 23:15:13.949 DEBG [1] It's time to notify for 528
Sep 22 23:15:13.949 INFO Completion from [1] id:528 status:true
Sep 22 23:15:13.949 INFO [529/752] Repair commands completed
Sep 22 23:15:13.949 INFO Pop front: ReconcileIO { id: ReconciliationId(529), op: ExtentClose { repair_id: ReconciliationId(529), extent_id: 46 }, state: ClientData([New, New, New]) }
Sep 22 23:15:13.949 INFO Sent repair work, now wait for resp
Sep 22 23:15:13.949 INFO [0] received reconcile message
Sep 22 23:15:13.949 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(529), op: ExtentClose { repair_id: ReconciliationId(529), extent_id: 46 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:13.949 INFO [0] client ExtentClose { repair_id: ReconciliationId(529), extent_id: 46 }
Sep 22 23:15:13.949 INFO [1] received reconcile message
Sep 22 23:15:13.949 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(529), op: ExtentClose { repair_id: ReconciliationId(529), extent_id: 46 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:13.949 INFO [1] client ExtentClose { repair_id: ReconciliationId(529), extent_id: 46 }
Sep 22 23:15:13.949 INFO [2] received reconcile message
Sep 22 23:15:13.949 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(529), op: ExtentClose { repair_id: ReconciliationId(529), extent_id: 46 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:13.949 INFO [2] client ExtentClose { repair_id: ReconciliationId(529), extent_id: 46 }
Sep 22 23:15:13.949 DEBG 529 Close extent 46
Sep 22 23:15:13.950 DEBG 529 Close extent 46
Sep 22 23:15:13.950 DEBG 529 Close extent 46
Sep 22 23:15:13.950 DEBG [2] It's time to notify for 529
Sep 22 23:15:13.950 INFO Completion from [2] id:529 status:true
Sep 22 23:15:13.950 INFO [530/752] Repair commands completed
Sep 22 23:15:13.950 INFO Pop front: ReconcileIO { id: ReconciliationId(530), op: ExtentRepair { repair_id: ReconciliationId(530), extent_id: 46, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:13.950 INFO Sent repair work, now wait for resp
Sep 22 23:15:13.950 INFO [0] received reconcile message
Sep 22 23:15:13.950 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(530), op: ExtentRepair { repair_id: ReconciliationId(530), extent_id: 46, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:13.950 INFO [0] client ExtentRepair { repair_id: ReconciliationId(530), extent_id: 46, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:13.950 INFO [0] Sending repair request ReconciliationId(530)
Sep 22 23:15:13.950 INFO [1] received reconcile message
Sep 22 23:15:13.950 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(530), op: ExtentRepair { repair_id: ReconciliationId(530), extent_id: 46, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:13.950 INFO [1] client ExtentRepair { repair_id: ReconciliationId(530), extent_id: 46, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:13.951 INFO [1] No action required ReconciliationId(530)
Sep 22 23:15:13.951 INFO [2] received reconcile message
Sep 22 23:15:13.951 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(530), op: ExtentRepair { repair_id: ReconciliationId(530), extent_id: 46, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:13.951 INFO [2] client ExtentRepair { repair_id: ReconciliationId(530), extent_id: 46, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:13.951 INFO [2] No action required ReconciliationId(530)
Sep 22 23:15:13.951 DEBG 530 Repair extent 46 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:13.951 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/02E.copy"
Sep 22 23:15:14.016 INFO accepted connection, remote_addr: 127.0.0.1:46657, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.016 TRCE incoming request, uri: /extent/46/files, method: GET, req_id: 6a7f8166-98c2-4bf6-88e7-1fdb39d9cf88, remote_addr: 127.0.0.1:46657, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.016 INFO request completed, latency_us: 266, response_code: 200, uri: /extent/46/files, method: GET, req_id: 6a7f8166-98c2-4bf6-88e7-1fdb39d9cf88, remote_addr: 127.0.0.1:46657, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.017 INFO eid:46 Found repair files: ["02E", "02E.db"]
Sep 22 23:15:14.017 TRCE incoming request, uri: /newextent/46/data, method: GET, req_id: 8b1a172d-9453-4122-9664-af183f4cb360, remote_addr: 127.0.0.1:46657, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.017 INFO request completed, latency_us: 356, response_code: 200, uri: /newextent/46/data, method: GET, req_id: 8b1a172d-9453-4122-9664-af183f4cb360, remote_addr: 127.0.0.1:46657, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.023 TRCE incoming request, uri: /newextent/46/db, method: GET, req_id: dbce502e-9300-4180-88be-aa363f185c98, remote_addr: 127.0.0.1:46657, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.023 INFO request completed, latency_us: 292, response_code: 200, uri: /newextent/46/db, method: GET, req_id: dbce502e-9300-4180-88be-aa363f185c98, remote_addr: 127.0.0.1:46657, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.024 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/02E.copy" to "/tmp/downstairs-vrx8aK6L/00/000/02E.replace"
Sep 22 23:15:14.024 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:14.025 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/02E.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:14.026 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/02E"
Sep 22 23:15:14.026 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/02E.db"
Sep 22 23:15:14.026 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:14.026 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/02E.replace" to "/tmp/downstairs-vrx8aK6L/00/000/02E.completed"
Sep 22 23:15:14.026 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:14.026 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:14.026 DEBG [0] It's time to notify for 530
Sep 22 23:15:14.026 INFO Completion from [0] id:530 status:true
Sep 22 23:15:14.026 INFO [531/752] Repair commands completed
Sep 22 23:15:14.026 INFO Pop front: ReconcileIO { id: ReconciliationId(531), op: ExtentReopen { repair_id: ReconciliationId(531), extent_id: 46 }, state: ClientData([New, New, New]) }
Sep 22 23:15:14.026 INFO Sent repair work, now wait for resp
Sep 22 23:15:14.026 INFO [0] received reconcile message
Sep 22 23:15:14.026 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(531), op: ExtentReopen { repair_id: ReconciliationId(531), extent_id: 46 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:14.027 INFO [0] client ExtentReopen { repair_id: ReconciliationId(531), extent_id: 46 }
Sep 22 23:15:14.027 INFO [1] received reconcile message
Sep 22 23:15:14.027 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(531), op: ExtentReopen { repair_id: ReconciliationId(531), extent_id: 46 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:14.027 INFO [1] client ExtentReopen { repair_id: ReconciliationId(531), extent_id: 46 }
Sep 22 23:15:14.027 INFO [2] received reconcile message
Sep 22 23:15:14.027 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(531), op: ExtentReopen { repair_id: ReconciliationId(531), extent_id: 46 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:14.027 INFO [2] client ExtentReopen { repair_id: ReconciliationId(531), extent_id: 46 }
Sep 22 23:15:14.027 DEBG 531 Reopen extent 46
Sep 22 23:15:14.028 DEBG 531 Reopen extent 46
Sep 22 23:15:14.028 DEBG 531 Reopen extent 46
Sep 22 23:15:14.029 DEBG [2] It's time to notify for 531
Sep 22 23:15:14.029 INFO Completion from [2] id:531 status:true
Sep 22 23:15:14.029 INFO [532/752] Repair commands completed
Sep 22 23:15:14.029 INFO Pop front: ReconcileIO { id: ReconciliationId(532), op: ExtentFlush { repair_id: ReconciliationId(532), extent_id: 41, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:14.029 INFO Sent repair work, now wait for resp
Sep 22 23:15:14.029 INFO [0] received reconcile message
Sep 22 23:15:14.029 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(532), op: ExtentFlush { repair_id: ReconciliationId(532), extent_id: 41, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:14.029 INFO [0] client ExtentFlush { repair_id: ReconciliationId(532), extent_id: 41, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:14.029 INFO [1] received reconcile message
Sep 22 23:15:14.029 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(532), op: ExtentFlush { repair_id: ReconciliationId(532), extent_id: 41, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:14.029 INFO [1] client ExtentFlush { repair_id: ReconciliationId(532), extent_id: 41, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:14.029 INFO [2] received reconcile message
Sep 22 23:15:14.029 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(532), op: ExtentFlush { repair_id: ReconciliationId(532), extent_id: 41, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:14.029 INFO [2] client ExtentFlush { repair_id: ReconciliationId(532), extent_id: 41, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:14.029 DEBG 532 Flush extent 41 with f:2 g:2
Sep 22 23:15:14.029 DEBG Flush just extent 41 with f:2 and g:2
Sep 22 23:15:14.029 DEBG [1] It's time to notify for 532
Sep 22 23:15:14.029 INFO Completion from [1] id:532 status:true
Sep 22 23:15:14.029 INFO [533/752] Repair commands completed
Sep 22 23:15:14.029 INFO Pop front: ReconcileIO { id: ReconciliationId(533), op: ExtentClose { repair_id: ReconciliationId(533), extent_id: 41 }, state: ClientData([New, New, New]) }
Sep 22 23:15:14.030 INFO Sent repair work, now wait for resp
Sep 22 23:15:14.030 INFO [0] received reconcile message
Sep 22 23:15:14.030 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(533), op: ExtentClose { repair_id: ReconciliationId(533), extent_id: 41 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:14.030 INFO [0] client ExtentClose { repair_id: ReconciliationId(533), extent_id: 41 }
Sep 22 23:15:14.030 INFO [1] received reconcile message
Sep 22 23:15:14.030 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(533), op: ExtentClose { repair_id: ReconciliationId(533), extent_id: 41 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:14.030 INFO [1] client ExtentClose { repair_id: ReconciliationId(533), extent_id: 41 }
Sep 22 23:15:14.030 INFO [2] received reconcile message
Sep 22 23:15:14.030 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(533), op: ExtentClose { repair_id: ReconciliationId(533), extent_id: 41 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:14.030 INFO [2] client ExtentClose { repair_id: ReconciliationId(533), extent_id: 41 }
Sep 22 23:15:14.030 DEBG 533 Close extent 41
Sep 22 23:15:14.030 DEBG 533 Close extent 41
Sep 22 23:15:14.030 DEBG 533 Close extent 41
Sep 22 23:15:14.031 DEBG [2] It's time to notify for 533
Sep 22 23:15:14.031 INFO Completion from [2] id:533 status:true
Sep 22 23:15:14.031 INFO [534/752] Repair commands completed
Sep 22 23:15:14.031 INFO Pop front: ReconcileIO { id: ReconciliationId(534), op: ExtentRepair { repair_id: ReconciliationId(534), extent_id: 41, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:14.031 INFO Sent repair work, now wait for resp
Sep 22 23:15:14.031 INFO [0] received reconcile message
Sep 22 23:15:14.031 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(534), op: ExtentRepair { repair_id: ReconciliationId(534), extent_id: 41, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:14.031 INFO [0] client ExtentRepair { repair_id: ReconciliationId(534), extent_id: 41, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:14.031 INFO [0] Sending repair request ReconciliationId(534)
Sep 22 23:15:14.031 INFO [1] received reconcile message
Sep 22 23:15:14.031 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(534), op: ExtentRepair { repair_id: ReconciliationId(534), extent_id: 41, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:14.031 INFO [1] client ExtentRepair { repair_id: ReconciliationId(534), extent_id: 41, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:14.031 INFO [1] No action required ReconciliationId(534)
Sep 22 23:15:14.031 INFO [2] received reconcile message
Sep 22 23:15:14.031 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(534), op: ExtentRepair { repair_id: ReconciliationId(534), extent_id: 41, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:14.031 INFO [2] client ExtentRepair { repair_id: ReconciliationId(534), extent_id: 41, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:14.031 INFO [2] No action required ReconciliationId(534)
Sep 22 23:15:14.031 DEBG 534 Repair extent 41 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:14.031 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/029.copy"
Sep 22 23:15:14.096 INFO accepted connection, remote_addr: 127.0.0.1:63961, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.097 TRCE incoming request, uri: /extent/41/files, method: GET, req_id: 725ac720-fad4-4c89-98d9-0b1ababd161e, remote_addr: 127.0.0.1:63961, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.097 INFO request completed, latency_us: 244, response_code: 200, uri: /extent/41/files, method: GET, req_id: 725ac720-fad4-4c89-98d9-0b1ababd161e, remote_addr: 127.0.0.1:63961, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.097 INFO eid:41 Found repair files: ["029", "029.db"]
Sep 22 23:15:14.097 TRCE incoming request, uri: /newextent/41/data, method: GET, req_id: 009e6c50-8441-4e9e-9246-4ca0383bac29, remote_addr: 127.0.0.1:63961, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.098 INFO request completed, latency_us: 341, response_code: 200, uri: /newextent/41/data, method: GET, req_id: 009e6c50-8441-4e9e-9246-4ca0383bac29, remote_addr: 127.0.0.1:63961, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.098 DEBG Read      :1086 deps:[JobId(1085)] res:true
Sep 22 23:15:14.103 TRCE incoming request, uri: /newextent/41/db, method: GET, req_id: 76cde1a7-856c-409c-9926-a5a2011a5da1, remote_addr: 127.0.0.1:63961, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.103 INFO request completed, latency_us: 292, response_code: 200, uri: /newextent/41/db, method: GET, req_id: 76cde1a7-856c-409c-9926-a5a2011a5da1, remote_addr: 127.0.0.1:63961, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.105 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/029.copy" to "/tmp/downstairs-vrx8aK6L/00/000/029.replace"
Sep 22 23:15:14.105 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:14.106 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/029.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:14.106 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/029"
Sep 22 23:15:14.106 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/029.db"
Sep 22 23:15:14.106 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:14.106 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/029.replace" to "/tmp/downstairs-vrx8aK6L/00/000/029.completed"
Sep 22 23:15:14.106 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:14.106 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:14.106 DEBG [0] It's time to notify for 534
Sep 22 23:15:14.107 INFO Completion from [0] id:534 status:true
Sep 22 23:15:14.107 INFO [535/752] Repair commands completed
Sep 22 23:15:14.107 INFO Pop front: ReconcileIO { id: ReconciliationId(535), op: ExtentReopen { repair_id: ReconciliationId(535), extent_id: 41 }, state: ClientData([New, New, New]) }
Sep 22 23:15:14.107 INFO Sent repair work, now wait for resp
Sep 22 23:15:14.107 INFO [0] received reconcile message
Sep 22 23:15:14.107 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(535), op: ExtentReopen { repair_id: ReconciliationId(535), extent_id: 41 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:14.107 INFO [0] client ExtentReopen { repair_id: ReconciliationId(535), extent_id: 41 }
Sep 22 23:15:14.107 INFO [1] received reconcile message
Sep 22 23:15:14.107 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(535), op: ExtentReopen { repair_id: ReconciliationId(535), extent_id: 41 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:14.107 INFO [1] client ExtentReopen { repair_id: ReconciliationId(535), extent_id: 41 }
Sep 22 23:15:14.107 INFO [2] received reconcile message
Sep 22 23:15:14.107 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(535), op: ExtentReopen { repair_id: ReconciliationId(535), extent_id: 41 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:14.107 INFO [2] client ExtentReopen { repair_id: ReconciliationId(535), extent_id: 41 }
Sep 22 23:15:14.107 DEBG 535 Reopen extent 41
Sep 22 23:15:14.108 DEBG 535 Reopen extent 41
Sep 22 23:15:14.108 DEBG 535 Reopen extent 41
Sep 22 23:15:14.109 DEBG [2] It's time to notify for 535
Sep 22 23:15:14.109 INFO Completion from [2] id:535 status:true
Sep 22 23:15:14.109 INFO [536/752] Repair commands completed
Sep 22 23:15:14.109 INFO Pop front: ReconcileIO { id: ReconciliationId(536), op: ExtentFlush { repair_id: ReconciliationId(536), extent_id: 22, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:14.109 INFO Sent repair work, now wait for resp
Sep 22 23:15:14.109 INFO [0] received reconcile message
Sep 22 23:15:14.109 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(536), op: ExtentFlush { repair_id: ReconciliationId(536), extent_id: 22, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:14.109 INFO [0] client ExtentFlush { repair_id: ReconciliationId(536), extent_id: 22, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:14.109 INFO [1] received reconcile message
Sep 22 23:15:14.109 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(536), op: ExtentFlush { repair_id: ReconciliationId(536), extent_id: 22, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:14.109 INFO [1] client ExtentFlush { repair_id: ReconciliationId(536), extent_id: 22, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:14.109 INFO [2] received reconcile message
Sep 22 23:15:14.109 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(536), op: ExtentFlush { repair_id: ReconciliationId(536), extent_id: 22, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:14.109 INFO [2] client ExtentFlush { repair_id: ReconciliationId(536), extent_id: 22, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:14.109 DEBG 536 Flush extent 22 with f:2 g:2
Sep 22 23:15:14.109 DEBG Flush just extent 22 with f:2 and g:2
Sep 22 23:15:14.109 DEBG [1] It's time to notify for 536
Sep 22 23:15:14.110 INFO Completion from [1] id:536 status:true
Sep 22 23:15:14.110 INFO [537/752] Repair commands completed
Sep 22 23:15:14.110 INFO Pop front: ReconcileIO { id: ReconciliationId(537), op: ExtentClose { repair_id: ReconciliationId(537), extent_id: 22 }, state: ClientData([New, New, New]) }
Sep 22 23:15:14.110 INFO Sent repair work, now wait for resp
Sep 22 23:15:14.110 INFO [0] received reconcile message
Sep 22 23:15:14.110 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(537), op: ExtentClose { repair_id: ReconciliationId(537), extent_id: 22 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:14.110 INFO [0] client ExtentClose { repair_id: ReconciliationId(537), extent_id: 22 }
Sep 22 23:15:14.110 INFO [1] received reconcile message
Sep 22 23:15:14.110 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(537), op: ExtentClose { repair_id: ReconciliationId(537), extent_id: 22 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:14.110 INFO [1] client ExtentClose { repair_id: ReconciliationId(537), extent_id: 22 }
Sep 22 23:15:14.110 INFO [2] received reconcile message
Sep 22 23:15:14.110 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(537), op: ExtentClose { repair_id: ReconciliationId(537), extent_id: 22 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:14.110 INFO [2] client ExtentClose { repair_id: ReconciliationId(537), extent_id: 22 }
Sep 22 23:15:14.110 DEBG 537 Close extent 22
Sep 22 23:15:14.110 DEBG 537 Close extent 22
Sep 22 23:15:14.111 DEBG 537 Close extent 22
Sep 22 23:15:14.111 DEBG [2] It's time to notify for 537
Sep 22 23:15:14.111 INFO Completion from [2] id:537 status:true
Sep 22 23:15:14.111 INFO [538/752] Repair commands completed
Sep 22 23:15:14.111 INFO Pop front: ReconcileIO { id: ReconciliationId(538), op: ExtentRepair { repair_id: ReconciliationId(538), extent_id: 22, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:14.111 INFO Sent repair work, now wait for resp
Sep 22 23:15:14.111 INFO [0] received reconcile message
Sep 22 23:15:14.111 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(538), op: ExtentRepair { repair_id: ReconciliationId(538), extent_id: 22, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:14.111 INFO [0] client ExtentRepair { repair_id: ReconciliationId(538), extent_id: 22, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:14.111 INFO [0] Sending repair request ReconciliationId(538)
Sep 22 23:15:14.111 INFO [1] received reconcile message
Sep 22 23:15:14.111 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(538), op: ExtentRepair { repair_id: ReconciliationId(538), extent_id: 22, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:14.111 INFO [1] client ExtentRepair { repair_id: ReconciliationId(538), extent_id: 22, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:14.111 INFO [1] No action required ReconciliationId(538)
Sep 22 23:15:14.111 INFO [2] received reconcile message
Sep 22 23:15:14.111 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(538), op: ExtentRepair { repair_id: ReconciliationId(538), extent_id: 22, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:14.111 INFO [2] client ExtentRepair { repair_id: ReconciliationId(538), extent_id: 22, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:14.111 INFO [2] No action required ReconciliationId(538)
Sep 22 23:15:14.111 DEBG 538 Repair extent 22 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:14.111 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/016.copy"
Sep 22 23:15:14.120 INFO [lossy] skipping 1086
Sep 22 23:15:14.126 DEBG Read      :1086 deps:[JobId(1085)] res:true
Sep 22 23:15:14.150 INFO [lossy] skipping 1087
Sep 22 23:15:14.150 INFO [lossy] skipping 1088
Sep 22 23:15:14.150 DEBG Flush     :1087 extent_limit None deps:[JobId(1086), JobId(1085)] res:true f:33 g:1
Sep 22 23:15:14.156 DEBG Read      :1088 deps:[JobId(1087)] res:true
Sep 22 23:15:14.176 INFO accepted connection, remote_addr: 127.0.0.1:39939, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.176 TRCE incoming request, uri: /extent/22/files, method: GET, req_id: 7abee1b1-15f9-410b-bbf9-acef2c8b6778, remote_addr: 127.0.0.1:39939, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.176 INFO request completed, latency_us: 277, response_code: 200, uri: /extent/22/files, method: GET, req_id: 7abee1b1-15f9-410b-bbf9-acef2c8b6778, remote_addr: 127.0.0.1:39939, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.177 INFO eid:22 Found repair files: ["016", "016.db"]
Sep 22 23:15:14.177 TRCE incoming request, uri: /newextent/22/data, method: GET, req_id: a01f6768-a5ae-4063-af4d-f1fb5a44c48a, remote_addr: 127.0.0.1:39939, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.177 INFO request completed, latency_us: 379, response_code: 200, uri: /newextent/22/data, method: GET, req_id: a01f6768-a5ae-4063-af4d-f1fb5a44c48a, remote_addr: 127.0.0.1:39939, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.179 INFO [lossy] sleeping 1 second
Sep 22 23:15:14.183 TRCE incoming request, uri: /newextent/22/db, method: GET, req_id: bb3d2cac-681c-4778-b707-7f86389e59bf, remote_addr: 127.0.0.1:39939, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.183 INFO request completed, latency_us: 295, response_code: 200, uri: /newextent/22/db, method: GET, req_id: bb3d2cac-681c-4778-b707-7f86389e59bf, remote_addr: 127.0.0.1:39939, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.184 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/016.copy" to "/tmp/downstairs-vrx8aK6L/00/000/016.replace"
Sep 22 23:15:14.184 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:14.185 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/016.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:14.185 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/016"
Sep 22 23:15:14.186 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/016.db"
Sep 22 23:15:14.186 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:14.186 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/016.replace" to "/tmp/downstairs-vrx8aK6L/00/000/016.completed"
Sep 22 23:15:14.186 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:14.186 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:14.186 DEBG [0] It's time to notify for 538
Sep 22 23:15:14.186 INFO Completion from [0] id:538 status:true
Sep 22 23:15:14.186 INFO [539/752] Repair commands completed
Sep 22 23:15:14.186 INFO Pop front: ReconcileIO { id: ReconciliationId(539), op: ExtentReopen { repair_id: ReconciliationId(539), extent_id: 22 }, state: ClientData([New, New, New]) }
Sep 22 23:15:14.186 INFO Sent repair work, now wait for resp
Sep 22 23:15:14.186 INFO [0] received reconcile message
Sep 22 23:15:14.186 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(539), op: ExtentReopen { repair_id: ReconciliationId(539), extent_id: 22 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:14.186 INFO [0] client ExtentReopen { repair_id: ReconciliationId(539), extent_id: 22 }
Sep 22 23:15:14.186 INFO [1] received reconcile message
Sep 22 23:15:14.186 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(539), op: ExtentReopen { repair_id: ReconciliationId(539), extent_id: 22 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:14.186 INFO [1] client ExtentReopen { repair_id: ReconciliationId(539), extent_id: 22 }
Sep 22 23:15:14.186 INFO [2] received reconcile message
Sep 22 23:15:14.186 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(539), op: ExtentReopen { repair_id: ReconciliationId(539), extent_id: 22 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:14.186 INFO [2] client ExtentReopen { repair_id: ReconciliationId(539), extent_id: 22 }
Sep 22 23:15:14.186 DEBG 539 Reopen extent 22
Sep 22 23:15:14.187 DEBG 539 Reopen extent 22
Sep 22 23:15:14.188 DEBG 539 Reopen extent 22
Sep 22 23:15:14.188 DEBG [2] It's time to notify for 539
Sep 22 23:15:14.188 INFO Completion from [2] id:539 status:true
Sep 22 23:15:14.188 INFO [540/752] Repair commands completed
Sep 22 23:15:14.188 INFO Pop front: ReconcileIO { id: ReconciliationId(540), op: ExtentFlush { repair_id: ReconciliationId(540), extent_id: 56, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:14.189 INFO Sent repair work, now wait for resp
Sep 22 23:15:14.189 INFO [0] received reconcile message
Sep 22 23:15:14.189 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(540), op: ExtentFlush { repair_id: ReconciliationId(540), extent_id: 56, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:14.189 INFO [0] client ExtentFlush { repair_id: ReconciliationId(540), extent_id: 56, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:14.189 INFO [1] received reconcile message
Sep 22 23:15:14.189 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(540), op: ExtentFlush { repair_id: ReconciliationId(540), extent_id: 56, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:14.189 INFO [1] client ExtentFlush { repair_id: ReconciliationId(540), extent_id: 56, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:14.189 INFO [2] received reconcile message
Sep 22 23:15:14.189 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(540), op: ExtentFlush { repair_id: ReconciliationId(540), extent_id: 56, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:14.189 INFO [2] client ExtentFlush { repair_id: ReconciliationId(540), extent_id: 56, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:14.189 DEBG 540 Flush extent 56 with f:2 g:2
Sep 22 23:15:14.189 DEBG Flush just extent 56 with f:2 and g:2
Sep 22 23:15:14.189 DEBG [1] It's time to notify for 540
Sep 22 23:15:14.189 INFO Completion from [1] id:540 status:true
Sep 22 23:15:14.189 INFO [541/752] Repair commands completed
Sep 22 23:15:14.189 INFO Pop front: ReconcileIO { id: ReconciliationId(541), op: ExtentClose { repair_id: ReconciliationId(541), extent_id: 56 }, state: ClientData([New, New, New]) }
Sep 22 23:15:14.189 INFO Sent repair work, now wait for resp
Sep 22 23:15:14.189 INFO [0] received reconcile message
Sep 22 23:15:14.189 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(541), op: ExtentClose { repair_id: ReconciliationId(541), extent_id: 56 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:14.189 INFO [0] client ExtentClose { repair_id: ReconciliationId(541), extent_id: 56 }
Sep 22 23:15:14.189 INFO [1] received reconcile message
Sep 22 23:15:14.189 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(541), op: ExtentClose { repair_id: ReconciliationId(541), extent_id: 56 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:14.189 INFO [1] client ExtentClose { repair_id: ReconciliationId(541), extent_id: 56 }
Sep 22 23:15:14.189 INFO [2] received reconcile message
Sep 22 23:15:14.189 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(541), op: ExtentClose { repair_id: ReconciliationId(541), extent_id: 56 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:14.189 INFO [2] client ExtentClose { repair_id: ReconciliationId(541), extent_id: 56 }
Sep 22 23:15:14.189 DEBG 541 Close extent 56
Sep 22 23:15:14.190 DEBG 541 Close extent 56
Sep 22 23:15:14.190 DEBG 541 Close extent 56
Sep 22 23:15:14.190 DEBG [2] It's time to notify for 541
Sep 22 23:15:14.190 INFO Completion from [2] id:541 status:true
Sep 22 23:15:14.190 INFO [542/752] Repair commands completed
Sep 22 23:15:14.190 INFO Pop front: ReconcileIO { id: ReconciliationId(542), op: ExtentRepair { repair_id: ReconciliationId(542), extent_id: 56, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:14.191 INFO Sent repair work, now wait for resp
Sep 22 23:15:14.191 INFO [0] received reconcile message
Sep 22 23:15:14.191 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(542), op: ExtentRepair { repair_id: ReconciliationId(542), extent_id: 56, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:14.191 INFO [0] client ExtentRepair { repair_id: ReconciliationId(542), extent_id: 56, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:14.191 INFO [0] Sending repair request ReconciliationId(542)
Sep 22 23:15:14.191 INFO [1] received reconcile message
Sep 22 23:15:14.191 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(542), op: ExtentRepair { repair_id: ReconciliationId(542), extent_id: 56, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:14.191 INFO [1] client ExtentRepair { repair_id: ReconciliationId(542), extent_id: 56, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:14.191 INFO [1] No action required ReconciliationId(542)
Sep 22 23:15:14.191 INFO [2] received reconcile message
Sep 22 23:15:14.191 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(542), op: ExtentRepair { repair_id: ReconciliationId(542), extent_id: 56, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:14.191 INFO [2] client ExtentRepair { repair_id: ReconciliationId(542), extent_id: 56, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:14.191 INFO [2] No action required ReconciliationId(542)
Sep 22 23:15:14.191 DEBG 542 Repair extent 56 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:14.191 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/038.copy"
Sep 22 23:15:14.254 INFO accepted connection, remote_addr: 127.0.0.1:58337, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.254 TRCE incoming request, uri: /extent/56/files, method: GET, req_id: 8b7ef680-3225-4809-af9a-8f716014fdff, remote_addr: 127.0.0.1:58337, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.254 INFO request completed, latency_us: 214, response_code: 200, uri: /extent/56/files, method: GET, req_id: 8b7ef680-3225-4809-af9a-8f716014fdff, remote_addr: 127.0.0.1:58337, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.254 INFO eid:56 Found repair files: ["038", "038.db"]
Sep 22 23:15:14.255 TRCE incoming request, uri: /newextent/56/data, method: GET, req_id: 0079d7d8-d962-42b5-854a-6f4b6300c886, remote_addr: 127.0.0.1:58337, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.255 INFO request completed, latency_us: 251, response_code: 200, uri: /newextent/56/data, method: GET, req_id: 0079d7d8-d962-42b5-854a-6f4b6300c886, remote_addr: 127.0.0.1:58337, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.260 TRCE incoming request, uri: /newextent/56/db, method: GET, req_id: 49725e55-88b2-4383-a07f-0c525114522b, remote_addr: 127.0.0.1:58337, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.260 INFO request completed, latency_us: 295, response_code: 200, uri: /newextent/56/db, method: GET, req_id: 49725e55-88b2-4383-a07f-0c525114522b, remote_addr: 127.0.0.1:58337, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.262 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/038.copy" to "/tmp/downstairs-vrx8aK6L/00/000/038.replace"
Sep 22 23:15:14.262 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:14.262 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/038.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:14.263 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/038"
Sep 22 23:15:14.263 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/038.db"
Sep 22 23:15:14.263 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:14.263 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/038.replace" to "/tmp/downstairs-vrx8aK6L/00/000/038.completed"
Sep 22 23:15:14.263 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:14.263 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:14.263 DEBG [0] It's time to notify for 542
Sep 22 23:15:14.263 INFO Completion from [0] id:542 status:true
Sep 22 23:15:14.263 INFO [543/752] Repair commands completed
Sep 22 23:15:14.263 INFO Pop front: ReconcileIO { id: ReconciliationId(543), op: ExtentReopen { repair_id: ReconciliationId(543), extent_id: 56 }, state: ClientData([New, New, New]) }
Sep 22 23:15:14.263 INFO Sent repair work, now wait for resp
Sep 22 23:15:14.263 INFO [0] received reconcile message
Sep 22 23:15:14.263 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(543), op: ExtentReopen { repair_id: ReconciliationId(543), extent_id: 56 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:14.263 INFO [0] client ExtentReopen { repair_id: ReconciliationId(543), extent_id: 56 }
Sep 22 23:15:14.263 INFO [1] received reconcile message
Sep 22 23:15:14.263 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(543), op: ExtentReopen { repair_id: ReconciliationId(543), extent_id: 56 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:14.263 INFO [1] client ExtentReopen { repair_id: ReconciliationId(543), extent_id: 56 }
Sep 22 23:15:14.264 INFO [2] received reconcile message
Sep 22 23:15:14.264 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(543), op: ExtentReopen { repair_id: ReconciliationId(543), extent_id: 56 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:14.264 INFO [2] client ExtentReopen { repair_id: ReconciliationId(543), extent_id: 56 }
Sep 22 23:15:14.264 DEBG 543 Reopen extent 56
Sep 22 23:15:14.264 DEBG 543 Reopen extent 56
Sep 22 23:15:14.265 DEBG 543 Reopen extent 56
Sep 22 23:15:14.265 DEBG [2] It's time to notify for 543
Sep 22 23:15:14.265 INFO Completion from [2] id:543 status:true
Sep 22 23:15:14.265 INFO [544/752] Repair commands completed
Sep 22 23:15:14.265 INFO Pop front: ReconcileIO { id: ReconciliationId(544), op: ExtentFlush { repair_id: ReconciliationId(544), extent_id: 23, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:14.265 INFO Sent repair work, now wait for resp
Sep 22 23:15:14.265 INFO [0] received reconcile message
Sep 22 23:15:14.266 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(544), op: ExtentFlush { repair_id: ReconciliationId(544), extent_id: 23, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:14.266 INFO [0] client ExtentFlush { repair_id: ReconciliationId(544), extent_id: 23, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:14.266 INFO [1] received reconcile message
Sep 22 23:15:14.266 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(544), op: ExtentFlush { repair_id: ReconciliationId(544), extent_id: 23, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:14.266 INFO [1] client ExtentFlush { repair_id: ReconciliationId(544), extent_id: 23, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:14.266 INFO [2] received reconcile message
Sep 22 23:15:14.266 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(544), op: ExtentFlush { repair_id: ReconciliationId(544), extent_id: 23, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:14.266 INFO [2] client ExtentFlush { repair_id: ReconciliationId(544), extent_id: 23, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:14.266 DEBG 544 Flush extent 23 with f:2 g:2
Sep 22 23:15:14.266 DEBG Flush just extent 23 with f:2 and g:2
Sep 22 23:15:14.266 DEBG [1] It's time to notify for 544
Sep 22 23:15:14.266 INFO Completion from [1] id:544 status:true
Sep 22 23:15:14.266 INFO [545/752] Repair commands completed
Sep 22 23:15:14.266 INFO Pop front: ReconcileIO { id: ReconciliationId(545), op: ExtentClose { repair_id: ReconciliationId(545), extent_id: 23 }, state: ClientData([New, New, New]) }
Sep 22 23:15:14.266 INFO Sent repair work, now wait for resp
Sep 22 23:15:14.266 INFO [0] received reconcile message
Sep 22 23:15:14.266 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(545), op: ExtentClose { repair_id: ReconciliationId(545), extent_id: 23 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:14.266 INFO [0] client ExtentClose { repair_id: ReconciliationId(545), extent_id: 23 }
Sep 22 23:15:14.266 INFO [1] received reconcile message
Sep 22 23:15:14.266 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(545), op: ExtentClose { repair_id: ReconciliationId(545), extent_id: 23 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:14.266 INFO [1] client ExtentClose { repair_id: ReconciliationId(545), extent_id: 23 }
Sep 22 23:15:14.266 INFO [2] received reconcile message
Sep 22 23:15:14.266 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(545), op: ExtentClose { repair_id: ReconciliationId(545), extent_id: 23 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:14.266 INFO [2] client ExtentClose { repair_id: ReconciliationId(545), extent_id: 23 }
Sep 22 23:15:14.266 DEBG 545 Close extent 23
Sep 22 23:15:14.267 DEBG 545 Close extent 23
Sep 22 23:15:14.267 DEBG 545 Close extent 23
Sep 22 23:15:14.267 DEBG [2] It's time to notify for 545
Sep 22 23:15:14.267 INFO Completion from [2] id:545 status:true
Sep 22 23:15:14.267 INFO [546/752] Repair commands completed
Sep 22 23:15:14.267 INFO Pop front: ReconcileIO { id: ReconciliationId(546), op: ExtentRepair { repair_id: ReconciliationId(546), extent_id: 23, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:14.267 INFO Sent repair work, now wait for resp
Sep 22 23:15:14.267 INFO [0] received reconcile message
Sep 22 23:15:14.267 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(546), op: ExtentRepair { repair_id: ReconciliationId(546), extent_id: 23, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:14.268 INFO [0] client ExtentRepair { repair_id: ReconciliationId(546), extent_id: 23, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:14.268 INFO [0] Sending repair request ReconciliationId(546)
Sep 22 23:15:14.268 INFO [1] received reconcile message
Sep 22 23:15:14.268 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(546), op: ExtentRepair { repair_id: ReconciliationId(546), extent_id: 23, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:14.268 INFO [1] client ExtentRepair { repair_id: ReconciliationId(546), extent_id: 23, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:14.268 INFO [1] No action required ReconciliationId(546)
Sep 22 23:15:14.268 INFO [2] received reconcile message
Sep 22 23:15:14.268 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(546), op: ExtentRepair { repair_id: ReconciliationId(546), extent_id: 23, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:14.268 INFO [2] client ExtentRepair { repair_id: ReconciliationId(546), extent_id: 23, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:14.268 INFO [2] No action required ReconciliationId(546)
Sep 22 23:15:14.268 DEBG 546 Repair extent 23 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:14.268 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/017.copy"
Sep 22 23:15:14.329 INFO accepted connection, remote_addr: 127.0.0.1:61275, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.329 TRCE incoming request, uri: /extent/23/files, method: GET, req_id: f0bfd5b6-e5f3-41ae-bccc-999bf35a13c2, remote_addr: 127.0.0.1:61275, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.329 INFO request completed, latency_us: 186, response_code: 200, uri: /extent/23/files, method: GET, req_id: f0bfd5b6-e5f3-41ae-bccc-999bf35a13c2, remote_addr: 127.0.0.1:61275, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.329 INFO eid:23 Found repair files: ["017", "017.db"]
Sep 22 23:15:14.330 TRCE incoming request, uri: /newextent/23/data, method: GET, req_id: 46bb07aa-63e7-427c-bd77-50e2c2f01fdd, remote_addr: 127.0.0.1:61275, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.330 INFO request completed, latency_us: 328, response_code: 200, uri: /newextent/23/data, method: GET, req_id: 46bb07aa-63e7-427c-bd77-50e2c2f01fdd, remote_addr: 127.0.0.1:61275, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.335 TRCE incoming request, uri: /newextent/23/db, method: GET, req_id: 13d70a99-a549-478b-a2e0-fd4d4afabfaa, remote_addr: 127.0.0.1:61275, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.336 INFO request completed, latency_us: 333, response_code: 200, uri: /newextent/23/db, method: GET, req_id: 13d70a99-a549-478b-a2e0-fd4d4afabfaa, remote_addr: 127.0.0.1:61275, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.337 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/017.copy" to "/tmp/downstairs-vrx8aK6L/00/000/017.replace"
Sep 22 23:15:14.337 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:14.338 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/017.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:14.338 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/017"
Sep 22 23:15:14.338 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/017.db"
Sep 22 23:15:14.338 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:14.338 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/017.replace" to "/tmp/downstairs-vrx8aK6L/00/000/017.completed"
Sep 22 23:15:14.338 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:14.338 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:14.338 DEBG [0] It's time to notify for 546
Sep 22 23:15:14.338 INFO Completion from [0] id:546 status:true
Sep 22 23:15:14.338 INFO [547/752] Repair commands completed
Sep 22 23:15:14.339 INFO Pop front: ReconcileIO { id: ReconciliationId(547), op: ExtentReopen { repair_id: ReconciliationId(547), extent_id: 23 }, state: ClientData([New, New, New]) }
Sep 22 23:15:14.339 INFO Sent repair work, now wait for resp
Sep 22 23:15:14.339 INFO [0] received reconcile message
Sep 22 23:15:14.339 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(547), op: ExtentReopen { repair_id: ReconciliationId(547), extent_id: 23 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:14.339 INFO [0] client ExtentReopen { repair_id: ReconciliationId(547), extent_id: 23 }
Sep 22 23:15:14.339 INFO [1] received reconcile message
Sep 22 23:15:14.339 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(547), op: ExtentReopen { repair_id: ReconciliationId(547), extent_id: 23 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:14.339 INFO [1] client ExtentReopen { repair_id: ReconciliationId(547), extent_id: 23 }
Sep 22 23:15:14.339 INFO [2] received reconcile message
Sep 22 23:15:14.339 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(547), op: ExtentReopen { repair_id: ReconciliationId(547), extent_id: 23 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:14.339 INFO [2] client ExtentReopen { repair_id: ReconciliationId(547), extent_id: 23 }
Sep 22 23:15:14.339 DEBG 547 Reopen extent 23
Sep 22 23:15:14.339 DEBG 547 Reopen extent 23
Sep 22 23:15:14.340 DEBG 547 Reopen extent 23
Sep 22 23:15:14.341 DEBG [2] It's time to notify for 547
Sep 22 23:15:14.341 INFO Completion from [2] id:547 status:true
Sep 22 23:15:14.341 INFO [548/752] Repair commands completed
Sep 22 23:15:14.341 INFO Pop front: ReconcileIO { id: ReconciliationId(548), op: ExtentFlush { repair_id: ReconciliationId(548), extent_id: 10, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:14.341 INFO Sent repair work, now wait for resp
Sep 22 23:15:14.341 INFO [0] received reconcile message
Sep 22 23:15:14.341 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(548), op: ExtentFlush { repair_id: ReconciliationId(548), extent_id: 10, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:14.341 INFO [0] client ExtentFlush { repair_id: ReconciliationId(548), extent_id: 10, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:14.341 INFO [1] received reconcile message
Sep 22 23:15:14.341 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(548), op: ExtentFlush { repair_id: ReconciliationId(548), extent_id: 10, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:14.341 INFO [1] client ExtentFlush { repair_id: ReconciliationId(548), extent_id: 10, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:14.341 INFO [2] received reconcile message
Sep 22 23:15:14.341 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(548), op: ExtentFlush { repair_id: ReconciliationId(548), extent_id: 10, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:14.341 INFO [2] client ExtentFlush { repair_id: ReconciliationId(548), extent_id: 10, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:14.341 DEBG 548 Flush extent 10 with f:2 g:2
Sep 22 23:15:14.341 DEBG Flush just extent 10 with f:2 and g:2
Sep 22 23:15:14.341 DEBG [1] It's time to notify for 548
Sep 22 23:15:14.341 INFO Completion from [1] id:548 status:true
Sep 22 23:15:14.341 INFO [549/752] Repair commands completed
Sep 22 23:15:14.341 INFO Pop front: ReconcileIO { id: ReconciliationId(549), op: ExtentClose { repair_id: ReconciliationId(549), extent_id: 10 }, state: ClientData([New, New, New]) }
Sep 22 23:15:14.341 INFO Sent repair work, now wait for resp
Sep 22 23:15:14.341 INFO [0] received reconcile message
Sep 22 23:15:14.341 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(549), op: ExtentClose { repair_id: ReconciliationId(549), extent_id: 10 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:14.341 INFO [0] client ExtentClose { repair_id: ReconciliationId(549), extent_id: 10 }
Sep 22 23:15:14.341 INFO [1] received reconcile message
Sep 22 23:15:14.341 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(549), op: ExtentClose { repair_id: ReconciliationId(549), extent_id: 10 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:14.341 INFO [1] client ExtentClose { repair_id: ReconciliationId(549), extent_id: 10 }
Sep 22 23:15:14.341 INFO [2] received reconcile message
Sep 22 23:15:14.341 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(549), op: ExtentClose { repair_id: ReconciliationId(549), extent_id: 10 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:14.341 INFO [2] client ExtentClose { repair_id: ReconciliationId(549), extent_id: 10 }
Sep 22 23:15:14.342 DEBG 549 Close extent 10
Sep 22 23:15:14.342 DEBG 549 Close extent 10
Sep 22 23:15:14.342 DEBG 549 Close extent 10
Sep 22 23:15:14.343 DEBG [2] It's time to notify for 549
Sep 22 23:15:14.343 INFO Completion from [2] id:549 status:true
Sep 22 23:15:14.343 INFO [550/752] Repair commands completed
Sep 22 23:15:14.343 INFO Pop front: ReconcileIO { id: ReconciliationId(550), op: ExtentRepair { repair_id: ReconciliationId(550), extent_id: 10, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:14.343 INFO Sent repair work, now wait for resp
Sep 22 23:15:14.343 INFO [0] received reconcile message
Sep 22 23:15:14.343 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(550), op: ExtentRepair { repair_id: ReconciliationId(550), extent_id: 10, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:14.343 INFO [0] client ExtentRepair { repair_id: ReconciliationId(550), extent_id: 10, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:14.343 INFO [0] Sending repair request ReconciliationId(550)
Sep 22 23:15:14.343 INFO [1] received reconcile message
Sep 22 23:15:14.343 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(550), op: ExtentRepair { repair_id: ReconciliationId(550), extent_id: 10, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:14.343 INFO [1] client ExtentRepair { repair_id: ReconciliationId(550), extent_id: 10, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:14.343 INFO [1] No action required ReconciliationId(550)
Sep 22 23:15:14.343 INFO [2] received reconcile message
Sep 22 23:15:14.343 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(550), op: ExtentRepair { repair_id: ReconciliationId(550), extent_id: 10, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:14.343 INFO [2] client ExtentRepair { repair_id: ReconciliationId(550), extent_id: 10, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:14.343 INFO [2] No action required ReconciliationId(550)
Sep 22 23:15:14.343 DEBG 550 Repair extent 10 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:14.343 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/00A.copy"
Sep 22 23:15:14.407 INFO accepted connection, remote_addr: 127.0.0.1:42941, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.407 TRCE incoming request, uri: /extent/10/files, method: GET, req_id: aae45806-4bc9-48f8-a2c1-e2cf3fd80b16, remote_addr: 127.0.0.1:42941, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.408 INFO request completed, latency_us: 199, response_code: 200, uri: /extent/10/files, method: GET, req_id: aae45806-4bc9-48f8-a2c1-e2cf3fd80b16, remote_addr: 127.0.0.1:42941, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.408 INFO eid:10 Found repair files: ["00A", "00A.db"]
Sep 22 23:15:14.408 TRCE incoming request, uri: /newextent/10/data, method: GET, req_id: c6d77998-d667-457b-8ece-8cfe4da06af2, remote_addr: 127.0.0.1:42941, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.408 INFO request completed, latency_us: 303, response_code: 200, uri: /newextent/10/data, method: GET, req_id: c6d77998-d667-457b-8ece-8cfe4da06af2, remote_addr: 127.0.0.1:42941, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.414 TRCE incoming request, uri: /newextent/10/db, method: GET, req_id: 0343c35f-053e-419b-8136-54c5d6584e17, remote_addr: 127.0.0.1:42941, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.414 INFO request completed, latency_us: 293, response_code: 200, uri: /newextent/10/db, method: GET, req_id: 0343c35f-053e-419b-8136-54c5d6584e17, remote_addr: 127.0.0.1:42941, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.415 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/00A.copy" to "/tmp/downstairs-vrx8aK6L/00/000/00A.replace"
Sep 22 23:15:14.415 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:14.416 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/00A.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:14.416 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/00A"
Sep 22 23:15:14.416 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/00A.db"
Sep 22 23:15:14.416 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:14.416 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/00A.replace" to "/tmp/downstairs-vrx8aK6L/00/000/00A.completed"
Sep 22 23:15:14.416 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:14.416 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:14.417 DEBG [0] It's time to notify for 550
Sep 22 23:15:14.417 INFO Completion from [0] id:550 status:true
Sep 22 23:15:14.417 INFO [551/752] Repair commands completed
Sep 22 23:15:14.417 INFO Pop front: ReconcileIO { id: ReconciliationId(551), op: ExtentReopen { repair_id: ReconciliationId(551), extent_id: 10 }, state: ClientData([New, New, New]) }
Sep 22 23:15:14.417 INFO Sent repair work, now wait for resp
Sep 22 23:15:14.417 INFO [0] received reconcile message
Sep 22 23:15:14.417 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(551), op: ExtentReopen { repair_id: ReconciliationId(551), extent_id: 10 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:14.417 INFO [0] client ExtentReopen { repair_id: ReconciliationId(551), extent_id: 10 }
Sep 22 23:15:14.417 INFO [1] received reconcile message
Sep 22 23:15:14.417 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(551), op: ExtentReopen { repair_id: ReconciliationId(551), extent_id: 10 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:14.417 INFO [1] client ExtentReopen { repair_id: ReconciliationId(551), extent_id: 10 }
Sep 22 23:15:14.417 INFO [2] received reconcile message
Sep 22 23:15:14.417 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(551), op: ExtentReopen { repair_id: ReconciliationId(551), extent_id: 10 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:14.417 INFO [2] client ExtentReopen { repair_id: ReconciliationId(551), extent_id: 10 }
Sep 22 23:15:14.417 DEBG 551 Reopen extent 10
Sep 22 23:15:14.418 DEBG 551 Reopen extent 10
Sep 22 23:15:14.418 DEBG 551 Reopen extent 10
Sep 22 23:15:14.419 DEBG [2] It's time to notify for 551
Sep 22 23:15:14.419 INFO Completion from [2] id:551 status:true
Sep 22 23:15:14.419 INFO [552/752] Repair commands completed
Sep 22 23:15:14.419 INFO Pop front: ReconcileIO { id: ReconciliationId(552), op: ExtentFlush { repair_id: ReconciliationId(552), extent_id: 44, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:14.419 INFO Sent repair work, now wait for resp
Sep 22 23:15:14.419 INFO [0] received reconcile message
Sep 22 23:15:14.419 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(552), op: ExtentFlush { repair_id: ReconciliationId(552), extent_id: 44, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:14.419 INFO [0] client ExtentFlush { repair_id: ReconciliationId(552), extent_id: 44, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:14.419 INFO [1] received reconcile message
Sep 22 23:15:14.419 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(552), op: ExtentFlush { repair_id: ReconciliationId(552), extent_id: 44, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:14.419 INFO [1] client ExtentFlush { repair_id: ReconciliationId(552), extent_id: 44, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:14.419 INFO [2] received reconcile message
Sep 22 23:15:14.419 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(552), op: ExtentFlush { repair_id: ReconciliationId(552), extent_id: 44, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:14.419 INFO [2] client ExtentFlush { repair_id: ReconciliationId(552), extent_id: 44, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:14.419 DEBG 552 Flush extent 44 with f:2 g:2
Sep 22 23:15:14.419 DEBG Flush just extent 44 with f:2 and g:2
Sep 22 23:15:14.419 DEBG [1] It's time to notify for 552
Sep 22 23:15:14.420 INFO Completion from [1] id:552 status:true
Sep 22 23:15:14.420 INFO [553/752] Repair commands completed
Sep 22 23:15:14.420 INFO Pop front: ReconcileIO { id: ReconciliationId(553), op: ExtentClose { repair_id: ReconciliationId(553), extent_id: 44 }, state: ClientData([New, New, New]) }
Sep 22 23:15:14.420 INFO Sent repair work, now wait for resp
Sep 22 23:15:14.420 INFO [0] received reconcile message
Sep 22 23:15:14.420 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(553), op: ExtentClose { repair_id: ReconciliationId(553), extent_id: 44 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:14.420 INFO [0] client ExtentClose { repair_id: ReconciliationId(553), extent_id: 44 }
Sep 22 23:15:14.420 INFO [1] received reconcile message
Sep 22 23:15:14.420 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(553), op: ExtentClose { repair_id: ReconciliationId(553), extent_id: 44 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:14.420 INFO [1] client ExtentClose { repair_id: ReconciliationId(553), extent_id: 44 }
Sep 22 23:15:14.420 INFO [2] received reconcile message
Sep 22 23:15:14.420 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(553), op: ExtentClose { repair_id: ReconciliationId(553), extent_id: 44 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:14.420 INFO [2] client ExtentClose { repair_id: ReconciliationId(553), extent_id: 44 }
Sep 22 23:15:14.420 DEBG 553 Close extent 44
Sep 22 23:15:14.420 DEBG 553 Close extent 44
Sep 22 23:15:14.420 DEBG 553 Close extent 44
Sep 22 23:15:14.421 DEBG [2] It's time to notify for 553
Sep 22 23:15:14.421 INFO Completion from [2] id:553 status:true
Sep 22 23:15:14.421 INFO [554/752] Repair commands completed
Sep 22 23:15:14.421 INFO Pop front: ReconcileIO { id: ReconciliationId(554), op: ExtentRepair { repair_id: ReconciliationId(554), extent_id: 44, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:14.421 INFO Sent repair work, now wait for resp
Sep 22 23:15:14.421 INFO [0] received reconcile message
Sep 22 23:15:14.421 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(554), op: ExtentRepair { repair_id: ReconciliationId(554), extent_id: 44, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:14.421 INFO [0] client ExtentRepair { repair_id: ReconciliationId(554), extent_id: 44, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:14.421 INFO [0] Sending repair request ReconciliationId(554)
Sep 22 23:15:14.421 INFO [1] received reconcile message
Sep 22 23:15:14.421 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(554), op: ExtentRepair { repair_id: ReconciliationId(554), extent_id: 44, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:14.421 INFO [1] client ExtentRepair { repair_id: ReconciliationId(554), extent_id: 44, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:14.421 INFO [1] No action required ReconciliationId(554)
Sep 22 23:15:14.421 INFO [2] received reconcile message
Sep 22 23:15:14.421 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(554), op: ExtentRepair { repair_id: ReconciliationId(554), extent_id: 44, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:14.421 INFO [2] client ExtentRepair { repair_id: ReconciliationId(554), extent_id: 44, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:14.421 INFO [2] No action required ReconciliationId(554)
Sep 22 23:15:14.421 DEBG 554 Repair extent 44 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:14.421 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/02C.copy"
Sep 22 23:15:14.484 INFO accepted connection, remote_addr: 127.0.0.1:50605, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.484 TRCE incoming request, uri: /extent/44/files, method: GET, req_id: 16e33043-1d06-4ad0-9699-84e5d47ec2e7, remote_addr: 127.0.0.1:50605, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.484 INFO request completed, latency_us: 206, response_code: 200, uri: /extent/44/files, method: GET, req_id: 16e33043-1d06-4ad0-9699-84e5d47ec2e7, remote_addr: 127.0.0.1:50605, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.484 INFO eid:44 Found repair files: ["02C", "02C.db"]
Sep 22 23:15:14.485 TRCE incoming request, uri: /newextent/44/data, method: GET, req_id: 45eee28d-e6ff-4d69-bde1-3c093aced218, remote_addr: 127.0.0.1:50605, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.485 INFO request completed, latency_us: 256, response_code: 200, uri: /newextent/44/data, method: GET, req_id: 45eee28d-e6ff-4d69-bde1-3c093aced218, remote_addr: 127.0.0.1:50605, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.490 TRCE incoming request, uri: /newextent/44/db, method: GET, req_id: b5a3db77-8da9-46e7-ad80-24a3699d85cc, remote_addr: 127.0.0.1:50605, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.490 INFO request completed, latency_us: 295, response_code: 200, uri: /newextent/44/db, method: GET, req_id: b5a3db77-8da9-46e7-ad80-24a3699d85cc, remote_addr: 127.0.0.1:50605, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.491 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/02C.copy" to "/tmp/downstairs-vrx8aK6L/00/000/02C.replace"
Sep 22 23:15:14.491 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:14.492 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/02C.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:14.492 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/02C"
Sep 22 23:15:14.492 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/02C.db"
Sep 22 23:15:14.492 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:14.492 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/02C.replace" to "/tmp/downstairs-vrx8aK6L/00/000/02C.completed"
Sep 22 23:15:14.492 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:14.493 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:14.493 DEBG [0] It's time to notify for 554
Sep 22 23:15:14.493 INFO Completion from [0] id:554 status:true
Sep 22 23:15:14.493 INFO [555/752] Repair commands completed
Sep 22 23:15:14.493 INFO Pop front: ReconcileIO { id: ReconciliationId(555), op: ExtentReopen { repair_id: ReconciliationId(555), extent_id: 44 }, state: ClientData([New, New, New]) }
Sep 22 23:15:14.493 INFO Sent repair work, now wait for resp
Sep 22 23:15:14.493 INFO [0] received reconcile message
Sep 22 23:15:14.493 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(555), op: ExtentReopen { repair_id: ReconciliationId(555), extent_id: 44 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:14.493 INFO [0] client ExtentReopen { repair_id: ReconciliationId(555), extent_id: 44 }
Sep 22 23:15:14.493 INFO [1] received reconcile message
Sep 22 23:15:14.493 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(555), op: ExtentReopen { repair_id: ReconciliationId(555), extent_id: 44 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:14.493 INFO [1] client ExtentReopen { repair_id: ReconciliationId(555), extent_id: 44 }
Sep 22 23:15:14.493 INFO [2] received reconcile message
Sep 22 23:15:14.493 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(555), op: ExtentReopen { repair_id: ReconciliationId(555), extent_id: 44 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:14.493 INFO [2] client ExtentReopen { repair_id: ReconciliationId(555), extent_id: 44 }
Sep 22 23:15:14.493 DEBG 555 Reopen extent 44
Sep 22 23:15:14.494 DEBG 555 Reopen extent 44
Sep 22 23:15:14.494 DEBG 555 Reopen extent 44
Sep 22 23:15:14.495 DEBG [2] It's time to notify for 555
Sep 22 23:15:14.495 INFO Completion from [2] id:555 status:true
Sep 22 23:15:14.495 INFO [556/752] Repair commands completed
Sep 22 23:15:14.495 INFO Pop front: ReconcileIO { id: ReconciliationId(556), op: ExtentFlush { repair_id: ReconciliationId(556), extent_id: 59, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:14.495 INFO Sent repair work, now wait for resp
Sep 22 23:15:14.495 INFO [0] received reconcile message
Sep 22 23:15:14.495 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(556), op: ExtentFlush { repair_id: ReconciliationId(556), extent_id: 59, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:14.495 INFO [0] client ExtentFlush { repair_id: ReconciliationId(556), extent_id: 59, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:14.495 INFO [1] received reconcile message
Sep 22 23:15:14.495 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(556), op: ExtentFlush { repair_id: ReconciliationId(556), extent_id: 59, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:14.495 INFO [1] client ExtentFlush { repair_id: ReconciliationId(556), extent_id: 59, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:14.495 INFO [2] received reconcile message
Sep 22 23:15:14.495 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(556), op: ExtentFlush { repair_id: ReconciliationId(556), extent_id: 59, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:14.495 INFO [2] client ExtentFlush { repair_id: ReconciliationId(556), extent_id: 59, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:14.495 DEBG 556 Flush extent 59 with f:2 g:2
Sep 22 23:15:14.495 DEBG Flush just extent 59 with f:2 and g:2
Sep 22 23:15:14.496 DEBG [1] It's time to notify for 556
Sep 22 23:15:14.496 INFO Completion from [1] id:556 status:true
Sep 22 23:15:14.496 INFO [557/752] Repair commands completed
Sep 22 23:15:14.496 INFO Pop front: ReconcileIO { id: ReconciliationId(557), op: ExtentClose { repair_id: ReconciliationId(557), extent_id: 59 }, state: ClientData([New, New, New]) }
Sep 22 23:15:14.496 INFO Sent repair work, now wait for resp
Sep 22 23:15:14.496 INFO [0] received reconcile message
Sep 22 23:15:14.496 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(557), op: ExtentClose { repair_id: ReconciliationId(557), extent_id: 59 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:14.496 INFO [0] client ExtentClose { repair_id: ReconciliationId(557), extent_id: 59 }
Sep 22 23:15:14.496 INFO [1] received reconcile message
Sep 22 23:15:14.496 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(557), op: ExtentClose { repair_id: ReconciliationId(557), extent_id: 59 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:14.496 INFO [1] client ExtentClose { repair_id: ReconciliationId(557), extent_id: 59 }
Sep 22 23:15:14.496 INFO [2] received reconcile message
Sep 22 23:15:14.496 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(557), op: ExtentClose { repair_id: ReconciliationId(557), extent_id: 59 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:14.496 INFO [2] client ExtentClose { repair_id: ReconciliationId(557), extent_id: 59 }
Sep 22 23:15:14.496 DEBG 557 Close extent 59
Sep 22 23:15:14.496 DEBG 557 Close extent 59
Sep 22 23:15:14.497 DEBG 557 Close extent 59
Sep 22 23:15:14.497 DEBG [2] It's time to notify for 557
Sep 22 23:15:14.497 INFO Completion from [2] id:557 status:true
Sep 22 23:15:14.497 INFO [558/752] Repair commands completed
Sep 22 23:15:14.497 INFO Pop front: ReconcileIO { id: ReconciliationId(558), op: ExtentRepair { repair_id: ReconciliationId(558), extent_id: 59, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:14.497 INFO Sent repair work, now wait for resp
Sep 22 23:15:14.497 INFO [0] received reconcile message
Sep 22 23:15:14.497 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(558), op: ExtentRepair { repair_id: ReconciliationId(558), extent_id: 59, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:14.497 INFO [0] client ExtentRepair { repair_id: ReconciliationId(558), extent_id: 59, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:14.497 INFO [0] Sending repair request ReconciliationId(558)
Sep 22 23:15:14.497 INFO [1] received reconcile message
Sep 22 23:15:14.497 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(558), op: ExtentRepair { repair_id: ReconciliationId(558), extent_id: 59, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:14.497 INFO [1] client ExtentRepair { repair_id: ReconciliationId(558), extent_id: 59, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:14.497 INFO [1] No action required ReconciliationId(558)
Sep 22 23:15:14.497 INFO [2] received reconcile message
Sep 22 23:15:14.497 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(558), op: ExtentRepair { repair_id: ReconciliationId(558), extent_id: 59, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:14.497 INFO [2] client ExtentRepair { repair_id: ReconciliationId(558), extent_id: 59, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:14.497 INFO [2] No action required ReconciliationId(558)
Sep 22 23:15:14.497 DEBG 558 Repair extent 59 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:14.498 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/03B.copy"
Sep 22 23:15:14.562 INFO accepted connection, remote_addr: 127.0.0.1:60441, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.563 TRCE incoming request, uri: /extent/59/files, method: GET, req_id: 397eae9c-467b-46b2-9403-6cd8e1698252, remote_addr: 127.0.0.1:60441, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.563 INFO request completed, latency_us: 193, response_code: 200, uri: /extent/59/files, method: GET, req_id: 397eae9c-467b-46b2-9403-6cd8e1698252, remote_addr: 127.0.0.1:60441, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.563 INFO eid:59 Found repair files: ["03B", "03B.db"]
Sep 22 23:15:14.563 TRCE incoming request, uri: /newextent/59/data, method: GET, req_id: ed35b770-5183-4d65-a620-0cb4724c0d49, remote_addr: 127.0.0.1:60441, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.564 INFO request completed, latency_us: 301, response_code: 200, uri: /newextent/59/data, method: GET, req_id: ed35b770-5183-4d65-a620-0cb4724c0d49, remote_addr: 127.0.0.1:60441, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.569 TRCE incoming request, uri: /newextent/59/db, method: GET, req_id: e3714a00-51f3-4de4-9bbf-9b4fa3f91c63, remote_addr: 127.0.0.1:60441, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.569 INFO request completed, latency_us: 307, response_code: 200, uri: /newextent/59/db, method: GET, req_id: e3714a00-51f3-4de4-9bbf-9b4fa3f91c63, remote_addr: 127.0.0.1:60441, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.570 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/03B.copy" to "/tmp/downstairs-vrx8aK6L/00/000/03B.replace"
Sep 22 23:15:14.570 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:14.571 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/03B.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:14.571 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/03B"
Sep 22 23:15:14.571 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/03B.db"
Sep 22 23:15:14.571 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:14.571 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/03B.replace" to "/tmp/downstairs-vrx8aK6L/00/000/03B.completed"
Sep 22 23:15:14.571 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:14.572 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:14.572 DEBG [0] It's time to notify for 558
Sep 22 23:15:14.572 INFO Completion from [0] id:558 status:true
Sep 22 23:15:14.572 INFO [559/752] Repair commands completed
Sep 22 23:15:14.572 INFO Pop front: ReconcileIO { id: ReconciliationId(559), op: ExtentReopen { repair_id: ReconciliationId(559), extent_id: 59 }, state: ClientData([New, New, New]) }
Sep 22 23:15:14.572 INFO Sent repair work, now wait for resp
Sep 22 23:15:14.572 INFO [0] received reconcile message
Sep 22 23:15:14.572 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(559), op: ExtentReopen { repair_id: ReconciliationId(559), extent_id: 59 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:14.572 INFO [0] client ExtentReopen { repair_id: ReconciliationId(559), extent_id: 59 }
Sep 22 23:15:14.572 INFO [1] received reconcile message
Sep 22 23:15:14.572 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(559), op: ExtentReopen { repair_id: ReconciliationId(559), extent_id: 59 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:14.572 INFO [1] client ExtentReopen { repair_id: ReconciliationId(559), extent_id: 59 }
Sep 22 23:15:14.572 INFO [2] received reconcile message
Sep 22 23:15:14.572 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(559), op: ExtentReopen { repair_id: ReconciliationId(559), extent_id: 59 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:14.572 INFO [2] client ExtentReopen { repair_id: ReconciliationId(559), extent_id: 59 }
Sep 22 23:15:14.572 DEBG 559 Reopen extent 59
Sep 22 23:15:14.573 DEBG 559 Reopen extent 59
Sep 22 23:15:14.573 DEBG 559 Reopen extent 59
Sep 22 23:15:14.574 DEBG [2] It's time to notify for 559
Sep 22 23:15:14.574 INFO Completion from [2] id:559 status:true
Sep 22 23:15:14.574 INFO [560/752] Repair commands completed
Sep 22 23:15:14.574 INFO Pop front: ReconcileIO { id: ReconciliationId(560), op: ExtentFlush { repair_id: ReconciliationId(560), extent_id: 154, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:14.574 INFO Sent repair work, now wait for resp
Sep 22 23:15:14.574 INFO [0] received reconcile message
Sep 22 23:15:14.574 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(560), op: ExtentFlush { repair_id: ReconciliationId(560), extent_id: 154, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:14.574 INFO [0] client ExtentFlush { repair_id: ReconciliationId(560), extent_id: 154, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:14.574 INFO [1] received reconcile message
Sep 22 23:15:14.574 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(560), op: ExtentFlush { repair_id: ReconciliationId(560), extent_id: 154, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:14.574 INFO [1] client ExtentFlush { repair_id: ReconciliationId(560), extent_id: 154, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:14.574 INFO [2] received reconcile message
Sep 22 23:15:14.574 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(560), op: ExtentFlush { repair_id: ReconciliationId(560), extent_id: 154, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:14.574 INFO [2] client ExtentFlush { repair_id: ReconciliationId(560), extent_id: 154, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:14.574 DEBG 560 Flush extent 154 with f:2 g:2
Sep 22 23:15:14.574 DEBG Flush just extent 154 with f:2 and g:2
Sep 22 23:15:14.574 DEBG [1] It's time to notify for 560
Sep 22 23:15:14.575 INFO Completion from [1] id:560 status:true
Sep 22 23:15:14.575 INFO [561/752] Repair commands completed
Sep 22 23:15:14.575 INFO Pop front: ReconcileIO { id: ReconciliationId(561), op: ExtentClose { repair_id: ReconciliationId(561), extent_id: 154 }, state: ClientData([New, New, New]) }
Sep 22 23:15:14.575 INFO Sent repair work, now wait for resp
Sep 22 23:15:14.575 INFO [0] received reconcile message
Sep 22 23:15:14.575 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(561), op: ExtentClose { repair_id: ReconciliationId(561), extent_id: 154 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:14.575 INFO [0] client ExtentClose { repair_id: ReconciliationId(561), extent_id: 154 }
Sep 22 23:15:14.575 INFO [1] received reconcile message
Sep 22 23:15:14.575 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(561), op: ExtentClose { repair_id: ReconciliationId(561), extent_id: 154 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:14.575 INFO [1] client ExtentClose { repair_id: ReconciliationId(561), extent_id: 154 }
Sep 22 23:15:14.575 INFO [2] received reconcile message
Sep 22 23:15:14.575 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(561), op: ExtentClose { repair_id: ReconciliationId(561), extent_id: 154 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:14.575 INFO [2] client ExtentClose { repair_id: ReconciliationId(561), extent_id: 154 }
Sep 22 23:15:14.575 DEBG 561 Close extent 154
Sep 22 23:15:14.575 DEBG 561 Close extent 154
Sep 22 23:15:14.575 DEBG 561 Close extent 154
Sep 22 23:15:14.576 DEBG [2] It's time to notify for 561
Sep 22 23:15:14.576 INFO Completion from [2] id:561 status:true
Sep 22 23:15:14.576 INFO [562/752] Repair commands completed
Sep 22 23:15:14.576 INFO Pop front: ReconcileIO { id: ReconciliationId(562), op: ExtentRepair { repair_id: ReconciliationId(562), extent_id: 154, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:14.576 INFO Sent repair work, now wait for resp
Sep 22 23:15:14.576 INFO [0] received reconcile message
Sep 22 23:15:14.576 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(562), op: ExtentRepair { repair_id: ReconciliationId(562), extent_id: 154, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:14.576 INFO [0] client ExtentRepair { repair_id: ReconciliationId(562), extent_id: 154, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:14.576 INFO [0] Sending repair request ReconciliationId(562)
Sep 22 23:15:14.576 INFO [1] received reconcile message
Sep 22 23:15:14.576 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(562), op: ExtentRepair { repair_id: ReconciliationId(562), extent_id: 154, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:14.576 INFO [1] client ExtentRepair { repair_id: ReconciliationId(562), extent_id: 154, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:14.576 INFO [1] No action required ReconciliationId(562)
Sep 22 23:15:14.576 INFO [2] received reconcile message
Sep 22 23:15:14.576 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(562), op: ExtentRepair { repair_id: ReconciliationId(562), extent_id: 154, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:14.576 INFO [2] client ExtentRepair { repair_id: ReconciliationId(562), extent_id: 154, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:14.576 INFO [2] No action required ReconciliationId(562)
Sep 22 23:15:14.576 DEBG 562 Repair extent 154 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:14.576 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/09A.copy"
Sep 22 23:15:14.641 INFO accepted connection, remote_addr: 127.0.0.1:35750, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.641 TRCE incoming request, uri: /extent/154/files, method: GET, req_id: 5d637d86-796a-4103-b055-fdd5a151823a, remote_addr: 127.0.0.1:35750, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.641 INFO request completed, latency_us: 210, response_code: 200, uri: /extent/154/files, method: GET, req_id: 5d637d86-796a-4103-b055-fdd5a151823a, remote_addr: 127.0.0.1:35750, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.641 INFO eid:154 Found repair files: ["09A", "09A.db"]
Sep 22 23:15:14.642 TRCE incoming request, uri: /newextent/154/data, method: GET, req_id: 5fa35195-9700-48d6-a03e-5b0fc9bcbdc4, remote_addr: 127.0.0.1:35750, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.642 INFO request completed, latency_us: 253, response_code: 200, uri: /newextent/154/data, method: GET, req_id: 5fa35195-9700-48d6-a03e-5b0fc9bcbdc4, remote_addr: 127.0.0.1:35750, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.647 TRCE incoming request, uri: /newextent/154/db, method: GET, req_id: 8b970be9-6d7a-4474-9868-0a00299e5940, remote_addr: 127.0.0.1:35750, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.647 INFO request completed, latency_us: 297, response_code: 200, uri: /newextent/154/db, method: GET, req_id: 8b970be9-6d7a-4474-9868-0a00299e5940, remote_addr: 127.0.0.1:35750, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.649 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/09A.copy" to "/tmp/downstairs-vrx8aK6L/00/000/09A.replace"
Sep 22 23:15:14.649 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:14.649 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/09A.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:14.650 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/09A"
Sep 22 23:15:14.650 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/09A.db"
Sep 22 23:15:14.650 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:14.650 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/09A.replace" to "/tmp/downstairs-vrx8aK6L/00/000/09A.completed"
Sep 22 23:15:14.650 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:14.650 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:14.650 DEBG [0] It's time to notify for 562
Sep 22 23:15:14.650 INFO Completion from [0] id:562 status:true
Sep 22 23:15:14.650 INFO [563/752] Repair commands completed
Sep 22 23:15:14.650 INFO Pop front: ReconcileIO { id: ReconciliationId(563), op: ExtentReopen { repair_id: ReconciliationId(563), extent_id: 154 }, state: ClientData([New, New, New]) }
Sep 22 23:15:14.650 INFO Sent repair work, now wait for resp
Sep 22 23:15:14.650 INFO [0] received reconcile message
Sep 22 23:15:14.650 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(563), op: ExtentReopen { repair_id: ReconciliationId(563), extent_id: 154 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:14.650 INFO [0] client ExtentReopen { repair_id: ReconciliationId(563), extent_id: 154 }
Sep 22 23:15:14.650 INFO [1] received reconcile message
Sep 22 23:15:14.650 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(563), op: ExtentReopen { repair_id: ReconciliationId(563), extent_id: 154 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:14.650 INFO [1] client ExtentReopen { repair_id: ReconciliationId(563), extent_id: 154 }
Sep 22 23:15:14.650 INFO [2] received reconcile message
Sep 22 23:15:14.650 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(563), op: ExtentReopen { repair_id: ReconciliationId(563), extent_id: 154 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:14.650 INFO [2] client ExtentReopen { repair_id: ReconciliationId(563), extent_id: 154 }
Sep 22 23:15:14.651 DEBG 563 Reopen extent 154
Sep 22 23:15:14.651 DEBG 563 Reopen extent 154
Sep 22 23:15:14.652 DEBG 563 Reopen extent 154
Sep 22 23:15:14.652 DEBG [2] It's time to notify for 563
Sep 22 23:15:14.652 INFO Completion from [2] id:563 status:true
Sep 22 23:15:14.652 INFO [564/752] Repair commands completed
Sep 22 23:15:14.652 INFO Pop front: ReconcileIO { id: ReconciliationId(564), op: ExtentFlush { repair_id: ReconciliationId(564), extent_id: 159, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:14.652 INFO Sent repair work, now wait for resp
Sep 22 23:15:14.652 INFO [0] received reconcile message
Sep 22 23:15:14.652 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(564), op: ExtentFlush { repair_id: ReconciliationId(564), extent_id: 159, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:14.652 INFO [0] client ExtentFlush { repair_id: ReconciliationId(564), extent_id: 159, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:14.652 INFO [1] received reconcile message
Sep 22 23:15:14.653 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(564), op: ExtentFlush { repair_id: ReconciliationId(564), extent_id: 159, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:14.653 INFO [1] client ExtentFlush { repair_id: ReconciliationId(564), extent_id: 159, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:14.653 INFO [2] received reconcile message
Sep 22 23:15:14.653 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(564), op: ExtentFlush { repair_id: ReconciliationId(564), extent_id: 159, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:14.653 INFO [2] client ExtentFlush { repair_id: ReconciliationId(564), extent_id: 159, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:14.653 DEBG 564 Flush extent 159 with f:2 g:2
Sep 22 23:15:14.653 DEBG Flush just extent 159 with f:2 and g:2
Sep 22 23:15:14.653 DEBG [1] It's time to notify for 564
Sep 22 23:15:14.653 INFO Completion from [1] id:564 status:true
Sep 22 23:15:14.653 INFO [565/752] Repair commands completed
Sep 22 23:15:14.653 INFO Pop front: ReconcileIO { id: ReconciliationId(565), op: ExtentClose { repair_id: ReconciliationId(565), extent_id: 159 }, state: ClientData([New, New, New]) }
Sep 22 23:15:14.653 INFO Sent repair work, now wait for resp
Sep 22 23:15:14.653 INFO [0] received reconcile message
Sep 22 23:15:14.653 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(565), op: ExtentClose { repair_id: ReconciliationId(565), extent_id: 159 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:14.653 INFO [0] client ExtentClose { repair_id: ReconciliationId(565), extent_id: 159 }
Sep 22 23:15:14.653 INFO [1] received reconcile message
Sep 22 23:15:14.653 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(565), op: ExtentClose { repair_id: ReconciliationId(565), extent_id: 159 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:14.653 INFO [1] client ExtentClose { repair_id: ReconciliationId(565), extent_id: 159 }
Sep 22 23:15:14.653 INFO [2] received reconcile message
Sep 22 23:15:14.653 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(565), op: ExtentClose { repair_id: ReconciliationId(565), extent_id: 159 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:14.653 INFO [2] client ExtentClose { repair_id: ReconciliationId(565), extent_id: 159 }
Sep 22 23:15:14.653 DEBG 565 Close extent 159
Sep 22 23:15:14.654 DEBG 565 Close extent 159
Sep 22 23:15:14.654 DEBG 565 Close extent 159
Sep 22 23:15:14.654 DEBG [2] It's time to notify for 565
Sep 22 23:15:14.654 INFO Completion from [2] id:565 status:true
Sep 22 23:15:14.654 INFO [566/752] Repair commands completed
Sep 22 23:15:14.654 INFO Pop front: ReconcileIO { id: ReconciliationId(566), op: ExtentRepair { repair_id: ReconciliationId(566), extent_id: 159, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:14.654 INFO Sent repair work, now wait for resp
Sep 22 23:15:14.654 INFO [0] received reconcile message
Sep 22 23:15:14.654 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(566), op: ExtentRepair { repair_id: ReconciliationId(566), extent_id: 159, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:14.654 INFO [0] client ExtentRepair { repair_id: ReconciliationId(566), extent_id: 159, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:14.654 INFO [0] Sending repair request ReconciliationId(566)
Sep 22 23:15:14.654 INFO [1] received reconcile message
Sep 22 23:15:14.654 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(566), op: ExtentRepair { repair_id: ReconciliationId(566), extent_id: 159, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:14.655 INFO [1] client ExtentRepair { repair_id: ReconciliationId(566), extent_id: 159, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:14.655 INFO [1] No action required ReconciliationId(566)
Sep 22 23:15:14.655 INFO [2] received reconcile message
Sep 22 23:15:14.655 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(566), op: ExtentRepair { repair_id: ReconciliationId(566), extent_id: 159, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:14.655 INFO [2] client ExtentRepair { repair_id: ReconciliationId(566), extent_id: 159, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:14.655 INFO [2] No action required ReconciliationId(566)
Sep 22 23:15:14.655 DEBG 566 Repair extent 159 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:14.655 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/09F.copy"
Sep 22 23:15:14.717 INFO accepted connection, remote_addr: 127.0.0.1:56877, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.717 TRCE incoming request, uri: /extent/159/files, method: GET, req_id: 5ea05213-b42a-4633-8ab3-c1d96e620c84, remote_addr: 127.0.0.1:56877, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.717 INFO request completed, latency_us: 197, response_code: 200, uri: /extent/159/files, method: GET, req_id: 5ea05213-b42a-4633-8ab3-c1d96e620c84, remote_addr: 127.0.0.1:56877, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.718 INFO eid:159 Found repair files: ["09F", "09F.db"]
Sep 22 23:15:14.718 TRCE incoming request, uri: /newextent/159/data, method: GET, req_id: 23e998fb-99a1-448c-8c04-2d6488a752c0, remote_addr: 127.0.0.1:56877, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.718 INFO request completed, latency_us: 314, response_code: 200, uri: /newextent/159/data, method: GET, req_id: 23e998fb-99a1-448c-8c04-2d6488a752c0, remote_addr: 127.0.0.1:56877, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.723 TRCE incoming request, uri: /newextent/159/db, method: GET, req_id: addef73c-1b6c-4b62-a7df-b01183742f63, remote_addr: 127.0.0.1:56877, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.724 INFO request completed, latency_us: 295, response_code: 200, uri: /newextent/159/db, method: GET, req_id: addef73c-1b6c-4b62-a7df-b01183742f63, remote_addr: 127.0.0.1:56877, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.725 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/09F.copy" to "/tmp/downstairs-vrx8aK6L/00/000/09F.replace"
Sep 22 23:15:14.725 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:14.725 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/09F.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:14.726 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/09F"
Sep 22 23:15:14.726 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/09F.db"
Sep 22 23:15:14.726 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:14.726 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/09F.replace" to "/tmp/downstairs-vrx8aK6L/00/000/09F.completed"
Sep 22 23:15:14.726 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:14.726 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:14.726 DEBG [0] It's time to notify for 566
Sep 22 23:15:14.726 INFO Completion from [0] id:566 status:true
Sep 22 23:15:14.726 INFO [567/752] Repair commands completed
Sep 22 23:15:14.726 INFO Pop front: ReconcileIO { id: ReconciliationId(567), op: ExtentReopen { repair_id: ReconciliationId(567), extent_id: 159 }, state: ClientData([New, New, New]) }
Sep 22 23:15:14.726 INFO Sent repair work, now wait for resp
Sep 22 23:15:14.726 INFO [0] received reconcile message
Sep 22 23:15:14.726 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(567), op: ExtentReopen { repair_id: ReconciliationId(567), extent_id: 159 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:14.726 INFO [0] client ExtentReopen { repair_id: ReconciliationId(567), extent_id: 159 }
Sep 22 23:15:14.726 INFO [1] received reconcile message
Sep 22 23:15:14.726 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(567), op: ExtentReopen { repair_id: ReconciliationId(567), extent_id: 159 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:14.726 INFO [1] client ExtentReopen { repair_id: ReconciliationId(567), extent_id: 159 }
Sep 22 23:15:14.727 INFO [2] received reconcile message
Sep 22 23:15:14.727 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(567), op: ExtentReopen { repair_id: ReconciliationId(567), extent_id: 159 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:14.727 INFO [2] client ExtentReopen { repair_id: ReconciliationId(567), extent_id: 159 }
Sep 22 23:15:14.727 DEBG 567 Reopen extent 159
Sep 22 23:15:14.727 DEBG 567 Reopen extent 159
Sep 22 23:15:14.728 DEBG 567 Reopen extent 159
Sep 22 23:15:14.728 DEBG [2] It's time to notify for 567
Sep 22 23:15:14.728 INFO Completion from [2] id:567 status:true
Sep 22 23:15:14.728 INFO [568/752] Repair commands completed
Sep 22 23:15:14.728 INFO Pop front: ReconcileIO { id: ReconciliationId(568), op: ExtentFlush { repair_id: ReconciliationId(568), extent_id: 31, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:14.728 INFO Sent repair work, now wait for resp
Sep 22 23:15:14.728 INFO [0] received reconcile message
Sep 22 23:15:14.728 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(568), op: ExtentFlush { repair_id: ReconciliationId(568), extent_id: 31, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:14.728 INFO [0] client ExtentFlush { repair_id: ReconciliationId(568), extent_id: 31, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:14.729 INFO [1] received reconcile message
Sep 22 23:15:14.729 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(568), op: ExtentFlush { repair_id: ReconciliationId(568), extent_id: 31, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:14.729 INFO [1] client ExtentFlush { repair_id: ReconciliationId(568), extent_id: 31, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:14.729 INFO [2] received reconcile message
Sep 22 23:15:14.729 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(568), op: ExtentFlush { repair_id: ReconciliationId(568), extent_id: 31, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:14.729 INFO [2] client ExtentFlush { repair_id: ReconciliationId(568), extent_id: 31, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:14.729 DEBG 568 Flush extent 31 with f:2 g:2
Sep 22 23:15:14.729 DEBG Flush just extent 31 with f:2 and g:2
Sep 22 23:15:14.729 DEBG [1] It's time to notify for 568
Sep 22 23:15:14.729 INFO Completion from [1] id:568 status:true
Sep 22 23:15:14.729 INFO [569/752] Repair commands completed
Sep 22 23:15:14.729 INFO Pop front: ReconcileIO { id: ReconciliationId(569), op: ExtentClose { repair_id: ReconciliationId(569), extent_id: 31 }, state: ClientData([New, New, New]) }
Sep 22 23:15:14.729 INFO Sent repair work, now wait for resp
Sep 22 23:15:14.729 INFO [0] received reconcile message
Sep 22 23:15:14.729 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(569), op: ExtentClose { repair_id: ReconciliationId(569), extent_id: 31 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:14.729 INFO [0] client ExtentClose { repair_id: ReconciliationId(569), extent_id: 31 }
Sep 22 23:15:14.729 INFO [1] received reconcile message
Sep 22 23:15:14.729 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(569), op: ExtentClose { repair_id: ReconciliationId(569), extent_id: 31 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:14.729 INFO [1] client ExtentClose { repair_id: ReconciliationId(569), extent_id: 31 }
Sep 22 23:15:14.729 INFO [2] received reconcile message
Sep 22 23:15:14.729 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(569), op: ExtentClose { repair_id: ReconciliationId(569), extent_id: 31 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:14.729 INFO [2] client ExtentClose { repair_id: ReconciliationId(569), extent_id: 31 }
Sep 22 23:15:14.729 DEBG 569 Close extent 31
Sep 22 23:15:14.730 DEBG 569 Close extent 31
Sep 22 23:15:14.730 DEBG 569 Close extent 31
Sep 22 23:15:14.730 DEBG [2] It's time to notify for 569
Sep 22 23:15:14.730 INFO Completion from [2] id:569 status:true
Sep 22 23:15:14.730 INFO [570/752] Repair commands completed
Sep 22 23:15:14.730 INFO Pop front: ReconcileIO { id: ReconciliationId(570), op: ExtentRepair { repair_id: ReconciliationId(570), extent_id: 31, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:14.730 INFO Sent repair work, now wait for resp
Sep 22 23:15:14.730 INFO [0] received reconcile message
Sep 22 23:15:14.730 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(570), op: ExtentRepair { repair_id: ReconciliationId(570), extent_id: 31, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:14.730 INFO [0] client ExtentRepair { repair_id: ReconciliationId(570), extent_id: 31, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:14.730 INFO [0] Sending repair request ReconciliationId(570)
Sep 22 23:15:14.731 INFO [1] received reconcile message
Sep 22 23:15:14.731 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(570), op: ExtentRepair { repair_id: ReconciliationId(570), extent_id: 31, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:14.731 INFO [1] client ExtentRepair { repair_id: ReconciliationId(570), extent_id: 31, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:14.731 INFO [1] No action required ReconciliationId(570)
Sep 22 23:15:14.731 INFO [2] received reconcile message
Sep 22 23:15:14.731 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(570), op: ExtentRepair { repair_id: ReconciliationId(570), extent_id: 31, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:14.731 INFO [2] client ExtentRepair { repair_id: ReconciliationId(570), extent_id: 31, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:14.731 INFO [2] No action required ReconciliationId(570)
Sep 22 23:15:14.731 DEBG 570 Repair extent 31 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:14.731 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/01F.copy"
Sep 22 23:15:14.794 INFO accepted connection, remote_addr: 127.0.0.1:56105, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.794 TRCE incoming request, uri: /extent/31/files, method: GET, req_id: d56d3b6d-a0ed-45b0-aca4-a556ee13558e, remote_addr: 127.0.0.1:56105, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.794 INFO request completed, latency_us: 194, response_code: 200, uri: /extent/31/files, method: GET, req_id: d56d3b6d-a0ed-45b0-aca4-a556ee13558e, remote_addr: 127.0.0.1:56105, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.794 INFO eid:31 Found repair files: ["01F", "01F.db"]
Sep 22 23:15:14.795 TRCE incoming request, uri: /newextent/31/data, method: GET, req_id: 8e798a32-233e-46fe-a1e8-54c5f216fe80, remote_addr: 127.0.0.1:56105, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.795 INFO request completed, latency_us: 252, response_code: 200, uri: /newextent/31/data, method: GET, req_id: 8e798a32-233e-46fe-a1e8-54c5f216fe80, remote_addr: 127.0.0.1:56105, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.800 TRCE incoming request, uri: /newextent/31/db, method: GET, req_id: 479c5ec0-d964-4894-8ca0-a2a3e6064c2c, remote_addr: 127.0.0.1:56105, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.800 INFO request completed, latency_us: 294, response_code: 200, uri: /newextent/31/db, method: GET, req_id: 479c5ec0-d964-4894-8ca0-a2a3e6064c2c, remote_addr: 127.0.0.1:56105, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.801 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/01F.copy" to "/tmp/downstairs-vrx8aK6L/00/000/01F.replace"
Sep 22 23:15:14.801 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:14.802 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/01F.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:14.802 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/01F"
Sep 22 23:15:14.802 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/01F.db"
Sep 22 23:15:14.802 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:14.802 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/01F.replace" to "/tmp/downstairs-vrx8aK6L/00/000/01F.completed"
Sep 22 23:15:14.802 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:14.803 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:14.803 DEBG [0] It's time to notify for 570
Sep 22 23:15:14.803 INFO Completion from [0] id:570 status:true
Sep 22 23:15:14.803 INFO [571/752] Repair commands completed
Sep 22 23:15:14.803 INFO Pop front: ReconcileIO { id: ReconciliationId(571), op: ExtentReopen { repair_id: ReconciliationId(571), extent_id: 31 }, state: ClientData([New, New, New]) }
Sep 22 23:15:14.803 INFO Sent repair work, now wait for resp
Sep 22 23:15:14.803 INFO [0] received reconcile message
Sep 22 23:15:14.803 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(571), op: ExtentReopen { repair_id: ReconciliationId(571), extent_id: 31 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:14.803 INFO [0] client ExtentReopen { repair_id: ReconciliationId(571), extent_id: 31 }
Sep 22 23:15:14.803 INFO [1] received reconcile message
Sep 22 23:15:14.803 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(571), op: ExtentReopen { repair_id: ReconciliationId(571), extent_id: 31 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:14.803 INFO [1] client ExtentReopen { repair_id: ReconciliationId(571), extent_id: 31 }
Sep 22 23:15:14.803 INFO [2] received reconcile message
Sep 22 23:15:14.803 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(571), op: ExtentReopen { repair_id: ReconciliationId(571), extent_id: 31 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:14.803 INFO [2] client ExtentReopen { repair_id: ReconciliationId(571), extent_id: 31 }
Sep 22 23:15:14.803 DEBG 571 Reopen extent 31
Sep 22 23:15:14.804 DEBG 571 Reopen extent 31
Sep 22 23:15:14.804 DEBG 571 Reopen extent 31
Sep 22 23:15:14.805 DEBG [2] It's time to notify for 571
Sep 22 23:15:14.805 INFO Completion from [2] id:571 status:true
Sep 22 23:15:14.805 INFO [572/752] Repair commands completed
Sep 22 23:15:14.805 INFO Pop front: ReconcileIO { id: ReconciliationId(572), op: ExtentFlush { repair_id: ReconciliationId(572), extent_id: 87, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:14.805 INFO Sent repair work, now wait for resp
Sep 22 23:15:14.805 INFO [0] received reconcile message
Sep 22 23:15:14.805 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(572), op: ExtentFlush { repair_id: ReconciliationId(572), extent_id: 87, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:14.805 INFO [0] client ExtentFlush { repair_id: ReconciliationId(572), extent_id: 87, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:14.805 INFO [1] received reconcile message
Sep 22 23:15:14.805 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(572), op: ExtentFlush { repair_id: ReconciliationId(572), extent_id: 87, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:14.805 INFO [1] client ExtentFlush { repair_id: ReconciliationId(572), extent_id: 87, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:14.805 INFO [2] received reconcile message
Sep 22 23:15:14.805 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(572), op: ExtentFlush { repair_id: ReconciliationId(572), extent_id: 87, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:14.805 INFO [2] client ExtentFlush { repair_id: ReconciliationId(572), extent_id: 87, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:14.805 DEBG 572 Flush extent 87 with f:2 g:2
Sep 22 23:15:14.805 DEBG Flush just extent 87 with f:2 and g:2
Sep 22 23:15:14.806 DEBG [1] It's time to notify for 572
Sep 22 23:15:14.806 INFO Completion from [1] id:572 status:true
Sep 22 23:15:14.806 INFO [573/752] Repair commands completed
Sep 22 23:15:14.806 INFO Pop front: ReconcileIO { id: ReconciliationId(573), op: ExtentClose { repair_id: ReconciliationId(573), extent_id: 87 }, state: ClientData([New, New, New]) }
Sep 22 23:15:14.806 INFO Sent repair work, now wait for resp
Sep 22 23:15:14.806 INFO [0] received reconcile message
Sep 22 23:15:14.806 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(573), op: ExtentClose { repair_id: ReconciliationId(573), extent_id: 87 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:14.806 INFO [0] client ExtentClose { repair_id: ReconciliationId(573), extent_id: 87 }
Sep 22 23:15:14.806 INFO [1] received reconcile message
Sep 22 23:15:14.806 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(573), op: ExtentClose { repair_id: ReconciliationId(573), extent_id: 87 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:14.806 INFO [1] client ExtentClose { repair_id: ReconciliationId(573), extent_id: 87 }
Sep 22 23:15:14.806 INFO [2] received reconcile message
Sep 22 23:15:14.806 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(573), op: ExtentClose { repair_id: ReconciliationId(573), extent_id: 87 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:14.806 INFO [2] client ExtentClose { repair_id: ReconciliationId(573), extent_id: 87 }
Sep 22 23:15:14.806 DEBG 573 Close extent 87
Sep 22 23:15:14.806 DEBG 573 Close extent 87
Sep 22 23:15:14.807 DEBG 573 Close extent 87
Sep 22 23:15:14.807 DEBG [2] It's time to notify for 573
Sep 22 23:15:14.807 INFO Completion from [2] id:573 status:true
Sep 22 23:15:14.807 INFO [574/752] Repair commands completed
Sep 22 23:15:14.807 INFO Pop front: ReconcileIO { id: ReconciliationId(574), op: ExtentRepair { repair_id: ReconciliationId(574), extent_id: 87, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:14.807 INFO Sent repair work, now wait for resp
Sep 22 23:15:14.807 INFO [0] received reconcile message
Sep 22 23:15:14.807 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(574), op: ExtentRepair { repair_id: ReconciliationId(574), extent_id: 87, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:14.807 INFO [0] client ExtentRepair { repair_id: ReconciliationId(574), extent_id: 87, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:14.807 INFO [0] Sending repair request ReconciliationId(574)
Sep 22 23:15:14.807 INFO [1] received reconcile message
Sep 22 23:15:14.807 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(574), op: ExtentRepair { repair_id: ReconciliationId(574), extent_id: 87, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:14.807 INFO [1] client ExtentRepair { repair_id: ReconciliationId(574), extent_id: 87, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:14.807 INFO [1] No action required ReconciliationId(574)
Sep 22 23:15:14.807 INFO [2] received reconcile message
Sep 22 23:15:14.807 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(574), op: ExtentRepair { repair_id: ReconciliationId(574), extent_id: 87, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:14.807 INFO [2] client ExtentRepair { repair_id: ReconciliationId(574), extent_id: 87, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:14.807 INFO [2] No action required ReconciliationId(574)
Sep 22 23:15:14.807 DEBG 574 Repair extent 87 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:14.808 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/057.copy"
Sep 22 23:15:14.870 INFO accepted connection, remote_addr: 127.0.0.1:42401, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.870 TRCE incoming request, uri: /extent/87/files, method: GET, req_id: 442cd468-bf3a-4438-b35f-8e947dede51e, remote_addr: 127.0.0.1:42401, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.870 INFO request completed, latency_us: 191, response_code: 200, uri: /extent/87/files, method: GET, req_id: 442cd468-bf3a-4438-b35f-8e947dede51e, remote_addr: 127.0.0.1:42401, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.871 INFO eid:87 Found repair files: ["057", "057.db"]
Sep 22 23:15:14.871 TRCE incoming request, uri: /newextent/87/data, method: GET, req_id: 663ae246-fd5d-47aa-9c1d-d9466e772733, remote_addr: 127.0.0.1:42401, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.871 INFO request completed, latency_us: 318, response_code: 200, uri: /newextent/87/data, method: GET, req_id: 663ae246-fd5d-47aa-9c1d-d9466e772733, remote_addr: 127.0.0.1:42401, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.876 TRCE incoming request, uri: /newextent/87/db, method: GET, req_id: b09e60ae-56d5-4052-ae2b-e6d235a8e518, remote_addr: 127.0.0.1:42401, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.876 INFO request completed, latency_us: 292, response_code: 200, uri: /newextent/87/db, method: GET, req_id: b09e60ae-56d5-4052-ae2b-e6d235a8e518, remote_addr: 127.0.0.1:42401, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.878 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/057.copy" to "/tmp/downstairs-vrx8aK6L/00/000/057.replace"
Sep 22 23:15:14.878 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:14.878 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/057.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:14.879 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/057"
Sep 22 23:15:14.879 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/057.db"
Sep 22 23:15:14.879 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:14.879 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/057.replace" to "/tmp/downstairs-vrx8aK6L/00/000/057.completed"
Sep 22 23:15:14.879 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:14.879 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:14.879 DEBG [0] It's time to notify for 574
Sep 22 23:15:14.879 INFO Completion from [0] id:574 status:true
Sep 22 23:15:14.879 INFO [575/752] Repair commands completed
Sep 22 23:15:14.879 INFO Pop front: ReconcileIO { id: ReconciliationId(575), op: ExtentReopen { repair_id: ReconciliationId(575), extent_id: 87 }, state: ClientData([New, New, New]) }
Sep 22 23:15:14.879 INFO Sent repair work, now wait for resp
Sep 22 23:15:14.879 INFO [0] received reconcile message
Sep 22 23:15:14.879 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(575), op: ExtentReopen { repair_id: ReconciliationId(575), extent_id: 87 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:14.879 INFO [0] client ExtentReopen { repair_id: ReconciliationId(575), extent_id: 87 }
Sep 22 23:15:14.879 INFO [1] received reconcile message
Sep 22 23:15:14.879 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(575), op: ExtentReopen { repair_id: ReconciliationId(575), extent_id: 87 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:14.879 INFO [1] client ExtentReopen { repair_id: ReconciliationId(575), extent_id: 87 }
Sep 22 23:15:14.879 INFO [2] received reconcile message
Sep 22 23:15:14.879 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(575), op: ExtentReopen { repair_id: ReconciliationId(575), extent_id: 87 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:14.879 INFO [2] client ExtentReopen { repair_id: ReconciliationId(575), extent_id: 87 }
Sep 22 23:15:14.880 DEBG 575 Reopen extent 87
Sep 22 23:15:14.880 DEBG 575 Reopen extent 87
Sep 22 23:15:14.881 DEBG 575 Reopen extent 87
Sep 22 23:15:14.881 DEBG [2] It's time to notify for 575
Sep 22 23:15:14.881 INFO Completion from [2] id:575 status:true
Sep 22 23:15:14.881 INFO [576/752] Repair commands completed
Sep 22 23:15:14.881 INFO Pop front: ReconcileIO { id: ReconciliationId(576), op: ExtentFlush { repair_id: ReconciliationId(576), extent_id: 148, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:14.881 INFO Sent repair work, now wait for resp
Sep 22 23:15:14.881 INFO [0] received reconcile message
Sep 22 23:15:14.881 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(576), op: ExtentFlush { repair_id: ReconciliationId(576), extent_id: 148, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:14.881 INFO [0] client ExtentFlush { repair_id: ReconciliationId(576), extent_id: 148, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:14.881 INFO [1] received reconcile message
Sep 22 23:15:14.881 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(576), op: ExtentFlush { repair_id: ReconciliationId(576), extent_id: 148, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:14.881 INFO [1] client ExtentFlush { repair_id: ReconciliationId(576), extent_id: 148, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:14.881 INFO [2] received reconcile message
Sep 22 23:15:14.881 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(576), op: ExtentFlush { repair_id: ReconciliationId(576), extent_id: 148, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:14.881 INFO [2] client ExtentFlush { repair_id: ReconciliationId(576), extent_id: 148, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:14.882 DEBG 576 Flush extent 148 with f:2 g:2
Sep 22 23:15:14.882 DEBG Flush just extent 148 with f:2 and g:2
Sep 22 23:15:14.882 DEBG [1] It's time to notify for 576
Sep 22 23:15:14.882 INFO Completion from [1] id:576 status:true
Sep 22 23:15:14.882 INFO [577/752] Repair commands completed
Sep 22 23:15:14.882 INFO Pop front: ReconcileIO { id: ReconciliationId(577), op: ExtentClose { repair_id: ReconciliationId(577), extent_id: 148 }, state: ClientData([New, New, New]) }
Sep 22 23:15:14.882 INFO Sent repair work, now wait for resp
Sep 22 23:15:14.882 INFO [0] received reconcile message
Sep 22 23:15:14.882 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(577), op: ExtentClose { repair_id: ReconciliationId(577), extent_id: 148 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:14.882 INFO [0] client ExtentClose { repair_id: ReconciliationId(577), extent_id: 148 }
Sep 22 23:15:14.882 INFO [1] received reconcile message
Sep 22 23:15:14.882 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(577), op: ExtentClose { repair_id: ReconciliationId(577), extent_id: 148 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:14.882 INFO [1] client ExtentClose { repair_id: ReconciliationId(577), extent_id: 148 }
Sep 22 23:15:14.882 INFO [2] received reconcile message
Sep 22 23:15:14.882 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(577), op: ExtentClose { repair_id: ReconciliationId(577), extent_id: 148 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:14.882 INFO [2] client ExtentClose { repair_id: ReconciliationId(577), extent_id: 148 }
Sep 22 23:15:14.882 DEBG 577 Close extent 148
Sep 22 23:15:14.882 DEBG 577 Close extent 148
Sep 22 23:15:14.883 DEBG 577 Close extent 148
Sep 22 23:15:14.883 DEBG [2] It's time to notify for 577
Sep 22 23:15:14.883 INFO Completion from [2] id:577 status:true
Sep 22 23:15:14.883 INFO [578/752] Repair commands completed
Sep 22 23:15:14.883 INFO Pop front: ReconcileIO { id: ReconciliationId(578), op: ExtentRepair { repair_id: ReconciliationId(578), extent_id: 148, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:14.883 INFO Sent repair work, now wait for resp
Sep 22 23:15:14.883 INFO [0] received reconcile message
Sep 22 23:15:14.883 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(578), op: ExtentRepair { repair_id: ReconciliationId(578), extent_id: 148, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:14.883 INFO [0] client ExtentRepair { repair_id: ReconciliationId(578), extent_id: 148, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:14.883 INFO [0] Sending repair request ReconciliationId(578)
Sep 22 23:15:14.883 INFO [1] received reconcile message
Sep 22 23:15:14.883 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(578), op: ExtentRepair { repair_id: ReconciliationId(578), extent_id: 148, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:14.883 INFO [1] client ExtentRepair { repair_id: ReconciliationId(578), extent_id: 148, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:14.883 INFO [1] No action required ReconciliationId(578)
Sep 22 23:15:14.883 INFO [2] received reconcile message
Sep 22 23:15:14.883 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(578), op: ExtentRepair { repair_id: ReconciliationId(578), extent_id: 148, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:14.884 INFO [2] client ExtentRepair { repair_id: ReconciliationId(578), extent_id: 148, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:14.884 INFO [2] No action required ReconciliationId(578)
Sep 22 23:15:14.884 DEBG 578 Repair extent 148 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:14.884 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/094.copy"
Sep 22 23:15:14.944 DEBG up_ds_listen was notified
Sep 22 23:15:14.944 DEBG up_ds_listen process 1087
Sep 22 23:15:14.944 DEBG [A] ack job 1087:88, : downstairs
Sep 22 23:15:14.944 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:14.944 DEBG IO Flush 1089 has deps [JobId(1088), JobId(1087)]
Sep 22 23:15:14.944 WARN returning error on read!
Sep 22 23:15:14.944 DEBG Read      :1088 deps:[JobId(1087)] res:false
Sep 22 23:15:14.948 INFO accepted connection, remote_addr: 127.0.0.1:46187, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.948 TRCE incoming request, uri: /extent/148/files, method: GET, req_id: 2135013c-1c22-4b14-95ad-2233260370b0, remote_addr: 127.0.0.1:46187, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.948 INFO request completed, latency_us: 191, response_code: 200, uri: /extent/148/files, method: GET, req_id: 2135013c-1c22-4b14-95ad-2233260370b0, remote_addr: 127.0.0.1:46187, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.949 INFO eid:148 Found repair files: ["094", "094.db"]
Sep 22 23:15:14.949 TRCE incoming request, uri: /newextent/148/data, method: GET, req_id: c70ccd0b-3da6-41a8-b3aa-d0e2004bdec1, remote_addr: 127.0.0.1:46187, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.949 INFO request completed, latency_us: 323, response_code: 200, uri: /newextent/148/data, method: GET, req_id: c70ccd0b-3da6-41a8-b3aa-d0e2004bdec1, remote_addr: 127.0.0.1:46187, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.950 DEBG Read      :1088 deps:[JobId(1087)] res:true
Sep 22 23:15:14.954 TRCE incoming request, uri: /newextent/148/db, method: GET, req_id: ca54f289-46d3-4a36-9f62-77ae8f77d56e, remote_addr: 127.0.0.1:46187, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.955 INFO request completed, latency_us: 293, response_code: 200, uri: /newextent/148/db, method: GET, req_id: ca54f289-46d3-4a36-9f62-77ae8f77d56e, remote_addr: 127.0.0.1:46187, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:14.956 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/094.copy" to "/tmp/downstairs-vrx8aK6L/00/000/094.replace"
Sep 22 23:15:14.956 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:14.956 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/094.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:14.957 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/094"
Sep 22 23:15:14.957 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/094.db"
Sep 22 23:15:14.957 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:14.957 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/094.replace" to "/tmp/downstairs-vrx8aK6L/00/000/094.completed"
Sep 22 23:15:14.957 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:14.957 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:14.957 DEBG [0] It's time to notify for 578
Sep 22 23:15:14.957 INFO Completion from [0] id:578 status:true
Sep 22 23:15:14.957 INFO [579/752] Repair commands completed
Sep 22 23:15:14.957 INFO Pop front: ReconcileIO { id: ReconciliationId(579), op: ExtentReopen { repair_id: ReconciliationId(579), extent_id: 148 }, state: ClientData([New, New, New]) }
Sep 22 23:15:14.957 INFO Sent repair work, now wait for resp
Sep 22 23:15:14.957 INFO [0] received reconcile message
Sep 22 23:15:14.957 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(579), op: ExtentReopen { repair_id: ReconciliationId(579), extent_id: 148 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:14.957 INFO [0] client ExtentReopen { repair_id: ReconciliationId(579), extent_id: 148 }
Sep 22 23:15:14.957 INFO [1] received reconcile message
Sep 22 23:15:14.957 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(579), op: ExtentReopen { repair_id: ReconciliationId(579), extent_id: 148 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:14.957 INFO [1] client ExtentReopen { repair_id: ReconciliationId(579), extent_id: 148 }
Sep 22 23:15:14.958 INFO [2] received reconcile message
Sep 22 23:15:14.958 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(579), op: ExtentReopen { repair_id: ReconciliationId(579), extent_id: 148 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:14.958 INFO [2] client ExtentReopen { repair_id: ReconciliationId(579), extent_id: 148 }
Sep 22 23:15:14.958 DEBG 579 Reopen extent 148
Sep 22 23:15:14.958 DEBG 579 Reopen extent 148
Sep 22 23:15:14.959 DEBG 579 Reopen extent 148
Sep 22 23:15:14.959 DEBG [2] It's time to notify for 579
Sep 22 23:15:14.959 INFO Completion from [2] id:579 status:true
Sep 22 23:15:14.959 INFO [580/752] Repair commands completed
Sep 22 23:15:14.959 INFO Pop front: ReconcileIO { id: ReconciliationId(580), op: ExtentFlush { repair_id: ReconciliationId(580), extent_id: 162, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:14.959 INFO Sent repair work, now wait for resp
Sep 22 23:15:14.959 INFO [0] received reconcile message
Sep 22 23:15:14.959 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(580), op: ExtentFlush { repair_id: ReconciliationId(580), extent_id: 162, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:14.959 INFO [0] client ExtentFlush { repair_id: ReconciliationId(580), extent_id: 162, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:14.960 INFO [1] received reconcile message
Sep 22 23:15:14.960 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(580), op: ExtentFlush { repair_id: ReconciliationId(580), extent_id: 162, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:14.960 INFO [1] client ExtentFlush { repair_id: ReconciliationId(580), extent_id: 162, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:14.960 INFO [2] received reconcile message
Sep 22 23:15:14.960 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(580), op: ExtentFlush { repair_id: ReconciliationId(580), extent_id: 162, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:14.960 INFO [2] client ExtentFlush { repair_id: ReconciliationId(580), extent_id: 162, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:14.960 DEBG 580 Flush extent 162 with f:2 g:2
Sep 22 23:15:14.960 DEBG Flush just extent 162 with f:2 and g:2
Sep 22 23:15:14.960 DEBG [1] It's time to notify for 580
Sep 22 23:15:14.960 INFO Completion from [1] id:580 status:true
Sep 22 23:15:14.960 INFO [581/752] Repair commands completed
Sep 22 23:15:14.960 INFO Pop front: ReconcileIO { id: ReconciliationId(581), op: ExtentClose { repair_id: ReconciliationId(581), extent_id: 162 }, state: ClientData([New, New, New]) }
Sep 22 23:15:14.960 INFO Sent repair work, now wait for resp
Sep 22 23:15:14.960 INFO [0] received reconcile message
Sep 22 23:15:14.960 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(581), op: ExtentClose { repair_id: ReconciliationId(581), extent_id: 162 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:14.960 INFO [0] client ExtentClose { repair_id: ReconciliationId(581), extent_id: 162 }
Sep 22 23:15:14.960 INFO [1] received reconcile message
Sep 22 23:15:14.960 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(581), op: ExtentClose { repair_id: ReconciliationId(581), extent_id: 162 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:14.960 INFO [1] client ExtentClose { repair_id: ReconciliationId(581), extent_id: 162 }
Sep 22 23:15:14.960 INFO [2] received reconcile message
Sep 22 23:15:14.960 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(581), op: ExtentClose { repair_id: ReconciliationId(581), extent_id: 162 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:14.960 INFO [2] client ExtentClose { repair_id: ReconciliationId(581), extent_id: 162 }
Sep 22 23:15:14.960 DEBG 581 Close extent 162
Sep 22 23:15:14.961 DEBG 581 Close extent 162
Sep 22 23:15:14.961 DEBG 581 Close extent 162
Sep 22 23:15:14.961 DEBG [2] It's time to notify for 581
Sep 22 23:15:14.961 INFO Completion from [2] id:581 status:true
Sep 22 23:15:14.961 INFO [582/752] Repair commands completed
Sep 22 23:15:14.961 INFO Pop front: ReconcileIO { id: ReconciliationId(582), op: ExtentRepair { repair_id: ReconciliationId(582), extent_id: 162, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:14.961 INFO Sent repair work, now wait for resp
Sep 22 23:15:14.961 INFO [0] received reconcile message
Sep 22 23:15:14.961 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(582), op: ExtentRepair { repair_id: ReconciliationId(582), extent_id: 162, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:14.961 INFO [0] client ExtentRepair { repair_id: ReconciliationId(582), extent_id: 162, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:14.961 INFO [0] Sending repair request ReconciliationId(582)
Sep 22 23:15:14.962 INFO [1] received reconcile message
Sep 22 23:15:14.962 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(582), op: ExtentRepair { repair_id: ReconciliationId(582), extent_id: 162, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:14.962 INFO [1] client ExtentRepair { repair_id: ReconciliationId(582), extent_id: 162, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:14.962 INFO [1] No action required ReconciliationId(582)
Sep 22 23:15:14.962 INFO [2] received reconcile message
Sep 22 23:15:14.962 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(582), op: ExtentRepair { repair_id: ReconciliationId(582), extent_id: 162, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:14.962 INFO [2] client ExtentRepair { repair_id: ReconciliationId(582), extent_id: 162, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:14.962 INFO [2] No action required ReconciliationId(582)
Sep 22 23:15:14.962 DEBG 582 Repair extent 162 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:14.962 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/0A2.copy"
Sep 22 23:15:14.973 ERRO [1] job id 1088 saw error GenericError("test error")
Sep 22 23:15:14.975 INFO [lossy] sleeping 1 second
Sep 22 23:15:15.026 INFO accepted connection, remote_addr: 127.0.0.1:58578, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.026 TRCE incoming request, uri: /extent/162/files, method: GET, req_id: a87aa6a4-b695-40a4-85bf-fab4b456057c, remote_addr: 127.0.0.1:58578, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.026 INFO request completed, latency_us: 266, response_code: 200, uri: /extent/162/files, method: GET, req_id: a87aa6a4-b695-40a4-85bf-fab4b456057c, remote_addr: 127.0.0.1:58578, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.026 INFO eid:162 Found repair files: ["0A2", "0A2.db"]
Sep 22 23:15:15.027 TRCE incoming request, uri: /newextent/162/data, method: GET, req_id: 64309948-5be8-4229-96a3-bd73a2762c45, remote_addr: 127.0.0.1:58578, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.027 INFO request completed, latency_us: 372, response_code: 200, uri: /newextent/162/data, method: GET, req_id: 64309948-5be8-4229-96a3-bd73a2762c45, remote_addr: 127.0.0.1:58578, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.032 TRCE incoming request, uri: /newextent/162/db, method: GET, req_id: d9bb3628-a437-471b-92e9-34a85eb25756, remote_addr: 127.0.0.1:58578, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.033 INFO request completed, latency_us: 293, response_code: 200, uri: /newextent/162/db, method: GET, req_id: d9bb3628-a437-471b-92e9-34a85eb25756, remote_addr: 127.0.0.1:58578, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.034 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/0A2.copy" to "/tmp/downstairs-vrx8aK6L/00/000/0A2.replace"
Sep 22 23:15:15.034 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:15.035 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/0A2.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:15.035 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/0A2"
Sep 22 23:15:15.035 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/0A2.db"
Sep 22 23:15:15.035 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:15.035 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/0A2.replace" to "/tmp/downstairs-vrx8aK6L/00/000/0A2.completed"
Sep 22 23:15:15.035 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:15.035 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:15.036 DEBG [0] It's time to notify for 582
Sep 22 23:15:15.036 INFO Completion from [0] id:582 status:true
Sep 22 23:15:15.036 INFO [583/752] Repair commands completed
Sep 22 23:15:15.036 INFO Pop front: ReconcileIO { id: ReconciliationId(583), op: ExtentReopen { repair_id: ReconciliationId(583), extent_id: 162 }, state: ClientData([New, New, New]) }
Sep 22 23:15:15.036 INFO Sent repair work, now wait for resp
Sep 22 23:15:15.036 INFO [0] received reconcile message
Sep 22 23:15:15.036 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(583), op: ExtentReopen { repair_id: ReconciliationId(583), extent_id: 162 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:15.036 INFO [0] client ExtentReopen { repair_id: ReconciliationId(583), extent_id: 162 }
Sep 22 23:15:15.036 INFO [1] received reconcile message
Sep 22 23:15:15.036 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(583), op: ExtentReopen { repair_id: ReconciliationId(583), extent_id: 162 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:15.036 INFO [1] client ExtentReopen { repair_id: ReconciliationId(583), extent_id: 162 }
Sep 22 23:15:15.036 INFO [2] received reconcile message
Sep 22 23:15:15.036 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(583), op: ExtentReopen { repair_id: ReconciliationId(583), extent_id: 162 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:15.036 INFO [2] client ExtentReopen { repair_id: ReconciliationId(583), extent_id: 162 }
Sep 22 23:15:15.036 DEBG 583 Reopen extent 162
Sep 22 23:15:15.037 DEBG 583 Reopen extent 162
Sep 22 23:15:15.037 DEBG 583 Reopen extent 162
Sep 22 23:15:15.038 DEBG [2] It's time to notify for 583
Sep 22 23:15:15.038 INFO Completion from [2] id:583 status:true
Sep 22 23:15:15.038 INFO [584/752] Repair commands completed
Sep 22 23:15:15.038 INFO Pop front: ReconcileIO { id: ReconciliationId(584), op: ExtentFlush { repair_id: ReconciliationId(584), extent_id: 51, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:15.038 INFO Sent repair work, now wait for resp
Sep 22 23:15:15.038 INFO [0] received reconcile message
Sep 22 23:15:15.038 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(584), op: ExtentFlush { repair_id: ReconciliationId(584), extent_id: 51, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:15.038 INFO [0] client ExtentFlush { repair_id: ReconciliationId(584), extent_id: 51, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:15.038 INFO [1] received reconcile message
Sep 22 23:15:15.038 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(584), op: ExtentFlush { repair_id: ReconciliationId(584), extent_id: 51, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:15.038 INFO [1] client ExtentFlush { repair_id: ReconciliationId(584), extent_id: 51, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:15.038 INFO [2] received reconcile message
Sep 22 23:15:15.038 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(584), op: ExtentFlush { repair_id: ReconciliationId(584), extent_id: 51, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:15.038 INFO [2] client ExtentFlush { repair_id: ReconciliationId(584), extent_id: 51, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:15.038 DEBG 584 Flush extent 51 with f:2 g:2
Sep 22 23:15:15.038 DEBG Flush just extent 51 with f:2 and g:2
Sep 22 23:15:15.039 DEBG [1] It's time to notify for 584
Sep 22 23:15:15.039 INFO Completion from [1] id:584 status:true
Sep 22 23:15:15.039 INFO [585/752] Repair commands completed
Sep 22 23:15:15.039 INFO Pop front: ReconcileIO { id: ReconciliationId(585), op: ExtentClose { repair_id: ReconciliationId(585), extent_id: 51 }, state: ClientData([New, New, New]) }
Sep 22 23:15:15.039 INFO Sent repair work, now wait for resp
Sep 22 23:15:15.039 INFO [0] received reconcile message
Sep 22 23:15:15.039 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(585), op: ExtentClose { repair_id: ReconciliationId(585), extent_id: 51 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:15.039 INFO [0] client ExtentClose { repair_id: ReconciliationId(585), extent_id: 51 }
Sep 22 23:15:15.039 INFO [1] received reconcile message
Sep 22 23:15:15.039 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(585), op: ExtentClose { repair_id: ReconciliationId(585), extent_id: 51 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:15.039 INFO [1] client ExtentClose { repair_id: ReconciliationId(585), extent_id: 51 }
Sep 22 23:15:15.039 INFO [2] received reconcile message
Sep 22 23:15:15.039 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(585), op: ExtentClose { repair_id: ReconciliationId(585), extent_id: 51 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:15.039 INFO [2] client ExtentClose { repair_id: ReconciliationId(585), extent_id: 51 }
Sep 22 23:15:15.039 DEBG 585 Close extent 51
Sep 22 23:15:15.039 DEBG 585 Close extent 51
Sep 22 23:15:15.040 DEBG 585 Close extent 51
Sep 22 23:15:15.040 DEBG [2] It's time to notify for 585
Sep 22 23:15:15.040 INFO Completion from [2] id:585 status:true
Sep 22 23:15:15.040 INFO [586/752] Repair commands completed
Sep 22 23:15:15.040 INFO Pop front: ReconcileIO { id: ReconciliationId(586), op: ExtentRepair { repair_id: ReconciliationId(586), extent_id: 51, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:15.040 INFO Sent repair work, now wait for resp
Sep 22 23:15:15.040 INFO [0] received reconcile message
Sep 22 23:15:15.040 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(586), op: ExtentRepair { repair_id: ReconciliationId(586), extent_id: 51, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:15.040 INFO [0] client ExtentRepair { repair_id: ReconciliationId(586), extent_id: 51, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:15.040 INFO [0] Sending repair request ReconciliationId(586)
Sep 22 23:15:15.040 INFO [1] received reconcile message
Sep 22 23:15:15.040 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(586), op: ExtentRepair { repair_id: ReconciliationId(586), extent_id: 51, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:15.040 INFO [1] client ExtentRepair { repair_id: ReconciliationId(586), extent_id: 51, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:15.040 INFO [1] No action required ReconciliationId(586)
Sep 22 23:15:15.040 INFO [2] received reconcile message
Sep 22 23:15:15.040 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(586), op: ExtentRepair { repair_id: ReconciliationId(586), extent_id: 51, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:15.040 INFO [2] client ExtentRepair { repair_id: ReconciliationId(586), extent_id: 51, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:15.040 INFO [2] No action required ReconciliationId(586)
Sep 22 23:15:15.040 DEBG 586 Repair extent 51 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:15.041 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/033.copy"
Sep 22 23:15:15.102 INFO accepted connection, remote_addr: 127.0.0.1:45475, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.103 TRCE incoming request, uri: /extent/51/files, method: GET, req_id: 0f6bc778-159c-4b7a-b14a-f6bd963b34ab, remote_addr: 127.0.0.1:45475, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.103 INFO request completed, latency_us: 203, response_code: 200, uri: /extent/51/files, method: GET, req_id: 0f6bc778-159c-4b7a-b14a-f6bd963b34ab, remote_addr: 127.0.0.1:45475, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.103 INFO eid:51 Found repair files: ["033", "033.db"]
Sep 22 23:15:15.103 TRCE incoming request, uri: /newextent/51/data, method: GET, req_id: 4a5b5cc1-7c08-4b86-9203-563aee94d012, remote_addr: 127.0.0.1:45475, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.104 INFO request completed, latency_us: 305, response_code: 200, uri: /newextent/51/data, method: GET, req_id: 4a5b5cc1-7c08-4b86-9203-563aee94d012, remote_addr: 127.0.0.1:45475, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.109 TRCE incoming request, uri: /newextent/51/db, method: GET, req_id: 86d92040-b160-4e5a-b356-2019f151fa2d, remote_addr: 127.0.0.1:45475, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.109 INFO request completed, latency_us: 291, response_code: 200, uri: /newextent/51/db, method: GET, req_id: 86d92040-b160-4e5a-b356-2019f151fa2d, remote_addr: 127.0.0.1:45475, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.110 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/033.copy" to "/tmp/downstairs-vrx8aK6L/00/000/033.replace"
Sep 22 23:15:15.110 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:15.111 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/033.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:15.111 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/033"
Sep 22 23:15:15.111 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/033.db"
Sep 22 23:15:15.111 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:15.111 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/033.replace" to "/tmp/downstairs-vrx8aK6L/00/000/033.completed"
Sep 22 23:15:15.111 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:15.111 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:15.111 DEBG [0] It's time to notify for 586
Sep 22 23:15:15.111 INFO Completion from [0] id:586 status:true
Sep 22 23:15:15.112 INFO [587/752] Repair commands completed
Sep 22 23:15:15.112 INFO Pop front: ReconcileIO { id: ReconciliationId(587), op: ExtentReopen { repair_id: ReconciliationId(587), extent_id: 51 }, state: ClientData([New, New, New]) }
Sep 22 23:15:15.112 INFO Sent repair work, now wait for resp
Sep 22 23:15:15.112 INFO [0] received reconcile message
Sep 22 23:15:15.112 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(587), op: ExtentReopen { repair_id: ReconciliationId(587), extent_id: 51 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:15.112 INFO [0] client ExtentReopen { repair_id: ReconciliationId(587), extent_id: 51 }
Sep 22 23:15:15.112 INFO [1] received reconcile message
Sep 22 23:15:15.112 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(587), op: ExtentReopen { repair_id: ReconciliationId(587), extent_id: 51 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:15.112 INFO [1] client ExtentReopen { repair_id: ReconciliationId(587), extent_id: 51 }
Sep 22 23:15:15.112 INFO [2] received reconcile message
Sep 22 23:15:15.112 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(587), op: ExtentReopen { repair_id: ReconciliationId(587), extent_id: 51 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:15.112 INFO [2] client ExtentReopen { repair_id: ReconciliationId(587), extent_id: 51 }
Sep 22 23:15:15.112 DEBG 587 Reopen extent 51
Sep 22 23:15:15.112 DEBG 587 Reopen extent 51
Sep 22 23:15:15.113 DEBG 587 Reopen extent 51
Sep 22 23:15:15.114 DEBG [2] It's time to notify for 587
Sep 22 23:15:15.114 INFO Completion from [2] id:587 status:true
Sep 22 23:15:15.114 INFO [588/752] Repair commands completed
Sep 22 23:15:15.114 INFO Pop front: ReconcileIO { id: ReconciliationId(588), op: ExtentFlush { repair_id: ReconciliationId(588), extent_id: 39, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:15.114 INFO Sent repair work, now wait for resp
Sep 22 23:15:15.114 INFO [0] received reconcile message
Sep 22 23:15:15.114 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(588), op: ExtentFlush { repair_id: ReconciliationId(588), extent_id: 39, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:15.114 INFO [0] client ExtentFlush { repair_id: ReconciliationId(588), extent_id: 39, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:15.114 INFO [1] received reconcile message
Sep 22 23:15:15.114 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(588), op: ExtentFlush { repair_id: ReconciliationId(588), extent_id: 39, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:15.114 INFO [1] client ExtentFlush { repair_id: ReconciliationId(588), extent_id: 39, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:15.114 INFO [2] received reconcile message
Sep 22 23:15:15.114 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(588), op: ExtentFlush { repair_id: ReconciliationId(588), extent_id: 39, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:15.114 INFO [2] client ExtentFlush { repair_id: ReconciliationId(588), extent_id: 39, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:15.114 DEBG 588 Flush extent 39 with f:2 g:2
Sep 22 23:15:15.114 DEBG Flush just extent 39 with f:2 and g:2
Sep 22 23:15:15.114 DEBG [1] It's time to notify for 588
Sep 22 23:15:15.114 INFO Completion from [1] id:588 status:true
Sep 22 23:15:15.114 INFO [589/752] Repair commands completed
Sep 22 23:15:15.114 INFO Pop front: ReconcileIO { id: ReconciliationId(589), op: ExtentClose { repair_id: ReconciliationId(589), extent_id: 39 }, state: ClientData([New, New, New]) }
Sep 22 23:15:15.114 INFO Sent repair work, now wait for resp
Sep 22 23:15:15.114 INFO [0] received reconcile message
Sep 22 23:15:15.114 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(589), op: ExtentClose { repair_id: ReconciliationId(589), extent_id: 39 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:15.114 INFO [0] client ExtentClose { repair_id: ReconciliationId(589), extent_id: 39 }
Sep 22 23:15:15.114 INFO [1] received reconcile message
Sep 22 23:15:15.114 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(589), op: ExtentClose { repair_id: ReconciliationId(589), extent_id: 39 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:15.114 INFO [1] client ExtentClose { repair_id: ReconciliationId(589), extent_id: 39 }
Sep 22 23:15:15.114 INFO [2] received reconcile message
Sep 22 23:15:15.114 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(589), op: ExtentClose { repair_id: ReconciliationId(589), extent_id: 39 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:15.115 INFO [2] client ExtentClose { repair_id: ReconciliationId(589), extent_id: 39 }
Sep 22 23:15:15.115 DEBG 589 Close extent 39
Sep 22 23:15:15.115 DEBG 589 Close extent 39
Sep 22 23:15:15.115 DEBG 589 Close extent 39
Sep 22 23:15:15.116 DEBG [2] It's time to notify for 589
Sep 22 23:15:15.116 INFO Completion from [2] id:589 status:true
Sep 22 23:15:15.116 INFO [590/752] Repair commands completed
Sep 22 23:15:15.116 INFO Pop front: ReconcileIO { id: ReconciliationId(590), op: ExtentRepair { repair_id: ReconciliationId(590), extent_id: 39, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:15.116 INFO Sent repair work, now wait for resp
Sep 22 23:15:15.116 INFO [0] received reconcile message
Sep 22 23:15:15.116 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(590), op: ExtentRepair { repair_id: ReconciliationId(590), extent_id: 39, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:15.116 INFO [0] client ExtentRepair { repair_id: ReconciliationId(590), extent_id: 39, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:15.116 INFO [0] Sending repair request ReconciliationId(590)
Sep 22 23:15:15.116 INFO [1] received reconcile message
Sep 22 23:15:15.116 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(590), op: ExtentRepair { repair_id: ReconciliationId(590), extent_id: 39, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:15.116 INFO [1] client ExtentRepair { repair_id: ReconciliationId(590), extent_id: 39, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:15.116 INFO [1] No action required ReconciliationId(590)
Sep 22 23:15:15.116 INFO [2] received reconcile message
Sep 22 23:15:15.116 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(590), op: ExtentRepair { repair_id: ReconciliationId(590), extent_id: 39, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:15.116 INFO [2] client ExtentRepair { repair_id: ReconciliationId(590), extent_id: 39, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:15.116 INFO [2] No action required ReconciliationId(590)
Sep 22 23:15:15.116 DEBG 590 Repair extent 39 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:15.116 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/027.copy"
Sep 22 23:15:15.180 INFO accepted connection, remote_addr: 127.0.0.1:57350, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.180 TRCE incoming request, uri: /extent/39/files, method: GET, req_id: b68b4c1a-ae9d-4bd9-aa41-5f91bb4c240e, remote_addr: 127.0.0.1:57350, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.180 INFO request completed, latency_us: 190, response_code: 200, uri: /extent/39/files, method: GET, req_id: b68b4c1a-ae9d-4bd9-aa41-5f91bb4c240e, remote_addr: 127.0.0.1:57350, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.180 INFO eid:39 Found repair files: ["027", "027.db"]
Sep 22 23:15:15.181 TRCE incoming request, uri: /newextent/39/data, method: GET, req_id: efa14085-a44b-4164-87d4-45db2581c6a9, remote_addr: 127.0.0.1:57350, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.181 INFO request completed, latency_us: 251, response_code: 200, uri: /newextent/39/data, method: GET, req_id: efa14085-a44b-4164-87d4-45db2581c6a9, remote_addr: 127.0.0.1:57350, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.186 TRCE incoming request, uri: /newextent/39/db, method: GET, req_id: a284ae0c-a07f-41ac-beab-f0497e27a379, remote_addr: 127.0.0.1:57350, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.186 INFO request completed, latency_us: 302, response_code: 200, uri: /newextent/39/db, method: GET, req_id: a284ae0c-a07f-41ac-beab-f0497e27a379, remote_addr: 127.0.0.1:57350, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.187 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/027.copy" to "/tmp/downstairs-vrx8aK6L/00/000/027.replace"
Sep 22 23:15:15.187 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:15.188 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/027.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:15.188 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/027"
Sep 22 23:15:15.188 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/027.db"
Sep 22 23:15:15.188 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:15.188 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/027.replace" to "/tmp/downstairs-vrx8aK6L/00/000/027.completed"
Sep 22 23:15:15.188 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:15.188 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:15.188 DEBG [0] It's time to notify for 590
Sep 22 23:15:15.189 INFO Completion from [0] id:590 status:true
Sep 22 23:15:15.189 INFO [591/752] Repair commands completed
Sep 22 23:15:15.189 INFO Pop front: ReconcileIO { id: ReconciliationId(591), op: ExtentReopen { repair_id: ReconciliationId(591), extent_id: 39 }, state: ClientData([New, New, New]) }
Sep 22 23:15:15.189 INFO Sent repair work, now wait for resp
Sep 22 23:15:15.189 INFO [0] received reconcile message
Sep 22 23:15:15.189 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(591), op: ExtentReopen { repair_id: ReconciliationId(591), extent_id: 39 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:15.189 INFO [0] client ExtentReopen { repair_id: ReconciliationId(591), extent_id: 39 }
Sep 22 23:15:15.189 INFO [1] received reconcile message
Sep 22 23:15:15.189 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(591), op: ExtentReopen { repair_id: ReconciliationId(591), extent_id: 39 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:15.189 INFO [1] client ExtentReopen { repair_id: ReconciliationId(591), extent_id: 39 }
Sep 22 23:15:15.189 INFO [2] received reconcile message
Sep 22 23:15:15.189 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(591), op: ExtentReopen { repair_id: ReconciliationId(591), extent_id: 39 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:15.189 INFO [2] client ExtentReopen { repair_id: ReconciliationId(591), extent_id: 39 }
Sep 22 23:15:15.189 DEBG 591 Reopen extent 39
Sep 22 23:15:15.189 DEBG 591 Reopen extent 39
Sep 22 23:15:15.190 DEBG 591 Reopen extent 39
Sep 22 23:15:15.191 DEBG [2] It's time to notify for 591
Sep 22 23:15:15.191 INFO Completion from [2] id:591 status:true
Sep 22 23:15:15.191 INFO [592/752] Repair commands completed
Sep 22 23:15:15.191 INFO Pop front: ReconcileIO { id: ReconciliationId(592), op: ExtentFlush { repair_id: ReconciliationId(592), extent_id: 155, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:15.191 INFO Sent repair work, now wait for resp
Sep 22 23:15:15.191 INFO [0] received reconcile message
Sep 22 23:15:15.191 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(592), op: ExtentFlush { repair_id: ReconciliationId(592), extent_id: 155, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:15.191 INFO [0] client ExtentFlush { repair_id: ReconciliationId(592), extent_id: 155, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:15.191 INFO [1] received reconcile message
Sep 22 23:15:15.191 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(592), op: ExtentFlush { repair_id: ReconciliationId(592), extent_id: 155, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:15.191 INFO [1] client ExtentFlush { repair_id: ReconciliationId(592), extent_id: 155, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:15.191 INFO [2] received reconcile message
Sep 22 23:15:15.191 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(592), op: ExtentFlush { repair_id: ReconciliationId(592), extent_id: 155, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:15.191 INFO [2] client ExtentFlush { repair_id: ReconciliationId(592), extent_id: 155, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:15.191 DEBG 592 Flush extent 155 with f:2 g:2
Sep 22 23:15:15.191 DEBG Flush just extent 155 with f:2 and g:2
Sep 22 23:15:15.191 DEBG [1] It's time to notify for 592
Sep 22 23:15:15.191 INFO Completion from [1] id:592 status:true
Sep 22 23:15:15.191 INFO [593/752] Repair commands completed
Sep 22 23:15:15.191 INFO Pop front: ReconcileIO { id: ReconciliationId(593), op: ExtentClose { repair_id: ReconciliationId(593), extent_id: 155 }, state: ClientData([New, New, New]) }
Sep 22 23:15:15.191 INFO Sent repair work, now wait for resp
Sep 22 23:15:15.191 INFO [0] received reconcile message
Sep 22 23:15:15.191 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(593), op: ExtentClose { repair_id: ReconciliationId(593), extent_id: 155 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:15.191 INFO [0] client ExtentClose { repair_id: ReconciliationId(593), extent_id: 155 }
Sep 22 23:15:15.191 INFO [1] received reconcile message
Sep 22 23:15:15.191 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(593), op: ExtentClose { repair_id: ReconciliationId(593), extent_id: 155 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:15.191 INFO [1] client ExtentClose { repair_id: ReconciliationId(593), extent_id: 155 }
Sep 22 23:15:15.191 INFO [2] received reconcile message
Sep 22 23:15:15.191 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(593), op: ExtentClose { repair_id: ReconciliationId(593), extent_id: 155 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:15.191 INFO [2] client ExtentClose { repair_id: ReconciliationId(593), extent_id: 155 }
Sep 22 23:15:15.192 DEBG 593 Close extent 155
Sep 22 23:15:15.192 DEBG 593 Close extent 155
Sep 22 23:15:15.192 DEBG 593 Close extent 155
Sep 22 23:15:15.193 DEBG [2] It's time to notify for 593
Sep 22 23:15:15.193 INFO Completion from [2] id:593 status:true
Sep 22 23:15:15.193 INFO [594/752] Repair commands completed
Sep 22 23:15:15.193 INFO Pop front: ReconcileIO { id: ReconciliationId(594), op: ExtentRepair { repair_id: ReconciliationId(594), extent_id: 155, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:15.193 INFO Sent repair work, now wait for resp
Sep 22 23:15:15.193 INFO [0] received reconcile message
Sep 22 23:15:15.193 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(594), op: ExtentRepair { repair_id: ReconciliationId(594), extent_id: 155, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:15.193 INFO [0] client ExtentRepair { repair_id: ReconciliationId(594), extent_id: 155, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:15.193 INFO [0] Sending repair request ReconciliationId(594)
Sep 22 23:15:15.193 INFO [1] received reconcile message
Sep 22 23:15:15.193 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(594), op: ExtentRepair { repair_id: ReconciliationId(594), extent_id: 155, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:15.193 INFO [1] client ExtentRepair { repair_id: ReconciliationId(594), extent_id: 155, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:15.193 INFO [1] No action required ReconciliationId(594)
Sep 22 23:15:15.193 INFO [2] received reconcile message
Sep 22 23:15:15.193 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(594), op: ExtentRepair { repair_id: ReconciliationId(594), extent_id: 155, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:15.193 INFO [2] client ExtentRepair { repair_id: ReconciliationId(594), extent_id: 155, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:15.193 INFO [2] No action required ReconciliationId(594)
Sep 22 23:15:15.193 DEBG 594 Repair extent 155 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:15.193 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/09B.copy"
Sep 22 23:15:15.257 INFO accepted connection, remote_addr: 127.0.0.1:37066, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.257 TRCE incoming request, uri: /extent/155/files, method: GET, req_id: 21739cb4-3a30-4450-8a8b-eb38ff565a76, remote_addr: 127.0.0.1:37066, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.257 INFO request completed, latency_us: 202, response_code: 200, uri: /extent/155/files, method: GET, req_id: 21739cb4-3a30-4450-8a8b-eb38ff565a76, remote_addr: 127.0.0.1:37066, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.257 INFO eid:155 Found repair files: ["09B", "09B.db"]
Sep 22 23:15:15.258 TRCE incoming request, uri: /newextent/155/data, method: GET, req_id: 89485d93-3a87-4c1b-be39-1e2eec364824, remote_addr: 127.0.0.1:37066, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.258 INFO request completed, latency_us: 250, response_code: 200, uri: /newextent/155/data, method: GET, req_id: 89485d93-3a87-4c1b-be39-1e2eec364824, remote_addr: 127.0.0.1:37066, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.263 TRCE incoming request, uri: /newextent/155/db, method: GET, req_id: d00832d5-c0ea-44a1-b07b-933af4e4889d, remote_addr: 127.0.0.1:37066, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.263 INFO request completed, latency_us: 292, response_code: 200, uri: /newextent/155/db, method: GET, req_id: d00832d5-c0ea-44a1-b07b-933af4e4889d, remote_addr: 127.0.0.1:37066, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.264 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/09B.copy" to "/tmp/downstairs-vrx8aK6L/00/000/09B.replace"
Sep 22 23:15:15.264 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:15.265 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/09B.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:15.265 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/09B"
Sep 22 23:15:15.265 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/09B.db"
Sep 22 23:15:15.265 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:15.265 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/09B.replace" to "/tmp/downstairs-vrx8aK6L/00/000/09B.completed"
Sep 22 23:15:15.265 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:15.265 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:15.266 DEBG [0] It's time to notify for 594
Sep 22 23:15:15.266 INFO Completion from [0] id:594 status:true
Sep 22 23:15:15.266 INFO [595/752] Repair commands completed
Sep 22 23:15:15.266 INFO Pop front: ReconcileIO { id: ReconciliationId(595), op: ExtentReopen { repair_id: ReconciliationId(595), extent_id: 155 }, state: ClientData([New, New, New]) }
Sep 22 23:15:15.266 INFO Sent repair work, now wait for resp
Sep 22 23:15:15.266 INFO [0] received reconcile message
Sep 22 23:15:15.266 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(595), op: ExtentReopen { repair_id: ReconciliationId(595), extent_id: 155 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:15.266 INFO [0] client ExtentReopen { repair_id: ReconciliationId(595), extent_id: 155 }
Sep 22 23:15:15.266 INFO [1] received reconcile message
Sep 22 23:15:15.266 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(595), op: ExtentReopen { repair_id: ReconciliationId(595), extent_id: 155 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:15.266 INFO [1] client ExtentReopen { repair_id: ReconciliationId(595), extent_id: 155 }
Sep 22 23:15:15.266 INFO [2] received reconcile message
Sep 22 23:15:15.266 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(595), op: ExtentReopen { repair_id: ReconciliationId(595), extent_id: 155 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:15.266 INFO [2] client ExtentReopen { repair_id: ReconciliationId(595), extent_id: 155 }
Sep 22 23:15:15.266 DEBG 595 Reopen extent 155
Sep 22 23:15:15.267 DEBG 595 Reopen extent 155
Sep 22 23:15:15.267 DEBG 595 Reopen extent 155
Sep 22 23:15:15.268 DEBG [2] It's time to notify for 595
Sep 22 23:15:15.268 INFO Completion from [2] id:595 status:true
Sep 22 23:15:15.268 INFO [596/752] Repair commands completed
Sep 22 23:15:15.268 INFO Pop front: ReconcileIO { id: ReconciliationId(596), op: ExtentFlush { repair_id: ReconciliationId(596), extent_id: 29, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:15.268 INFO Sent repair work, now wait for resp
Sep 22 23:15:15.268 INFO [0] received reconcile message
Sep 22 23:15:15.268 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(596), op: ExtentFlush { repair_id: ReconciliationId(596), extent_id: 29, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:15.268 INFO [0] client ExtentFlush { repair_id: ReconciliationId(596), extent_id: 29, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:15.268 INFO [1] received reconcile message
Sep 22 23:15:15.268 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(596), op: ExtentFlush { repair_id: ReconciliationId(596), extent_id: 29, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:15.268 INFO [1] client ExtentFlush { repair_id: ReconciliationId(596), extent_id: 29, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:15.268 INFO [2] received reconcile message
Sep 22 23:15:15.268 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(596), op: ExtentFlush { repair_id: ReconciliationId(596), extent_id: 29, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:15.268 INFO [2] client ExtentFlush { repair_id: ReconciliationId(596), extent_id: 29, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:15.268 DEBG 596 Flush extent 29 with f:2 g:2
Sep 22 23:15:15.268 DEBG Flush just extent 29 with f:2 and g:2
Sep 22 23:15:15.268 DEBG [1] It's time to notify for 596
Sep 22 23:15:15.268 INFO Completion from [1] id:596 status:true
Sep 22 23:15:15.268 INFO [597/752] Repair commands completed
Sep 22 23:15:15.268 INFO Pop front: ReconcileIO { id: ReconciliationId(597), op: ExtentClose { repair_id: ReconciliationId(597), extent_id: 29 }, state: ClientData([New, New, New]) }
Sep 22 23:15:15.268 INFO Sent repair work, now wait for resp
Sep 22 23:15:15.268 INFO [0] received reconcile message
Sep 22 23:15:15.268 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(597), op: ExtentClose { repair_id: ReconciliationId(597), extent_id: 29 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:15.268 INFO [0] client ExtentClose { repair_id: ReconciliationId(597), extent_id: 29 }
Sep 22 23:15:15.269 INFO [1] received reconcile message
Sep 22 23:15:15.269 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(597), op: ExtentClose { repair_id: ReconciliationId(597), extent_id: 29 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:15.269 INFO [1] client ExtentClose { repair_id: ReconciliationId(597), extent_id: 29 }
Sep 22 23:15:15.269 INFO [2] received reconcile message
Sep 22 23:15:15.269 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(597), op: ExtentClose { repair_id: ReconciliationId(597), extent_id: 29 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:15.269 INFO [2] client ExtentClose { repair_id: ReconciliationId(597), extent_id: 29 }
Sep 22 23:15:15.269 DEBG 597 Close extent 29
Sep 22 23:15:15.269 DEBG 597 Close extent 29
Sep 22 23:15:15.269 DEBG 597 Close extent 29
Sep 22 23:15:15.270 DEBG [2] It's time to notify for 597
Sep 22 23:15:15.270 INFO Completion from [2] id:597 status:true
Sep 22 23:15:15.270 INFO [598/752] Repair commands completed
Sep 22 23:15:15.270 INFO Pop front: ReconcileIO { id: ReconciliationId(598), op: ExtentRepair { repair_id: ReconciliationId(598), extent_id: 29, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:15.270 INFO Sent repair work, now wait for resp
Sep 22 23:15:15.270 INFO [0] received reconcile message
Sep 22 23:15:15.270 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(598), op: ExtentRepair { repair_id: ReconciliationId(598), extent_id: 29, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:15.270 INFO [0] client ExtentRepair { repair_id: ReconciliationId(598), extent_id: 29, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:15.270 INFO [0] Sending repair request ReconciliationId(598)
Sep 22 23:15:15.270 INFO [1] received reconcile message
Sep 22 23:15:15.270 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(598), op: ExtentRepair { repair_id: ReconciliationId(598), extent_id: 29, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:15.270 INFO [1] client ExtentRepair { repair_id: ReconciliationId(598), extent_id: 29, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:15.270 INFO [1] No action required ReconciliationId(598)
Sep 22 23:15:15.270 INFO [2] received reconcile message
Sep 22 23:15:15.270 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(598), op: ExtentRepair { repair_id: ReconciliationId(598), extent_id: 29, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:15.270 INFO [2] client ExtentRepair { repair_id: ReconciliationId(598), extent_id: 29, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:15.270 INFO [2] No action required ReconciliationId(598)
Sep 22 23:15:15.270 DEBG 598 Repair extent 29 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:15.270 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/01D.copy"
Sep 22 23:15:15.334 INFO accepted connection, remote_addr: 127.0.0.1:35146, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.334 TRCE incoming request, uri: /extent/29/files, method: GET, req_id: cde1244e-813d-4523-8907-5c7413d8771a, remote_addr: 127.0.0.1:35146, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.334 INFO request completed, latency_us: 191, response_code: 200, uri: /extent/29/files, method: GET, req_id: cde1244e-813d-4523-8907-5c7413d8771a, remote_addr: 127.0.0.1:35146, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.335 INFO eid:29 Found repair files: ["01D", "01D.db"]
Sep 22 23:15:15.335 TRCE incoming request, uri: /newextent/29/data, method: GET, req_id: 58004004-69cf-4a23-a165-eef694b6444f, remote_addr: 127.0.0.1:35146, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.335 INFO request completed, latency_us: 304, response_code: 200, uri: /newextent/29/data, method: GET, req_id: 58004004-69cf-4a23-a165-eef694b6444f, remote_addr: 127.0.0.1:35146, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.340 TRCE incoming request, uri: /newextent/29/db, method: GET, req_id: ede2c646-e7d3-4351-8585-5d8ff91f1cde, remote_addr: 127.0.0.1:35146, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.341 INFO request completed, latency_us: 312, response_code: 200, uri: /newextent/29/db, method: GET, req_id: ede2c646-e7d3-4351-8585-5d8ff91f1cde, remote_addr: 127.0.0.1:35146, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.342 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/01D.copy" to "/tmp/downstairs-vrx8aK6L/00/000/01D.replace"
Sep 22 23:15:15.342 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:15.343 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/01D.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:15.343 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/01D"
Sep 22 23:15:15.343 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/01D.db"
Sep 22 23:15:15.343 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:15.343 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/01D.replace" to "/tmp/downstairs-vrx8aK6L/00/000/01D.completed"
Sep 22 23:15:15.343 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:15.343 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:15.344 DEBG [0] It's time to notify for 598
Sep 22 23:15:15.344 INFO Completion from [0] id:598 status:true
Sep 22 23:15:15.344 INFO [599/752] Repair commands completed
Sep 22 23:15:15.344 INFO Pop front: ReconcileIO { id: ReconciliationId(599), op: ExtentReopen { repair_id: ReconciliationId(599), extent_id: 29 }, state: ClientData([New, New, New]) }
Sep 22 23:15:15.344 INFO Sent repair work, now wait for resp
Sep 22 23:15:15.344 INFO [0] received reconcile message
Sep 22 23:15:15.344 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(599), op: ExtentReopen { repair_id: ReconciliationId(599), extent_id: 29 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:15.344 INFO [0] client ExtentReopen { repair_id: ReconciliationId(599), extent_id: 29 }
Sep 22 23:15:15.344 INFO [1] received reconcile message
Sep 22 23:15:15.344 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(599), op: ExtentReopen { repair_id: ReconciliationId(599), extent_id: 29 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:15.344 INFO [1] client ExtentReopen { repair_id: ReconciliationId(599), extent_id: 29 }
Sep 22 23:15:15.344 INFO [2] received reconcile message
Sep 22 23:15:15.344 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(599), op: ExtentReopen { repair_id: ReconciliationId(599), extent_id: 29 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:15.344 INFO [2] client ExtentReopen { repair_id: ReconciliationId(599), extent_id: 29 }
Sep 22 23:15:15.344 DEBG 599 Reopen extent 29
Sep 22 23:15:15.345 DEBG 599 Reopen extent 29
Sep 22 23:15:15.345 DEBG 599 Reopen extent 29
Sep 22 23:15:15.346 DEBG [2] It's time to notify for 599
Sep 22 23:15:15.346 INFO Completion from [2] id:599 status:true
Sep 22 23:15:15.346 INFO [600/752] Repair commands completed
Sep 22 23:15:15.346 INFO Pop front: ReconcileIO { id: ReconciliationId(600), op: ExtentFlush { repair_id: ReconciliationId(600), extent_id: 135, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:15.346 INFO Sent repair work, now wait for resp
Sep 22 23:15:15.346 INFO [0] received reconcile message
Sep 22 23:15:15.346 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(600), op: ExtentFlush { repair_id: ReconciliationId(600), extent_id: 135, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:15.346 INFO [0] client ExtentFlush { repair_id: ReconciliationId(600), extent_id: 135, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:15.346 INFO [1] received reconcile message
Sep 22 23:15:15.346 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(600), op: ExtentFlush { repair_id: ReconciliationId(600), extent_id: 135, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:15.346 INFO [1] client ExtentFlush { repair_id: ReconciliationId(600), extent_id: 135, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:15.346 INFO [2] received reconcile message
Sep 22 23:15:15.346 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(600), op: ExtentFlush { repair_id: ReconciliationId(600), extent_id: 135, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:15.346 INFO [2] client ExtentFlush { repair_id: ReconciliationId(600), extent_id: 135, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:15.346 DEBG 600 Flush extent 135 with f:2 g:2
Sep 22 23:15:15.346 DEBG Flush just extent 135 with f:2 and g:2
Sep 22 23:15:15.347 DEBG [1] It's time to notify for 600
Sep 22 23:15:15.347 INFO Completion from [1] id:600 status:true
Sep 22 23:15:15.347 INFO [601/752] Repair commands completed
Sep 22 23:15:15.347 INFO Pop front: ReconcileIO { id: ReconciliationId(601), op: ExtentClose { repair_id: ReconciliationId(601), extent_id: 135 }, state: ClientData([New, New, New]) }
Sep 22 23:15:15.347 INFO Sent repair work, now wait for resp
Sep 22 23:15:15.347 INFO [0] received reconcile message
Sep 22 23:15:15.347 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(601), op: ExtentClose { repair_id: ReconciliationId(601), extent_id: 135 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:15.347 INFO [0] client ExtentClose { repair_id: ReconciliationId(601), extent_id: 135 }
Sep 22 23:15:15.347 INFO [1] received reconcile message
Sep 22 23:15:15.347 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(601), op: ExtentClose { repair_id: ReconciliationId(601), extent_id: 135 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:15.347 INFO [1] client ExtentClose { repair_id: ReconciliationId(601), extent_id: 135 }
Sep 22 23:15:15.347 INFO [2] received reconcile message
Sep 22 23:15:15.347 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(601), op: ExtentClose { repair_id: ReconciliationId(601), extent_id: 135 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:15.347 INFO [2] client ExtentClose { repair_id: ReconciliationId(601), extent_id: 135 }
Sep 22 23:15:15.347 DEBG 601 Close extent 135
Sep 22 23:15:15.347 DEBG 601 Close extent 135
Sep 22 23:15:15.348 DEBG 601 Close extent 135
Sep 22 23:15:15.348 DEBG [2] It's time to notify for 601
Sep 22 23:15:15.348 INFO Completion from [2] id:601 status:true
Sep 22 23:15:15.348 INFO [602/752] Repair commands completed
Sep 22 23:15:15.348 INFO Pop front: ReconcileIO { id: ReconciliationId(602), op: ExtentRepair { repair_id: ReconciliationId(602), extent_id: 135, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:15.348 INFO Sent repair work, now wait for resp
Sep 22 23:15:15.348 INFO [0] received reconcile message
Sep 22 23:15:15.348 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(602), op: ExtentRepair { repair_id: ReconciliationId(602), extent_id: 135, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:15.348 INFO [0] client ExtentRepair { repair_id: ReconciliationId(602), extent_id: 135, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:15.348 INFO [0] Sending repair request ReconciliationId(602)
Sep 22 23:15:15.348 INFO [1] received reconcile message
Sep 22 23:15:15.348 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(602), op: ExtentRepair { repair_id: ReconciliationId(602), extent_id: 135, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:15.348 INFO [1] client ExtentRepair { repair_id: ReconciliationId(602), extent_id: 135, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:15.348 INFO [1] No action required ReconciliationId(602)
Sep 22 23:15:15.348 INFO [2] received reconcile message
Sep 22 23:15:15.348 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(602), op: ExtentRepair { repair_id: ReconciliationId(602), extent_id: 135, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:15.348 INFO [2] client ExtentRepair { repair_id: ReconciliationId(602), extent_id: 135, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:15.348 INFO [2] No action required ReconciliationId(602)
Sep 22 23:15:15.348 DEBG 602 Repair extent 135 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:15.349 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/087.copy"
Sep 22 23:15:15.358 DEBG [2] Read AckReady 1088, : downstairs
Sep 22 23:15:15.359 DEBG up_ds_listen was notified
Sep 22 23:15:15.359 DEBG up_ds_listen process 1088
Sep 22 23:15:15.359 DEBG [A] ack job 1088:89, : downstairs
Sep 22 23:15:15.413 INFO accepted connection, remote_addr: 127.0.0.1:63153, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.413 TRCE incoming request, uri: /extent/135/files, method: GET, req_id: fef095b1-80c5-4f78-b8e6-71fa6fbcf3ca, remote_addr: 127.0.0.1:63153, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.413 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:15.413 INFO request completed, latency_us: 202, response_code: 200, uri: /extent/135/files, method: GET, req_id: fef095b1-80c5-4f78-b8e6-71fa6fbcf3ca, remote_addr: 127.0.0.1:63153, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.414 INFO eid:135 Found repair files: ["087", "087.db"]
Sep 22 23:15:15.414 TRCE incoming request, uri: /newextent/135/data, method: GET, req_id: dffa5365-7e2b-4ec5-931d-79d7f2a8d759, remote_addr: 127.0.0.1:63153, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.414 INFO request completed, latency_us: 333, response_code: 200, uri: /newextent/135/data, method: GET, req_id: dffa5365-7e2b-4ec5-931d-79d7f2a8d759, remote_addr: 127.0.0.1:63153, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.416 DEBG Flush     :1089 extent_limit None deps:[JobId(1088), JobId(1087)] res:true f:34 g:1
Sep 22 23:15:15.416 INFO [lossy] sleeping 1 second
Sep 22 23:15:15.416 INFO [lossy] skipping 1087
Sep 22 23:15:15.416 INFO [lossy] skipping 1087
Sep 22 23:15:15.416 INFO [lossy] skipping 1087
Sep 22 23:15:15.416 DEBG Flush     :1087 extent_limit None deps:[JobId(1086), JobId(1085)] res:true f:33 g:1
Sep 22 23:15:15.419 TRCE incoming request, uri: /newextent/135/db, method: GET, req_id: 035f5b90-205f-4cf9-9124-20a8c81e3a47, remote_addr: 127.0.0.1:63153, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.419 INFO request completed, latency_us: 303, response_code: 200, uri: /newextent/135/db, method: GET, req_id: 035f5b90-205f-4cf9-9124-20a8c81e3a47, remote_addr: 127.0.0.1:63153, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.420 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/087.copy" to "/tmp/downstairs-vrx8aK6L/00/000/087.replace"
Sep 22 23:15:15.420 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:15.421 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/087.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:15.422 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/087"
Sep 22 23:15:15.422 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/087.db"
Sep 22 23:15:15.422 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:15.422 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/087.replace" to "/tmp/downstairs-vrx8aK6L/00/000/087.completed"
Sep 22 23:15:15.422 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:15.422 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:15.422 DEBG [0] It's time to notify for 602
Sep 22 23:15:15.422 INFO Completion from [0] id:602 status:true
Sep 22 23:15:15.422 INFO [603/752] Repair commands completed
Sep 22 23:15:15.422 DEBG Read      :1088 deps:[JobId(1087)] res:true
Sep 22 23:15:15.422 INFO Pop front: ReconcileIO { id: ReconciliationId(603), op: ExtentReopen { repair_id: ReconciliationId(603), extent_id: 135 }, state: ClientData([New, New, New]) }
Sep 22 23:15:15.422 INFO Sent repair work, now wait for resp
Sep 22 23:15:15.422 INFO [0] received reconcile message
Sep 22 23:15:15.422 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(603), op: ExtentReopen { repair_id: ReconciliationId(603), extent_id: 135 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:15.422 INFO [0] client ExtentReopen { repair_id: ReconciliationId(603), extent_id: 135 }
Sep 22 23:15:15.422 INFO [1] received reconcile message
Sep 22 23:15:15.422 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(603), op: ExtentReopen { repair_id: ReconciliationId(603), extent_id: 135 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:15.422 INFO [1] client ExtentReopen { repair_id: ReconciliationId(603), extent_id: 135 }
Sep 22 23:15:15.422 INFO [2] received reconcile message
Sep 22 23:15:15.422 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(603), op: ExtentReopen { repair_id: ReconciliationId(603), extent_id: 135 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:15.422 INFO [2] client ExtentReopen { repair_id: ReconciliationId(603), extent_id: 135 }
Sep 22 23:15:15.423 DEBG 603 Reopen extent 135
Sep 22 23:15:15.423 DEBG 603 Reopen extent 135
Sep 22 23:15:15.424 DEBG 603 Reopen extent 135
Sep 22 23:15:15.424 DEBG [2] It's time to notify for 603
Sep 22 23:15:15.424 INFO Completion from [2] id:603 status:true
Sep 22 23:15:15.424 INFO [604/752] Repair commands completed
Sep 22 23:15:15.424 INFO Pop front: ReconcileIO { id: ReconciliationId(604), op: ExtentFlush { repair_id: ReconciliationId(604), extent_id: 161, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:15.424 INFO Sent repair work, now wait for resp
Sep 22 23:15:15.424 INFO [0] received reconcile message
Sep 22 23:15:15.424 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(604), op: ExtentFlush { repair_id: ReconciliationId(604), extent_id: 161, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:15.425 INFO [0] client ExtentFlush { repair_id: ReconciliationId(604), extent_id: 161, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:15.425 INFO [1] received reconcile message
Sep 22 23:15:15.425 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(604), op: ExtentFlush { repair_id: ReconciliationId(604), extent_id: 161, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:15.425 INFO [1] client ExtentFlush { repair_id: ReconciliationId(604), extent_id: 161, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:15.425 INFO [2] received reconcile message
Sep 22 23:15:15.425 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(604), op: ExtentFlush { repair_id: ReconciliationId(604), extent_id: 161, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:15.425 INFO [2] client ExtentFlush { repair_id: ReconciliationId(604), extent_id: 161, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:15.425 DEBG 604 Flush extent 161 with f:2 g:2
Sep 22 23:15:15.425 DEBG Flush just extent 161 with f:2 and g:2
Sep 22 23:15:15.425 DEBG [1] It's time to notify for 604
Sep 22 23:15:15.425 INFO Completion from [1] id:604 status:true
Sep 22 23:15:15.425 INFO [605/752] Repair commands completed
Sep 22 23:15:15.425 INFO Pop front: ReconcileIO { id: ReconciliationId(605), op: ExtentClose { repair_id: ReconciliationId(605), extent_id: 161 }, state: ClientData([New, New, New]) }
Sep 22 23:15:15.425 INFO Sent repair work, now wait for resp
Sep 22 23:15:15.425 INFO [0] received reconcile message
Sep 22 23:15:15.425 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(605), op: ExtentClose { repair_id: ReconciliationId(605), extent_id: 161 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:15.425 INFO [0] client ExtentClose { repair_id: ReconciliationId(605), extent_id: 161 }
Sep 22 23:15:15.425 INFO [1] received reconcile message
Sep 22 23:15:15.425 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(605), op: ExtentClose { repair_id: ReconciliationId(605), extent_id: 161 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:15.425 INFO [1] client ExtentClose { repair_id: ReconciliationId(605), extent_id: 161 }
Sep 22 23:15:15.425 INFO [2] received reconcile message
Sep 22 23:15:15.425 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(605), op: ExtentClose { repair_id: ReconciliationId(605), extent_id: 161 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:15.425 INFO [2] client ExtentClose { repair_id: ReconciliationId(605), extent_id: 161 }
Sep 22 23:15:15.425 DEBG 605 Close extent 161
Sep 22 23:15:15.426 DEBG 605 Close extent 161
Sep 22 23:15:15.426 DEBG 605 Close extent 161
Sep 22 23:15:15.426 DEBG [2] It's time to notify for 605
Sep 22 23:15:15.426 INFO Completion from [2] id:605 status:true
Sep 22 23:15:15.426 INFO [606/752] Repair commands completed
Sep 22 23:15:15.426 INFO Pop front: ReconcileIO { id: ReconciliationId(606), op: ExtentRepair { repair_id: ReconciliationId(606), extent_id: 161, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:15.426 INFO Sent repair work, now wait for resp
Sep 22 23:15:15.427 INFO [0] received reconcile message
Sep 22 23:15:15.427 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(606), op: ExtentRepair { repair_id: ReconciliationId(606), extent_id: 161, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:15.427 INFO [0] client ExtentRepair { repair_id: ReconciliationId(606), extent_id: 161, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:15.427 INFO [0] Sending repair request ReconciliationId(606)
Sep 22 23:15:15.427 INFO [1] received reconcile message
Sep 22 23:15:15.427 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(606), op: ExtentRepair { repair_id: ReconciliationId(606), extent_id: 161, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:15.427 INFO [1] client ExtentRepair { repair_id: ReconciliationId(606), extent_id: 161, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:15.427 INFO [1] No action required ReconciliationId(606)
Sep 22 23:15:15.427 INFO [2] received reconcile message
Sep 22 23:15:15.427 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(606), op: ExtentRepair { repair_id: ReconciliationId(606), extent_id: 161, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:15.427 INFO [2] client ExtentRepair { repair_id: ReconciliationId(606), extent_id: 161, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:15.427 INFO [2] No action required ReconciliationId(606)
Sep 22 23:15:15.427 DEBG 606 Repair extent 161 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:15.427 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/0A1.copy"
Sep 22 23:15:15.489 INFO accepted connection, remote_addr: 127.0.0.1:60071, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.490 TRCE incoming request, uri: /extent/161/files, method: GET, req_id: 30a633ed-b022-4155-aa83-1be9a30a7a81, remote_addr: 127.0.0.1:60071, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.490 INFO request completed, latency_us: 218, response_code: 200, uri: /extent/161/files, method: GET, req_id: 30a633ed-b022-4155-aa83-1be9a30a7a81, remote_addr: 127.0.0.1:60071, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.490 INFO eid:161 Found repair files: ["0A1", "0A1.db"]
Sep 22 23:15:15.491 TRCE incoming request, uri: /newextent/161/data, method: GET, req_id: 0c23dab9-0102-476b-a2dd-3de7732747f2, remote_addr: 127.0.0.1:60071, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.491 INFO request completed, latency_us: 268, response_code: 200, uri: /newextent/161/data, method: GET, req_id: 0c23dab9-0102-476b-a2dd-3de7732747f2, remote_addr: 127.0.0.1:60071, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.495 DEBG IO Read  1090 has deps [JobId(1089)]
Sep 22 23:15:15.495 DEBG [rc] retire 1087 clears [JobId(1086), JobId(1087)], : downstairs
Sep 22 23:15:15.496 TRCE incoming request, uri: /newextent/161/db, method: GET, req_id: 21a0b6f5-827d-40dd-9f43-57d455f47741, remote_addr: 127.0.0.1:60071, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.496 INFO request completed, latency_us: 351, response_code: 200, uri: /newextent/161/db, method: GET, req_id: 21a0b6f5-827d-40dd-9f43-57d455f47741, remote_addr: 127.0.0.1:60071, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.498 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/0A1.copy" to "/tmp/downstairs-vrx8aK6L/00/000/0A1.replace"
Sep 22 23:15:15.498 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:15.499 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/0A1.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:15.499 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/0A1"
Sep 22 23:15:15.499 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/0A1.db"
Sep 22 23:15:15.499 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:15.499 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/0A1.replace" to "/tmp/downstairs-vrx8aK6L/00/000/0A1.completed"
Sep 22 23:15:15.500 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:15.500 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:15.500 DEBG [0] It's time to notify for 606
Sep 22 23:15:15.500 INFO Completion from [0] id:606 status:true
Sep 22 23:15:15.500 INFO [607/752] Repair commands completed
Sep 22 23:15:15.500 INFO Pop front: ReconcileIO { id: ReconciliationId(607), op: ExtentReopen { repair_id: ReconciliationId(607), extent_id: 161 }, state: ClientData([New, New, New]) }
Sep 22 23:15:15.500 INFO Sent repair work, now wait for resp
Sep 22 23:15:15.500 INFO [0] received reconcile message
Sep 22 23:15:15.500 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(607), op: ExtentReopen { repair_id: ReconciliationId(607), extent_id: 161 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:15.500 INFO [0] client ExtentReopen { repair_id: ReconciliationId(607), extent_id: 161 }
Sep 22 23:15:15.500 INFO [1] received reconcile message
Sep 22 23:15:15.500 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(607), op: ExtentReopen { repair_id: ReconciliationId(607), extent_id: 161 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:15.500 INFO [1] client ExtentReopen { repair_id: ReconciliationId(607), extent_id: 161 }
Sep 22 23:15:15.500 INFO [2] received reconcile message
Sep 22 23:15:15.500 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(607), op: ExtentReopen { repair_id: ReconciliationId(607), extent_id: 161 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:15.500 INFO [2] client ExtentReopen { repair_id: ReconciliationId(607), extent_id: 161 }
Sep 22 23:15:15.501 DEBG 607 Reopen extent 161
Sep 22 23:15:15.502 DEBG 607 Reopen extent 161
Sep 22 23:15:15.502 DEBG 607 Reopen extent 161
Sep 22 23:15:15.503 DEBG [2] It's time to notify for 607
Sep 22 23:15:15.503 INFO Completion from [2] id:607 status:true
Sep 22 23:15:15.503 INFO [608/752] Repair commands completed
Sep 22 23:15:15.503 INFO Pop front: ReconcileIO { id: ReconciliationId(608), op: ExtentFlush { repair_id: ReconciliationId(608), extent_id: 15, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:15.503 INFO Sent repair work, now wait for resp
Sep 22 23:15:15.503 INFO [0] received reconcile message
Sep 22 23:15:15.503 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(608), op: ExtentFlush { repair_id: ReconciliationId(608), extent_id: 15, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:15.503 INFO [0] client ExtentFlush { repair_id: ReconciliationId(608), extent_id: 15, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:15.503 INFO [1] received reconcile message
Sep 22 23:15:15.503 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(608), op: ExtentFlush { repair_id: ReconciliationId(608), extent_id: 15, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:15.503 INFO [1] client ExtentFlush { repair_id: ReconciliationId(608), extent_id: 15, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:15.503 INFO [2] received reconcile message
Sep 22 23:15:15.503 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(608), op: ExtentFlush { repair_id: ReconciliationId(608), extent_id: 15, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:15.503 INFO [2] client ExtentFlush { repair_id: ReconciliationId(608), extent_id: 15, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:15.504 DEBG 608 Flush extent 15 with f:2 g:2
Sep 22 23:15:15.504 DEBG Flush just extent 15 with f:2 and g:2
Sep 22 23:15:15.504 DEBG [1] It's time to notify for 608
Sep 22 23:15:15.504 INFO Completion from [1] id:608 status:true
Sep 22 23:15:15.504 INFO [609/752] Repair commands completed
Sep 22 23:15:15.504 INFO Pop front: ReconcileIO { id: ReconciliationId(609), op: ExtentClose { repair_id: ReconciliationId(609), extent_id: 15 }, state: ClientData([New, New, New]) }
Sep 22 23:15:15.504 INFO Sent repair work, now wait for resp
Sep 22 23:15:15.504 INFO [0] received reconcile message
Sep 22 23:15:15.504 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(609), op: ExtentClose { repair_id: ReconciliationId(609), extent_id: 15 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:15.504 INFO [0] client ExtentClose { repair_id: ReconciliationId(609), extent_id: 15 }
Sep 22 23:15:15.504 INFO [1] received reconcile message
Sep 22 23:15:15.504 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(609), op: ExtentClose { repair_id: ReconciliationId(609), extent_id: 15 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:15.504 INFO [1] client ExtentClose { repair_id: ReconciliationId(609), extent_id: 15 }
Sep 22 23:15:15.504 INFO [2] received reconcile message
Sep 22 23:15:15.504 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(609), op: ExtentClose { repair_id: ReconciliationId(609), extent_id: 15 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:15.504 INFO [2] client ExtentClose { repair_id: ReconciliationId(609), extent_id: 15 }
Sep 22 23:15:15.504 DEBG 609 Close extent 15
Sep 22 23:15:15.505 DEBG 609 Close extent 15
Sep 22 23:15:15.505 DEBG 609 Close extent 15
Sep 22 23:15:15.506 DEBG [2] It's time to notify for 609
Sep 22 23:15:15.506 INFO Completion from [2] id:609 status:true
Sep 22 23:15:15.506 INFO [610/752] Repair commands completed
Sep 22 23:15:15.506 INFO Pop front: ReconcileIO { id: ReconciliationId(610), op: ExtentRepair { repair_id: ReconciliationId(610), extent_id: 15, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:15.506 INFO Sent repair work, now wait for resp
Sep 22 23:15:15.506 INFO [0] received reconcile message
Sep 22 23:15:15.506 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(610), op: ExtentRepair { repair_id: ReconciliationId(610), extent_id: 15, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:15.506 INFO [0] client ExtentRepair { repair_id: ReconciliationId(610), extent_id: 15, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:15.506 INFO [0] Sending repair request ReconciliationId(610)
Sep 22 23:15:15.506 INFO [1] received reconcile message
Sep 22 23:15:15.506 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(610), op: ExtentRepair { repair_id: ReconciliationId(610), extent_id: 15, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:15.506 INFO [1] client ExtentRepair { repair_id: ReconciliationId(610), extent_id: 15, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:15.506 INFO [1] No action required ReconciliationId(610)
Sep 22 23:15:15.506 INFO [2] received reconcile message
Sep 22 23:15:15.506 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(610), op: ExtentRepair { repair_id: ReconciliationId(610), extent_id: 15, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:15.506 INFO [2] client ExtentRepair { repair_id: ReconciliationId(610), extent_id: 15, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:15.506 INFO [2] No action required ReconciliationId(610)
Sep 22 23:15:15.506 DEBG 610 Repair extent 15 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:15.506 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/00F.copy"
Sep 22 23:15:15.571 INFO accepted connection, remote_addr: 127.0.0.1:61878, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.571 TRCE incoming request, uri: /extent/15/files, method: GET, req_id: 259d5bcf-dd3a-4d7e-9b39-d67a93ef0430, remote_addr: 127.0.0.1:61878, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.571 INFO request completed, latency_us: 196, response_code: 200, uri: /extent/15/files, method: GET, req_id: 259d5bcf-dd3a-4d7e-9b39-d67a93ef0430, remote_addr: 127.0.0.1:61878, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.572 INFO eid:15 Found repair files: ["00F", "00F.db"]
Sep 22 23:15:15.572 TRCE incoming request, uri: /newextent/15/data, method: GET, req_id: c9192a78-4dd4-43a5-a0db-ee45628e0d94, remote_addr: 127.0.0.1:61878, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.572 INFO request completed, latency_us: 329, response_code: 200, uri: /newextent/15/data, method: GET, req_id: c9192a78-4dd4-43a5-a0db-ee45628e0d94, remote_addr: 127.0.0.1:61878, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.578 TRCE incoming request, uri: /newextent/15/db, method: GET, req_id: 59e3543d-a99b-4d8f-b64c-8643df5d758f, remote_addr: 127.0.0.1:61878, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.578 INFO request completed, latency_us: 300, response_code: 200, uri: /newextent/15/db, method: GET, req_id: 59e3543d-a99b-4d8f-b64c-8643df5d758f, remote_addr: 127.0.0.1:61878, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.579 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/00F.copy" to "/tmp/downstairs-vrx8aK6L/00/000/00F.replace"
Sep 22 23:15:15.579 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:15.580 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/00F.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:15.580 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/00F"
Sep 22 23:15:15.580 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/00F.db"
Sep 22 23:15:15.580 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:15.580 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/00F.replace" to "/tmp/downstairs-vrx8aK6L/00/000/00F.completed"
Sep 22 23:15:15.580 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:15.580 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:15.580 DEBG [0] It's time to notify for 610
Sep 22 23:15:15.581 INFO Completion from [0] id:610 status:true
Sep 22 23:15:15.581 INFO [611/752] Repair commands completed
Sep 22 23:15:15.581 INFO Pop front: ReconcileIO { id: ReconciliationId(611), op: ExtentReopen { repair_id: ReconciliationId(611), extent_id: 15 }, state: ClientData([New, New, New]) }
Sep 22 23:15:15.581 INFO Sent repair work, now wait for resp
Sep 22 23:15:15.581 INFO [0] received reconcile message
Sep 22 23:15:15.581 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(611), op: ExtentReopen { repair_id: ReconciliationId(611), extent_id: 15 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:15.581 INFO [0] client ExtentReopen { repair_id: ReconciliationId(611), extent_id: 15 }
Sep 22 23:15:15.581 INFO [1] received reconcile message
Sep 22 23:15:15.581 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(611), op: ExtentReopen { repair_id: ReconciliationId(611), extent_id: 15 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:15.581 INFO [1] client ExtentReopen { repair_id: ReconciliationId(611), extent_id: 15 }
Sep 22 23:15:15.581 INFO [2] received reconcile message
Sep 22 23:15:15.581 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(611), op: ExtentReopen { repair_id: ReconciliationId(611), extent_id: 15 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:15.581 INFO [2] client ExtentReopen { repair_id: ReconciliationId(611), extent_id: 15 }
Sep 22 23:15:15.581 DEBG 611 Reopen extent 15
Sep 22 23:15:15.582 DEBG 611 Reopen extent 15
Sep 22 23:15:15.582 DEBG 611 Reopen extent 15
Sep 22 23:15:15.583 DEBG [2] It's time to notify for 611
Sep 22 23:15:15.583 INFO Completion from [2] id:611 status:true
Sep 22 23:15:15.583 INFO [612/752] Repair commands completed
Sep 22 23:15:15.583 INFO Pop front: ReconcileIO { id: ReconciliationId(612), op: ExtentFlush { repair_id: ReconciliationId(612), extent_id: 64, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:15.583 INFO Sent repair work, now wait for resp
Sep 22 23:15:15.583 INFO [0] received reconcile message
Sep 22 23:15:15.583 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(612), op: ExtentFlush { repair_id: ReconciliationId(612), extent_id: 64, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:15.583 INFO [0] client ExtentFlush { repair_id: ReconciliationId(612), extent_id: 64, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:15.583 INFO [1] received reconcile message
Sep 22 23:15:15.583 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(612), op: ExtentFlush { repair_id: ReconciliationId(612), extent_id: 64, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:15.583 INFO [1] client ExtentFlush { repair_id: ReconciliationId(612), extent_id: 64, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:15.583 INFO [2] received reconcile message
Sep 22 23:15:15.583 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(612), op: ExtentFlush { repair_id: ReconciliationId(612), extent_id: 64, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:15.583 INFO [2] client ExtentFlush { repair_id: ReconciliationId(612), extent_id: 64, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:15.583 DEBG 612 Flush extent 64 with f:2 g:2
Sep 22 23:15:15.583 DEBG Flush just extent 64 with f:2 and g:2
Sep 22 23:15:15.583 DEBG [1] It's time to notify for 612
Sep 22 23:15:15.583 INFO Completion from [1] id:612 status:true
Sep 22 23:15:15.583 INFO [613/752] Repair commands completed
Sep 22 23:15:15.583 INFO Pop front: ReconcileIO { id: ReconciliationId(613), op: ExtentClose { repair_id: ReconciliationId(613), extent_id: 64 }, state: ClientData([New, New, New]) }
Sep 22 23:15:15.583 INFO Sent repair work, now wait for resp
Sep 22 23:15:15.583 INFO [0] received reconcile message
Sep 22 23:15:15.583 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(613), op: ExtentClose { repair_id: ReconciliationId(613), extent_id: 64 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:15.584 INFO [0] client ExtentClose { repair_id: ReconciliationId(613), extent_id: 64 }
Sep 22 23:15:15.584 INFO [1] received reconcile message
Sep 22 23:15:15.584 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(613), op: ExtentClose { repair_id: ReconciliationId(613), extent_id: 64 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:15.584 INFO [1] client ExtentClose { repair_id: ReconciliationId(613), extent_id: 64 }
Sep 22 23:15:15.584 INFO [2] received reconcile message
Sep 22 23:15:15.584 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(613), op: ExtentClose { repair_id: ReconciliationId(613), extent_id: 64 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:15.584 INFO [2] client ExtentClose { repair_id: ReconciliationId(613), extent_id: 64 }
Sep 22 23:15:15.584 DEBG 613 Close extent 64
Sep 22 23:15:15.584 DEBG 613 Close extent 64
Sep 22 23:15:15.584 DEBG 613 Close extent 64
Sep 22 23:15:15.585 DEBG [2] It's time to notify for 613
Sep 22 23:15:15.585 INFO Completion from [2] id:613 status:true
Sep 22 23:15:15.585 INFO [614/752] Repair commands completed
Sep 22 23:15:15.585 INFO Pop front: ReconcileIO { id: ReconciliationId(614), op: ExtentRepair { repair_id: ReconciliationId(614), extent_id: 64, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:15.585 INFO Sent repair work, now wait for resp
Sep 22 23:15:15.585 INFO [0] received reconcile message
Sep 22 23:15:15.585 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(614), op: ExtentRepair { repair_id: ReconciliationId(614), extent_id: 64, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:15.585 INFO [0] client ExtentRepair { repair_id: ReconciliationId(614), extent_id: 64, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:15.585 INFO [0] Sending repair request ReconciliationId(614)
Sep 22 23:15:15.585 INFO [1] received reconcile message
Sep 22 23:15:15.585 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(614), op: ExtentRepair { repair_id: ReconciliationId(614), extent_id: 64, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:15.585 INFO [1] client ExtentRepair { repair_id: ReconciliationId(614), extent_id: 64, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:15.585 INFO [1] No action required ReconciliationId(614)
Sep 22 23:15:15.585 INFO [2] received reconcile message
Sep 22 23:15:15.585 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(614), op: ExtentRepair { repair_id: ReconciliationId(614), extent_id: 64, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:15.585 INFO [2] client ExtentRepair { repair_id: ReconciliationId(614), extent_id: 64, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:15.585 INFO [2] No action required ReconciliationId(614)
Sep 22 23:15:15.585 DEBG 614 Repair extent 64 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:15.585 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/040.copy"
Sep 22 23:15:15.647 INFO accepted connection, remote_addr: 127.0.0.1:42578, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.647 TRCE incoming request, uri: /extent/64/files, method: GET, req_id: d1e32d03-3423-4eb9-ae75-0d3e7dc09d33, remote_addr: 127.0.0.1:42578, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.647 INFO request completed, latency_us: 209, response_code: 200, uri: /extent/64/files, method: GET, req_id: d1e32d03-3423-4eb9-ae75-0d3e7dc09d33, remote_addr: 127.0.0.1:42578, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.647 INFO eid:64 Found repair files: ["040", "040.db"]
Sep 22 23:15:15.648 TRCE incoming request, uri: /newextent/64/data, method: GET, req_id: 1f28e159-cf2a-4977-9450-44b1074010ac, remote_addr: 127.0.0.1:42578, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.648 INFO request completed, latency_us: 249, response_code: 200, uri: /newextent/64/data, method: GET, req_id: 1f28e159-cf2a-4977-9450-44b1074010ac, remote_addr: 127.0.0.1:42578, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.653 TRCE incoming request, uri: /newextent/64/db, method: GET, req_id: 3881377d-e802-46fa-b4c5-0ac67d4095c2, remote_addr: 127.0.0.1:42578, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.653 INFO request completed, latency_us: 293, response_code: 200, uri: /newextent/64/db, method: GET, req_id: 3881377d-e802-46fa-b4c5-0ac67d4095c2, remote_addr: 127.0.0.1:42578, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.654 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/040.copy" to "/tmp/downstairs-vrx8aK6L/00/000/040.replace"
Sep 22 23:15:15.654 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:15.655 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/040.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:15.655 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/040"
Sep 22 23:15:15.656 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/040.db"
Sep 22 23:15:15.656 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:15.656 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/040.replace" to "/tmp/downstairs-vrx8aK6L/00/000/040.completed"
Sep 22 23:15:15.656 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:15.656 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:15.656 DEBG [0] It's time to notify for 614
Sep 22 23:15:15.656 INFO Completion from [0] id:614 status:true
Sep 22 23:15:15.656 INFO [615/752] Repair commands completed
Sep 22 23:15:15.656 INFO Pop front: ReconcileIO { id: ReconciliationId(615), op: ExtentReopen { repair_id: ReconciliationId(615), extent_id: 64 }, state: ClientData([New, New, New]) }
Sep 22 23:15:15.656 INFO Sent repair work, now wait for resp
Sep 22 23:15:15.656 INFO [0] received reconcile message
Sep 22 23:15:15.656 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(615), op: ExtentReopen { repair_id: ReconciliationId(615), extent_id: 64 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:15.656 INFO [0] client ExtentReopen { repair_id: ReconciliationId(615), extent_id: 64 }
Sep 22 23:15:15.656 INFO [1] received reconcile message
Sep 22 23:15:15.656 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(615), op: ExtentReopen { repair_id: ReconciliationId(615), extent_id: 64 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:15.656 INFO [1] client ExtentReopen { repair_id: ReconciliationId(615), extent_id: 64 }
Sep 22 23:15:15.656 INFO [2] received reconcile message
Sep 22 23:15:15.656 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(615), op: ExtentReopen { repair_id: ReconciliationId(615), extent_id: 64 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:15.656 INFO [2] client ExtentReopen { repair_id: ReconciliationId(615), extent_id: 64 }
Sep 22 23:15:15.656 DEBG 615 Reopen extent 64
Sep 22 23:15:15.657 DEBG 615 Reopen extent 64
Sep 22 23:15:15.658 DEBG 615 Reopen extent 64
Sep 22 23:15:15.658 DEBG [2] It's time to notify for 615
Sep 22 23:15:15.658 INFO Completion from [2] id:615 status:true
Sep 22 23:15:15.658 INFO [616/752] Repair commands completed
Sep 22 23:15:15.658 INFO Pop front: ReconcileIO { id: ReconciliationId(616), op: ExtentFlush { repair_id: ReconciliationId(616), extent_id: 93, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:15.658 INFO Sent repair work, now wait for resp
Sep 22 23:15:15.658 INFO [0] received reconcile message
Sep 22 23:15:15.658 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(616), op: ExtentFlush { repair_id: ReconciliationId(616), extent_id: 93, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:15.658 INFO [0] client ExtentFlush { repair_id: ReconciliationId(616), extent_id: 93, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:15.658 INFO [1] received reconcile message
Sep 22 23:15:15.658 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(616), op: ExtentFlush { repair_id: ReconciliationId(616), extent_id: 93, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:15.658 INFO [1] client ExtentFlush { repair_id: ReconciliationId(616), extent_id: 93, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:15.658 INFO [2] received reconcile message
Sep 22 23:15:15.658 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(616), op: ExtentFlush { repair_id: ReconciliationId(616), extent_id: 93, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:15.658 INFO [2] client ExtentFlush { repair_id: ReconciliationId(616), extent_id: 93, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:15.659 DEBG 616 Flush extent 93 with f:2 g:2
Sep 22 23:15:15.659 DEBG Flush just extent 93 with f:2 and g:2
Sep 22 23:15:15.659 DEBG [1] It's time to notify for 616
Sep 22 23:15:15.659 INFO Completion from [1] id:616 status:true
Sep 22 23:15:15.659 INFO [617/752] Repair commands completed
Sep 22 23:15:15.659 INFO Pop front: ReconcileIO { id: ReconciliationId(617), op: ExtentClose { repair_id: ReconciliationId(617), extent_id: 93 }, state: ClientData([New, New, New]) }
Sep 22 23:15:15.659 INFO Sent repair work, now wait for resp
Sep 22 23:15:15.659 INFO [0] received reconcile message
Sep 22 23:15:15.659 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(617), op: ExtentClose { repair_id: ReconciliationId(617), extent_id: 93 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:15.659 INFO [0] client ExtentClose { repair_id: ReconciliationId(617), extent_id: 93 }
Sep 22 23:15:15.659 INFO [1] received reconcile message
Sep 22 23:15:15.659 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(617), op: ExtentClose { repair_id: ReconciliationId(617), extent_id: 93 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:15.659 INFO [1] client ExtentClose { repair_id: ReconciliationId(617), extent_id: 93 }
Sep 22 23:15:15.659 INFO [2] received reconcile message
Sep 22 23:15:15.659 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(617), op: ExtentClose { repair_id: ReconciliationId(617), extent_id: 93 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:15.659 INFO [2] client ExtentClose { repair_id: ReconciliationId(617), extent_id: 93 }
Sep 22 23:15:15.659 DEBG 617 Close extent 93
Sep 22 23:15:15.659 DEBG 617 Close extent 93
Sep 22 23:15:15.660 DEBG 617 Close extent 93
Sep 22 23:15:15.660 DEBG [2] It's time to notify for 617
Sep 22 23:15:15.660 INFO Completion from [2] id:617 status:true
Sep 22 23:15:15.660 INFO [618/752] Repair commands completed
Sep 22 23:15:15.660 INFO Pop front: ReconcileIO { id: ReconciliationId(618), op: ExtentRepair { repair_id: ReconciliationId(618), extent_id: 93, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:15.660 INFO Sent repair work, now wait for resp
Sep 22 23:15:15.660 INFO [0] received reconcile message
Sep 22 23:15:15.660 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(618), op: ExtentRepair { repair_id: ReconciliationId(618), extent_id: 93, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:15.660 INFO [0] client ExtentRepair { repair_id: ReconciliationId(618), extent_id: 93, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:15.660 INFO [0] Sending repair request ReconciliationId(618)
Sep 22 23:15:15.660 INFO [1] received reconcile message
Sep 22 23:15:15.660 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(618), op: ExtentRepair { repair_id: ReconciliationId(618), extent_id: 93, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:15.660 INFO [1] client ExtentRepair { repair_id: ReconciliationId(618), extent_id: 93, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:15.660 INFO [1] No action required ReconciliationId(618)
Sep 22 23:15:15.660 INFO [2] received reconcile message
Sep 22 23:15:15.660 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(618), op: ExtentRepair { repair_id: ReconciliationId(618), extent_id: 93, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:15.660 INFO [2] client ExtentRepair { repair_id: ReconciliationId(618), extent_id: 93, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:15.661 INFO [2] No action required ReconciliationId(618)
Sep 22 23:15:15.661 DEBG 618 Repair extent 93 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:15.661 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/05D.copy"
Sep 22 23:15:15.722 INFO accepted connection, remote_addr: 127.0.0.1:65206, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.722 TRCE incoming request, uri: /extent/93/files, method: GET, req_id: 54a4a2dd-1cc0-4e02-93b4-56566c1bf8b8, remote_addr: 127.0.0.1:65206, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.722 INFO request completed, latency_us: 190, response_code: 200, uri: /extent/93/files, method: GET, req_id: 54a4a2dd-1cc0-4e02-93b4-56566c1bf8b8, remote_addr: 127.0.0.1:65206, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.723 INFO eid:93 Found repair files: ["05D", "05D.db"]
Sep 22 23:15:15.723 TRCE incoming request, uri: /newextent/93/data, method: GET, req_id: 3943d24b-41ef-46a1-a230-f5e75478792c, remote_addr: 127.0.0.1:65206, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.723 INFO request completed, latency_us: 314, response_code: 200, uri: /newextent/93/data, method: GET, req_id: 3943d24b-41ef-46a1-a230-f5e75478792c, remote_addr: 127.0.0.1:65206, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.728 TRCE incoming request, uri: /newextent/93/db, method: GET, req_id: 424de2ab-7145-4ec1-adb1-c353320a7b56, remote_addr: 127.0.0.1:65206, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.729 INFO request completed, latency_us: 291, response_code: 200, uri: /newextent/93/db, method: GET, req_id: 424de2ab-7145-4ec1-adb1-c353320a7b56, remote_addr: 127.0.0.1:65206, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.730 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/05D.copy" to "/tmp/downstairs-vrx8aK6L/00/000/05D.replace"
Sep 22 23:15:15.730 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:15.731 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/05D.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:15.731 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/05D"
Sep 22 23:15:15.731 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/05D.db"
Sep 22 23:15:15.731 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:15.731 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/05D.replace" to "/tmp/downstairs-vrx8aK6L/00/000/05D.completed"
Sep 22 23:15:15.731 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:15.731 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:15.731 DEBG [0] It's time to notify for 618
Sep 22 23:15:15.731 INFO Completion from [0] id:618 status:true
Sep 22 23:15:15.731 INFO [619/752] Repair commands completed
Sep 22 23:15:15.731 INFO Pop front: ReconcileIO { id: ReconciliationId(619), op: ExtentReopen { repair_id: ReconciliationId(619), extent_id: 93 }, state: ClientData([New, New, New]) }
Sep 22 23:15:15.731 INFO Sent repair work, now wait for resp
Sep 22 23:15:15.731 INFO [0] received reconcile message
Sep 22 23:15:15.731 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(619), op: ExtentReopen { repair_id: ReconciliationId(619), extent_id: 93 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:15.731 INFO [0] client ExtentReopen { repair_id: ReconciliationId(619), extent_id: 93 }
Sep 22 23:15:15.732 INFO [1] received reconcile message
Sep 22 23:15:15.732 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(619), op: ExtentReopen { repair_id: ReconciliationId(619), extent_id: 93 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:15.732 INFO [1] client ExtentReopen { repair_id: ReconciliationId(619), extent_id: 93 }
Sep 22 23:15:15.732 INFO [2] received reconcile message
Sep 22 23:15:15.732 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(619), op: ExtentReopen { repair_id: ReconciliationId(619), extent_id: 93 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:15.732 INFO [2] client ExtentReopen { repair_id: ReconciliationId(619), extent_id: 93 }
Sep 22 23:15:15.732 DEBG 619 Reopen extent 93
Sep 22 23:15:15.732 DEBG 619 Reopen extent 93
Sep 22 23:15:15.733 DEBG 619 Reopen extent 93
Sep 22 23:15:15.733 DEBG [2] It's time to notify for 619
Sep 22 23:15:15.734 INFO Completion from [2] id:619 status:true
Sep 22 23:15:15.734 INFO [620/752] Repair commands completed
Sep 22 23:15:15.734 INFO Pop front: ReconcileIO { id: ReconciliationId(620), op: ExtentFlush { repair_id: ReconciliationId(620), extent_id: 40, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:15.734 INFO Sent repair work, now wait for resp
Sep 22 23:15:15.734 INFO [0] received reconcile message
Sep 22 23:15:15.734 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(620), op: ExtentFlush { repair_id: ReconciliationId(620), extent_id: 40, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:15.734 INFO [0] client ExtentFlush { repair_id: ReconciliationId(620), extent_id: 40, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:15.734 INFO [1] received reconcile message
Sep 22 23:15:15.734 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(620), op: ExtentFlush { repair_id: ReconciliationId(620), extent_id: 40, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:15.734 INFO [1] client ExtentFlush { repair_id: ReconciliationId(620), extent_id: 40, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:15.734 INFO [2] received reconcile message
Sep 22 23:15:15.734 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(620), op: ExtentFlush { repair_id: ReconciliationId(620), extent_id: 40, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:15.734 INFO [2] client ExtentFlush { repair_id: ReconciliationId(620), extent_id: 40, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:15.734 DEBG 620 Flush extent 40 with f:2 g:2
Sep 22 23:15:15.734 DEBG Flush just extent 40 with f:2 and g:2
Sep 22 23:15:15.734 DEBG [1] It's time to notify for 620
Sep 22 23:15:15.734 INFO Completion from [1] id:620 status:true
Sep 22 23:15:15.734 INFO [621/752] Repair commands completed
Sep 22 23:15:15.734 INFO Pop front: ReconcileIO { id: ReconciliationId(621), op: ExtentClose { repair_id: ReconciliationId(621), extent_id: 40 }, state: ClientData([New, New, New]) }
Sep 22 23:15:15.734 INFO Sent repair work, now wait for resp
Sep 22 23:15:15.734 INFO [0] received reconcile message
Sep 22 23:15:15.734 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(621), op: ExtentClose { repair_id: ReconciliationId(621), extent_id: 40 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:15.734 INFO [0] client ExtentClose { repair_id: ReconciliationId(621), extent_id: 40 }
Sep 22 23:15:15.734 INFO [1] received reconcile message
Sep 22 23:15:15.734 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(621), op: ExtentClose { repair_id: ReconciliationId(621), extent_id: 40 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:15.734 INFO [1] client ExtentClose { repair_id: ReconciliationId(621), extent_id: 40 }
Sep 22 23:15:15.734 INFO [2] received reconcile message
Sep 22 23:15:15.734 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(621), op: ExtentClose { repair_id: ReconciliationId(621), extent_id: 40 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:15.734 INFO [2] client ExtentClose { repair_id: ReconciliationId(621), extent_id: 40 }
Sep 22 23:15:15.735 DEBG 621 Close extent 40
Sep 22 23:15:15.735 DEBG 621 Close extent 40
Sep 22 23:15:15.735 DEBG 621 Close extent 40
Sep 22 23:15:15.735 DEBG [2] It's time to notify for 621
Sep 22 23:15:15.735 INFO Completion from [2] id:621 status:true
Sep 22 23:15:15.736 INFO [622/752] Repair commands completed
Sep 22 23:15:15.736 INFO Pop front: ReconcileIO { id: ReconciliationId(622), op: ExtentRepair { repair_id: ReconciliationId(622), extent_id: 40, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:15.736 INFO Sent repair work, now wait for resp
Sep 22 23:15:15.736 INFO [0] received reconcile message
Sep 22 23:15:15.736 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(622), op: ExtentRepair { repair_id: ReconciliationId(622), extent_id: 40, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:15.736 INFO [0] client ExtentRepair { repair_id: ReconciliationId(622), extent_id: 40, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:15.736 INFO [0] Sending repair request ReconciliationId(622)
Sep 22 23:15:15.736 INFO [1] received reconcile message
Sep 22 23:15:15.736 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(622), op: ExtentRepair { repair_id: ReconciliationId(622), extent_id: 40, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:15.736 INFO [1] client ExtentRepair { repair_id: ReconciliationId(622), extent_id: 40, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:15.736 INFO [1] No action required ReconciliationId(622)
Sep 22 23:15:15.736 INFO [2] received reconcile message
Sep 22 23:15:15.736 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(622), op: ExtentRepair { repair_id: ReconciliationId(622), extent_id: 40, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:15.736 INFO [2] client ExtentRepair { repair_id: ReconciliationId(622), extent_id: 40, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:15.736 INFO [2] No action required ReconciliationId(622)
Sep 22 23:15:15.736 DEBG 622 Repair extent 40 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:15.736 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/028.copy"
Sep 22 23:15:15.800 INFO accepted connection, remote_addr: 127.0.0.1:49461, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.800 TRCE incoming request, uri: /extent/40/files, method: GET, req_id: 1a4c69bb-3433-477a-8a4b-16a696305190, remote_addr: 127.0.0.1:49461, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.800 INFO request completed, latency_us: 195, response_code: 200, uri: /extent/40/files, method: GET, req_id: 1a4c69bb-3433-477a-8a4b-16a696305190, remote_addr: 127.0.0.1:49461, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.800 INFO eid:40 Found repair files: ["028", "028.db"]
Sep 22 23:15:15.801 TRCE incoming request, uri: /newextent/40/data, method: GET, req_id: 1748a9a3-a49b-4012-99a4-68ee072fa7eb, remote_addr: 127.0.0.1:49461, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.801 INFO request completed, latency_us: 326, response_code: 200, uri: /newextent/40/data, method: GET, req_id: 1748a9a3-a49b-4012-99a4-68ee072fa7eb, remote_addr: 127.0.0.1:49461, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.806 TRCE incoming request, uri: /newextent/40/db, method: GET, req_id: 964fce30-9850-4e3e-b2c4-7f5d119b95bf, remote_addr: 127.0.0.1:49461, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.806 INFO request completed, latency_us: 298, response_code: 200, uri: /newextent/40/db, method: GET, req_id: 964fce30-9850-4e3e-b2c4-7f5d119b95bf, remote_addr: 127.0.0.1:49461, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.808 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/028.copy" to "/tmp/downstairs-vrx8aK6L/00/000/028.replace"
Sep 22 23:15:15.808 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:15.808 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/028.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:15.809 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/028"
Sep 22 23:15:15.809 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/028.db"
Sep 22 23:15:15.809 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:15.809 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/028.replace" to "/tmp/downstairs-vrx8aK6L/00/000/028.completed"
Sep 22 23:15:15.809 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:15.809 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:15.809 DEBG [0] It's time to notify for 622
Sep 22 23:15:15.809 INFO Completion from [0] id:622 status:true
Sep 22 23:15:15.809 INFO [623/752] Repair commands completed
Sep 22 23:15:15.809 INFO Pop front: ReconcileIO { id: ReconciliationId(623), op: ExtentReopen { repair_id: ReconciliationId(623), extent_id: 40 }, state: ClientData([New, New, New]) }
Sep 22 23:15:15.809 INFO Sent repair work, now wait for resp
Sep 22 23:15:15.809 INFO [0] received reconcile message
Sep 22 23:15:15.809 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(623), op: ExtentReopen { repair_id: ReconciliationId(623), extent_id: 40 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:15.809 INFO [0] client ExtentReopen { repair_id: ReconciliationId(623), extent_id: 40 }
Sep 22 23:15:15.809 INFO [1] received reconcile message
Sep 22 23:15:15.809 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(623), op: ExtentReopen { repair_id: ReconciliationId(623), extent_id: 40 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:15.809 INFO [1] client ExtentReopen { repair_id: ReconciliationId(623), extent_id: 40 }
Sep 22 23:15:15.809 INFO [2] received reconcile message
Sep 22 23:15:15.809 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(623), op: ExtentReopen { repair_id: ReconciliationId(623), extent_id: 40 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:15.809 INFO [2] client ExtentReopen { repair_id: ReconciliationId(623), extent_id: 40 }
Sep 22 23:15:15.810 DEBG 623 Reopen extent 40
Sep 22 23:15:15.810 DEBG 623 Reopen extent 40
Sep 22 23:15:15.811 DEBG 623 Reopen extent 40
Sep 22 23:15:15.811 DEBG [2] It's time to notify for 623
Sep 22 23:15:15.811 INFO Completion from [2] id:623 status:true
Sep 22 23:15:15.811 INFO [624/752] Repair commands completed
Sep 22 23:15:15.811 INFO Pop front: ReconcileIO { id: ReconciliationId(624), op: ExtentFlush { repair_id: ReconciliationId(624), extent_id: 49, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:15.811 INFO Sent repair work, now wait for resp
Sep 22 23:15:15.811 INFO [0] received reconcile message
Sep 22 23:15:15.811 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(624), op: ExtentFlush { repair_id: ReconciliationId(624), extent_id: 49, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:15.811 INFO [0] client ExtentFlush { repair_id: ReconciliationId(624), extent_id: 49, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:15.812 INFO [1] received reconcile message
Sep 22 23:15:15.812 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(624), op: ExtentFlush { repair_id: ReconciliationId(624), extent_id: 49, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:15.812 INFO [1] client ExtentFlush { repair_id: ReconciliationId(624), extent_id: 49, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:15.812 INFO [2] received reconcile message
Sep 22 23:15:15.812 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(624), op: ExtentFlush { repair_id: ReconciliationId(624), extent_id: 49, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:15.812 INFO [2] client ExtentFlush { repair_id: ReconciliationId(624), extent_id: 49, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:15.812 DEBG 624 Flush extent 49 with f:2 g:2
Sep 22 23:15:15.812 DEBG Flush just extent 49 with f:2 and g:2
Sep 22 23:15:15.812 DEBG [1] It's time to notify for 624
Sep 22 23:15:15.812 INFO Completion from [1] id:624 status:true
Sep 22 23:15:15.812 INFO [625/752] Repair commands completed
Sep 22 23:15:15.812 INFO Pop front: ReconcileIO { id: ReconciliationId(625), op: ExtentClose { repair_id: ReconciliationId(625), extent_id: 49 }, state: ClientData([New, New, New]) }
Sep 22 23:15:15.812 INFO Sent repair work, now wait for resp
Sep 22 23:15:15.812 INFO [0] received reconcile message
Sep 22 23:15:15.812 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(625), op: ExtentClose { repair_id: ReconciliationId(625), extent_id: 49 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:15.812 INFO [0] client ExtentClose { repair_id: ReconciliationId(625), extent_id: 49 }
Sep 22 23:15:15.812 INFO [1] received reconcile message
Sep 22 23:15:15.812 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(625), op: ExtentClose { repair_id: ReconciliationId(625), extent_id: 49 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:15.812 INFO [1] client ExtentClose { repair_id: ReconciliationId(625), extent_id: 49 }
Sep 22 23:15:15.812 INFO [2] received reconcile message
Sep 22 23:15:15.812 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(625), op: ExtentClose { repair_id: ReconciliationId(625), extent_id: 49 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:15.812 INFO [2] client ExtentClose { repair_id: ReconciliationId(625), extent_id: 49 }
Sep 22 23:15:15.812 DEBG 625 Close extent 49
Sep 22 23:15:15.813 DEBG 625 Close extent 49
Sep 22 23:15:15.813 DEBG 625 Close extent 49
Sep 22 23:15:15.813 DEBG [2] It's time to notify for 625
Sep 22 23:15:15.813 INFO Completion from [2] id:625 status:true
Sep 22 23:15:15.813 INFO [626/752] Repair commands completed
Sep 22 23:15:15.813 INFO Pop front: ReconcileIO { id: ReconciliationId(626), op: ExtentRepair { repair_id: ReconciliationId(626), extent_id: 49, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:15.813 INFO Sent repair work, now wait for resp
Sep 22 23:15:15.813 INFO [0] received reconcile message
Sep 22 23:15:15.813 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(626), op: ExtentRepair { repair_id: ReconciliationId(626), extent_id: 49, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:15.813 INFO [0] client ExtentRepair { repair_id: ReconciliationId(626), extent_id: 49, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:15.813 INFO [0] Sending repair request ReconciliationId(626)
Sep 22 23:15:15.814 INFO [1] received reconcile message
Sep 22 23:15:15.814 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(626), op: ExtentRepair { repair_id: ReconciliationId(626), extent_id: 49, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:15.814 INFO [1] client ExtentRepair { repair_id: ReconciliationId(626), extent_id: 49, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:15.814 INFO [1] No action required ReconciliationId(626)
Sep 22 23:15:15.814 INFO [2] received reconcile message
Sep 22 23:15:15.814 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(626), op: ExtentRepair { repair_id: ReconciliationId(626), extent_id: 49, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:15.814 INFO [2] client ExtentRepair { repair_id: ReconciliationId(626), extent_id: 49, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:15.814 INFO [2] No action required ReconciliationId(626)
Sep 22 23:15:15.814 DEBG 626 Repair extent 49 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:15.814 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/031.copy"
Sep 22 23:15:15.848 DEBG IO Flush 1091 has deps [JobId(1090), JobId(1089)]
Sep 22 23:15:15.851 DEBG Flush     :1089 extent_limit None deps:[JobId(1088), JobId(1087)] res:true f:34 g:1
Sep 22 23:15:15.857 DEBG Read      :1090 deps:[JobId(1089)] res:true
Sep 22 23:15:15.877 INFO accepted connection, remote_addr: 127.0.0.1:47650, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.878 TRCE incoming request, uri: /extent/49/files, method: GET, req_id: 69056ed3-f253-4644-8347-812ca20dfb5f, remote_addr: 127.0.0.1:47650, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.878 INFO request completed, latency_us: 200, response_code: 200, uri: /extent/49/files, method: GET, req_id: 69056ed3-f253-4644-8347-812ca20dfb5f, remote_addr: 127.0.0.1:47650, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.878 INFO eid:49 Found repair files: ["031", "031.db"]
Sep 22 23:15:15.878 TRCE incoming request, uri: /newextent/49/data, method: GET, req_id: b04e0650-2f99-4855-991d-1cae2668e7c6, remote_addr: 127.0.0.1:47650, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.879 INFO request completed, latency_us: 255, response_code: 200, uri: /newextent/49/data, method: GET, req_id: b04e0650-2f99-4855-991d-1cae2668e7c6, remote_addr: 127.0.0.1:47650, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.884 TRCE incoming request, uri: /newextent/49/db, method: GET, req_id: 99a56e2c-0a8b-406c-ae8e-61e11e3b7f7d, remote_addr: 127.0.0.1:47650, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.884 INFO request completed, latency_us: 298, response_code: 200, uri: /newextent/49/db, method: GET, req_id: 99a56e2c-0a8b-406c-ae8e-61e11e3b7f7d, remote_addr: 127.0.0.1:47650, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.885 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/031.copy" to "/tmp/downstairs-vrx8aK6L/00/000/031.replace"
Sep 22 23:15:15.885 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:15.886 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/031.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:15.886 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/031"
Sep 22 23:15:15.887 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/031.db"
Sep 22 23:15:15.887 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:15.887 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/031.replace" to "/tmp/downstairs-vrx8aK6L/00/000/031.completed"
Sep 22 23:15:15.887 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:15.887 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:15.887 DEBG [0] It's time to notify for 626
Sep 22 23:15:15.887 INFO Completion from [0] id:626 status:true
Sep 22 23:15:15.887 INFO [627/752] Repair commands completed
Sep 22 23:15:15.887 INFO Pop front: ReconcileIO { id: ReconciliationId(627), op: ExtentReopen { repair_id: ReconciliationId(627), extent_id: 49 }, state: ClientData([New, New, New]) }
Sep 22 23:15:15.887 INFO Sent repair work, now wait for resp
Sep 22 23:15:15.887 INFO [0] received reconcile message
Sep 22 23:15:15.887 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(627), op: ExtentReopen { repair_id: ReconciliationId(627), extent_id: 49 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:15.887 INFO [0] client ExtentReopen { repair_id: ReconciliationId(627), extent_id: 49 }
Sep 22 23:15:15.887 INFO [1] received reconcile message
Sep 22 23:15:15.887 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(627), op: ExtentReopen { repair_id: ReconciliationId(627), extent_id: 49 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:15.887 INFO [1] client ExtentReopen { repair_id: ReconciliationId(627), extent_id: 49 }
Sep 22 23:15:15.887 INFO [2] received reconcile message
Sep 22 23:15:15.887 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(627), op: ExtentReopen { repair_id: ReconciliationId(627), extent_id: 49 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:15.887 INFO [2] client ExtentReopen { repair_id: ReconciliationId(627), extent_id: 49 }
Sep 22 23:15:15.887 DEBG 627 Reopen extent 49
Sep 22 23:15:15.888 DEBG 627 Reopen extent 49
Sep 22 23:15:15.889 DEBG 627 Reopen extent 49
Sep 22 23:15:15.889 DEBG [2] It's time to notify for 627
Sep 22 23:15:15.889 INFO Completion from [2] id:627 status:true
Sep 22 23:15:15.889 INFO [628/752] Repair commands completed
Sep 22 23:15:15.889 INFO Pop front: ReconcileIO { id: ReconciliationId(628), op: ExtentFlush { repair_id: ReconciliationId(628), extent_id: 81, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:15.889 INFO Sent repair work, now wait for resp
Sep 22 23:15:15.889 INFO [0] received reconcile message
Sep 22 23:15:15.889 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(628), op: ExtentFlush { repair_id: ReconciliationId(628), extent_id: 81, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:15.889 INFO [0] client ExtentFlush { repair_id: ReconciliationId(628), extent_id: 81, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:15.889 INFO [1] received reconcile message
Sep 22 23:15:15.889 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(628), op: ExtentFlush { repair_id: ReconciliationId(628), extent_id: 81, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:15.889 INFO [1] client ExtentFlush { repair_id: ReconciliationId(628), extent_id: 81, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:15.889 INFO [2] received reconcile message
Sep 22 23:15:15.889 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(628), op: ExtentFlush { repair_id: ReconciliationId(628), extent_id: 81, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:15.889 INFO [2] client ExtentFlush { repair_id: ReconciliationId(628), extent_id: 81, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:15.890 DEBG 628 Flush extent 81 with f:2 g:2
Sep 22 23:15:15.890 DEBG Flush just extent 81 with f:2 and g:2
Sep 22 23:15:15.890 DEBG [1] It's time to notify for 628
Sep 22 23:15:15.890 INFO Completion from [1] id:628 status:true
Sep 22 23:15:15.890 INFO [629/752] Repair commands completed
Sep 22 23:15:15.890 INFO Pop front: ReconcileIO { id: ReconciliationId(629), op: ExtentClose { repair_id: ReconciliationId(629), extent_id: 81 }, state: ClientData([New, New, New]) }
Sep 22 23:15:15.890 INFO Sent repair work, now wait for resp
Sep 22 23:15:15.890 INFO [0] received reconcile message
Sep 22 23:15:15.890 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(629), op: ExtentClose { repair_id: ReconciliationId(629), extent_id: 81 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:15.890 INFO [0] client ExtentClose { repair_id: ReconciliationId(629), extent_id: 81 }
Sep 22 23:15:15.890 INFO [1] received reconcile message
Sep 22 23:15:15.890 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(629), op: ExtentClose { repair_id: ReconciliationId(629), extent_id: 81 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:15.890 INFO [1] client ExtentClose { repair_id: ReconciliationId(629), extent_id: 81 }
Sep 22 23:15:15.890 INFO [2] received reconcile message
Sep 22 23:15:15.890 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(629), op: ExtentClose { repair_id: ReconciliationId(629), extent_id: 81 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:15.890 INFO [2] client ExtentClose { repair_id: ReconciliationId(629), extent_id: 81 }
Sep 22 23:15:15.890 DEBG 629 Close extent 81
Sep 22 23:15:15.890 DEBG 629 Close extent 81
Sep 22 23:15:15.891 DEBG 629 Close extent 81
Sep 22 23:15:15.891 DEBG [2] It's time to notify for 629
Sep 22 23:15:15.891 INFO Completion from [2] id:629 status:true
Sep 22 23:15:15.891 INFO [630/752] Repair commands completed
Sep 22 23:15:15.891 INFO Pop front: ReconcileIO { id: ReconciliationId(630), op: ExtentRepair { repair_id: ReconciliationId(630), extent_id: 81, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:15.891 INFO Sent repair work, now wait for resp
Sep 22 23:15:15.891 INFO [0] received reconcile message
Sep 22 23:15:15.891 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(630), op: ExtentRepair { repair_id: ReconciliationId(630), extent_id: 81, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:15.891 INFO [0] client ExtentRepair { repair_id: ReconciliationId(630), extent_id: 81, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:15.891 INFO [0] Sending repair request ReconciliationId(630)
Sep 22 23:15:15.891 INFO [1] received reconcile message
Sep 22 23:15:15.891 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(630), op: ExtentRepair { repair_id: ReconciliationId(630), extent_id: 81, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:15.891 INFO [1] client ExtentRepair { repair_id: ReconciliationId(630), extent_id: 81, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:15.891 INFO [1] No action required ReconciliationId(630)
Sep 22 23:15:15.891 INFO [2] received reconcile message
Sep 22 23:15:15.891 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(630), op: ExtentRepair { repair_id: ReconciliationId(630), extent_id: 81, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:15.891 INFO [2] client ExtentRepair { repair_id: ReconciliationId(630), extent_id: 81, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:15.891 INFO [2] No action required ReconciliationId(630)
Sep 22 23:15:15.892 DEBG 630 Repair extent 81 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:15.892 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/051.copy"
Sep 22 23:15:15.956 INFO accepted connection, remote_addr: 127.0.0.1:43364, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.956 TRCE incoming request, uri: /extent/81/files, method: GET, req_id: fec89fa8-a7a6-4d4f-9343-2d14afc6ce75, remote_addr: 127.0.0.1:43364, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.957 INFO request completed, latency_us: 198, response_code: 200, uri: /extent/81/files, method: GET, req_id: fec89fa8-a7a6-4d4f-9343-2d14afc6ce75, remote_addr: 127.0.0.1:43364, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.957 INFO eid:81 Found repair files: ["051", "051.db"]
Sep 22 23:15:15.957 TRCE incoming request, uri: /newextent/81/data, method: GET, req_id: ed6ea4bb-430a-4fb8-8e09-417758209545, remote_addr: 127.0.0.1:43364, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.957 INFO request completed, latency_us: 306, response_code: 200, uri: /newextent/81/data, method: GET, req_id: ed6ea4bb-430a-4fb8-8e09-417758209545, remote_addr: 127.0.0.1:43364, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.963 TRCE incoming request, uri: /newextent/81/db, method: GET, req_id: 9940fb4a-7356-4e37-9955-070dd08f83c5, remote_addr: 127.0.0.1:43364, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.963 INFO request completed, latency_us: 293, response_code: 200, uri: /newextent/81/db, method: GET, req_id: 9940fb4a-7356-4e37-9955-070dd08f83c5, remote_addr: 127.0.0.1:43364, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:15.964 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/051.copy" to "/tmp/downstairs-vrx8aK6L/00/000/051.replace"
Sep 22 23:15:15.964 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:15.965 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/051.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:15.965 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/051"
Sep 22 23:15:15.965 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/051.db"
Sep 22 23:15:15.965 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:15.965 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/051.replace" to "/tmp/downstairs-vrx8aK6L/00/000/051.completed"
Sep 22 23:15:15.965 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:15.965 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:15.966 DEBG [0] It's time to notify for 630
Sep 22 23:15:15.966 INFO Completion from [0] id:630 status:true
Sep 22 23:15:15.966 INFO [631/752] Repair commands completed
Sep 22 23:15:15.966 INFO Pop front: ReconcileIO { id: ReconciliationId(631), op: ExtentReopen { repair_id: ReconciliationId(631), extent_id: 81 }, state: ClientData([New, New, New]) }
Sep 22 23:15:15.966 INFO Sent repair work, now wait for resp
Sep 22 23:15:15.966 INFO [0] received reconcile message
Sep 22 23:15:15.966 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(631), op: ExtentReopen { repair_id: ReconciliationId(631), extent_id: 81 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:15.966 INFO [0] client ExtentReopen { repair_id: ReconciliationId(631), extent_id: 81 }
Sep 22 23:15:15.966 INFO [1] received reconcile message
Sep 22 23:15:15.966 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(631), op: ExtentReopen { repair_id: ReconciliationId(631), extent_id: 81 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:15.966 INFO [1] client ExtentReopen { repair_id: ReconciliationId(631), extent_id: 81 }
Sep 22 23:15:15.966 INFO [2] received reconcile message
Sep 22 23:15:15.966 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(631), op: ExtentReopen { repair_id: ReconciliationId(631), extent_id: 81 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:15.966 INFO [2] client ExtentReopen { repair_id: ReconciliationId(631), extent_id: 81 }
Sep 22 23:15:15.966 DEBG 631 Reopen extent 81
Sep 22 23:15:15.967 DEBG 631 Reopen extent 81
Sep 22 23:15:15.967 DEBG 631 Reopen extent 81
Sep 22 23:15:15.968 DEBG [2] It's time to notify for 631
Sep 22 23:15:15.968 INFO Completion from [2] id:631 status:true
Sep 22 23:15:15.968 INFO [632/752] Repair commands completed
Sep 22 23:15:15.968 INFO Pop front: ReconcileIO { id: ReconciliationId(632), op: ExtentFlush { repair_id: ReconciliationId(632), extent_id: 170, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:15.968 INFO Sent repair work, now wait for resp
Sep 22 23:15:15.968 INFO [0] received reconcile message
Sep 22 23:15:15.968 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(632), op: ExtentFlush { repair_id: ReconciliationId(632), extent_id: 170, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:15.968 INFO [0] client ExtentFlush { repair_id: ReconciliationId(632), extent_id: 170, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:15.968 INFO [1] received reconcile message
Sep 22 23:15:15.968 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(632), op: ExtentFlush { repair_id: ReconciliationId(632), extent_id: 170, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:15.968 INFO [1] client ExtentFlush { repair_id: ReconciliationId(632), extent_id: 170, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:15.968 INFO [2] received reconcile message
Sep 22 23:15:15.968 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(632), op: ExtentFlush { repair_id: ReconciliationId(632), extent_id: 170, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:15.968 INFO [2] client ExtentFlush { repair_id: ReconciliationId(632), extent_id: 170, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:15.968 DEBG 632 Flush extent 170 with f:2 g:2
Sep 22 23:15:15.968 DEBG Flush just extent 170 with f:2 and g:2
Sep 22 23:15:15.968 DEBG [1] It's time to notify for 632
Sep 22 23:15:15.968 INFO Completion from [1] id:632 status:true
Sep 22 23:15:15.968 INFO [633/752] Repair commands completed
Sep 22 23:15:15.968 INFO Pop front: ReconcileIO { id: ReconciliationId(633), op: ExtentClose { repair_id: ReconciliationId(633), extent_id: 170 }, state: ClientData([New, New, New]) }
Sep 22 23:15:15.968 INFO Sent repair work, now wait for resp
Sep 22 23:15:15.969 INFO [0] received reconcile message
Sep 22 23:15:15.969 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(633), op: ExtentClose { repair_id: ReconciliationId(633), extent_id: 170 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:15.969 INFO [0] client ExtentClose { repair_id: ReconciliationId(633), extent_id: 170 }
Sep 22 23:15:15.969 INFO [1] received reconcile message
Sep 22 23:15:15.969 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(633), op: ExtentClose { repair_id: ReconciliationId(633), extent_id: 170 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:15.969 INFO [1] client ExtentClose { repair_id: ReconciliationId(633), extent_id: 170 }
Sep 22 23:15:15.969 INFO [2] received reconcile message
Sep 22 23:15:15.969 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(633), op: ExtentClose { repair_id: ReconciliationId(633), extent_id: 170 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:15.969 INFO [2] client ExtentClose { repair_id: ReconciliationId(633), extent_id: 170 }
Sep 22 23:15:15.969 DEBG 633 Close extent 170
Sep 22 23:15:15.969 DEBG 633 Close extent 170
Sep 22 23:15:15.969 DEBG 633 Close extent 170
Sep 22 23:15:15.970 DEBG [2] It's time to notify for 633
Sep 22 23:15:15.970 INFO Completion from [2] id:633 status:true
Sep 22 23:15:15.970 INFO [634/752] Repair commands completed
Sep 22 23:15:15.970 INFO Pop front: ReconcileIO { id: ReconciliationId(634), op: ExtentRepair { repair_id: ReconciliationId(634), extent_id: 170, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:15.970 INFO Sent repair work, now wait for resp
Sep 22 23:15:15.970 INFO [0] received reconcile message
Sep 22 23:15:15.970 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(634), op: ExtentRepair { repair_id: ReconciliationId(634), extent_id: 170, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:15.970 INFO [0] client ExtentRepair { repair_id: ReconciliationId(634), extent_id: 170, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:15.970 INFO [0] Sending repair request ReconciliationId(634)
Sep 22 23:15:15.970 INFO [1] received reconcile message
Sep 22 23:15:15.970 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(634), op: ExtentRepair { repair_id: ReconciliationId(634), extent_id: 170, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:15.970 INFO [1] client ExtentRepair { repair_id: ReconciliationId(634), extent_id: 170, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:15.970 INFO [1] No action required ReconciliationId(634)
Sep 22 23:15:15.970 INFO [2] received reconcile message
Sep 22 23:15:15.970 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(634), op: ExtentRepair { repair_id: ReconciliationId(634), extent_id: 170, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:15.970 INFO [2] client ExtentRepair { repair_id: ReconciliationId(634), extent_id: 170, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:15.970 INFO [2] No action required ReconciliationId(634)
Sep 22 23:15:15.970 DEBG 634 Repair extent 170 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:15.970 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/0AA.copy"
Sep 22 23:15:16.034 INFO accepted connection, remote_addr: 127.0.0.1:63247, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.034 TRCE incoming request, uri: /extent/170/files, method: GET, req_id: e4617a76-c241-4332-af4f-2abc369abaf9, remote_addr: 127.0.0.1:63247, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.034 INFO request completed, latency_us: 251, response_code: 200, uri: /extent/170/files, method: GET, req_id: e4617a76-c241-4332-af4f-2abc369abaf9, remote_addr: 127.0.0.1:63247, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.035 INFO eid:170 Found repair files: ["0AA", "0AA.db"]
Sep 22 23:15:16.035 TRCE incoming request, uri: /newextent/170/data, method: GET, req_id: b38bde04-3774-4f40-bfc4-392d4c077a25, remote_addr: 127.0.0.1:63247, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.035 INFO request completed, latency_us: 344, response_code: 200, uri: /newextent/170/data, method: GET, req_id: b38bde04-3774-4f40-bfc4-392d4c077a25, remote_addr: 127.0.0.1:63247, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.040 TRCE incoming request, uri: /newextent/170/db, method: GET, req_id: a75aaa63-2b62-4bab-8f14-7481abc6a039, remote_addr: 127.0.0.1:63247, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.041 INFO request completed, latency_us: 295, response_code: 200, uri: /newextent/170/db, method: GET, req_id: a75aaa63-2b62-4bab-8f14-7481abc6a039, remote_addr: 127.0.0.1:63247, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.042 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/0AA.copy" to "/tmp/downstairs-vrx8aK6L/00/000/0AA.replace"
Sep 22 23:15:16.042 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:16.043 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/0AA.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:16.043 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/0AA"
Sep 22 23:15:16.043 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/0AA.db"
Sep 22 23:15:16.043 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:16.043 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/0AA.replace" to "/tmp/downstairs-vrx8aK6L/00/000/0AA.completed"
Sep 22 23:15:16.044 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:16.044 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:16.044 DEBG [0] It's time to notify for 634
Sep 22 23:15:16.044 INFO Completion from [0] id:634 status:true
Sep 22 23:15:16.044 INFO [635/752] Repair commands completed
Sep 22 23:15:16.044 INFO Pop front: ReconcileIO { id: ReconciliationId(635), op: ExtentReopen { repair_id: ReconciliationId(635), extent_id: 170 }, state: ClientData([New, New, New]) }
Sep 22 23:15:16.044 INFO Sent repair work, now wait for resp
Sep 22 23:15:16.044 INFO [0] received reconcile message
Sep 22 23:15:16.044 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(635), op: ExtentReopen { repair_id: ReconciliationId(635), extent_id: 170 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:16.044 INFO [0] client ExtentReopen { repair_id: ReconciliationId(635), extent_id: 170 }
Sep 22 23:15:16.044 INFO [1] received reconcile message
Sep 22 23:15:16.044 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(635), op: ExtentReopen { repair_id: ReconciliationId(635), extent_id: 170 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:16.044 INFO [1] client ExtentReopen { repair_id: ReconciliationId(635), extent_id: 170 }
Sep 22 23:15:16.044 INFO [2] received reconcile message
Sep 22 23:15:16.044 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(635), op: ExtentReopen { repair_id: ReconciliationId(635), extent_id: 170 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:16.044 INFO [2] client ExtentReopen { repair_id: ReconciliationId(635), extent_id: 170 }
Sep 22 23:15:16.044 DEBG 635 Reopen extent 170
Sep 22 23:15:16.045 DEBG 635 Reopen extent 170
Sep 22 23:15:16.046 DEBG 635 Reopen extent 170
Sep 22 23:15:16.046 DEBG [2] It's time to notify for 635
Sep 22 23:15:16.046 INFO Completion from [2] id:635 status:true
Sep 22 23:15:16.046 INFO [636/752] Repair commands completed
Sep 22 23:15:16.046 INFO Pop front: ReconcileIO { id: ReconciliationId(636), op: ExtentFlush { repair_id: ReconciliationId(636), extent_id: 13, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:16.046 INFO Sent repair work, now wait for resp
Sep 22 23:15:16.046 INFO [0] received reconcile message
Sep 22 23:15:16.046 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(636), op: ExtentFlush { repair_id: ReconciliationId(636), extent_id: 13, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:16.046 INFO [0] client ExtentFlush { repair_id: ReconciliationId(636), extent_id: 13, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:16.047 INFO [1] received reconcile message
Sep 22 23:15:16.047 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(636), op: ExtentFlush { repair_id: ReconciliationId(636), extent_id: 13, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:16.047 INFO [1] client ExtentFlush { repair_id: ReconciliationId(636), extent_id: 13, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:16.047 INFO [2] received reconcile message
Sep 22 23:15:16.047 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(636), op: ExtentFlush { repair_id: ReconciliationId(636), extent_id: 13, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:16.047 INFO [2] client ExtentFlush { repair_id: ReconciliationId(636), extent_id: 13, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:16.047 DEBG 636 Flush extent 13 with f:2 g:2
Sep 22 23:15:16.047 DEBG Flush just extent 13 with f:2 and g:2
Sep 22 23:15:16.047 DEBG [1] It's time to notify for 636
Sep 22 23:15:16.047 INFO Completion from [1] id:636 status:true
Sep 22 23:15:16.047 INFO [637/752] Repair commands completed
Sep 22 23:15:16.047 INFO Pop front: ReconcileIO { id: ReconciliationId(637), op: ExtentClose { repair_id: ReconciliationId(637), extent_id: 13 }, state: ClientData([New, New, New]) }
Sep 22 23:15:16.047 INFO Sent repair work, now wait for resp
Sep 22 23:15:16.047 INFO [0] received reconcile message
Sep 22 23:15:16.047 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(637), op: ExtentClose { repair_id: ReconciliationId(637), extent_id: 13 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:16.047 INFO [0] client ExtentClose { repair_id: ReconciliationId(637), extent_id: 13 }
Sep 22 23:15:16.047 INFO [1] received reconcile message
Sep 22 23:15:16.047 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(637), op: ExtentClose { repair_id: ReconciliationId(637), extent_id: 13 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:16.047 INFO [1] client ExtentClose { repair_id: ReconciliationId(637), extent_id: 13 }
Sep 22 23:15:16.047 INFO [2] received reconcile message
Sep 22 23:15:16.047 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(637), op: ExtentClose { repair_id: ReconciliationId(637), extent_id: 13 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:16.047 INFO [2] client ExtentClose { repair_id: ReconciliationId(637), extent_id: 13 }
Sep 22 23:15:16.047 DEBG 637 Close extent 13
Sep 22 23:15:16.048 DEBG 637 Close extent 13
Sep 22 23:15:16.048 DEBG 637 Close extent 13
Sep 22 23:15:16.048 DEBG [2] It's time to notify for 637
Sep 22 23:15:16.048 INFO Completion from [2] id:637 status:true
Sep 22 23:15:16.048 INFO [638/752] Repair commands completed
Sep 22 23:15:16.048 INFO Pop front: ReconcileIO { id: ReconciliationId(638), op: ExtentRepair { repair_id: ReconciliationId(638), extent_id: 13, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:16.048 INFO Sent repair work, now wait for resp
Sep 22 23:15:16.048 INFO [0] received reconcile message
Sep 22 23:15:16.048 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(638), op: ExtentRepair { repair_id: ReconciliationId(638), extent_id: 13, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:16.048 INFO [0] client ExtentRepair { repair_id: ReconciliationId(638), extent_id: 13, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:16.048 INFO [0] Sending repair request ReconciliationId(638)
Sep 22 23:15:16.049 INFO [1] received reconcile message
Sep 22 23:15:16.049 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(638), op: ExtentRepair { repair_id: ReconciliationId(638), extent_id: 13, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:16.049 INFO [1] client ExtentRepair { repair_id: ReconciliationId(638), extent_id: 13, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:16.049 INFO [1] No action required ReconciliationId(638)
Sep 22 23:15:16.049 INFO [2] received reconcile message
Sep 22 23:15:16.049 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(638), op: ExtentRepair { repair_id: ReconciliationId(638), extent_id: 13, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:16.049 INFO [2] client ExtentRepair { repair_id: ReconciliationId(638), extent_id: 13, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:16.049 INFO [2] No action required ReconciliationId(638)
Sep 22 23:15:16.049 DEBG 638 Repair extent 13 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:16.049 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/00D.copy"
Sep 22 23:15:16.112 INFO accepted connection, remote_addr: 127.0.0.1:42474, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.112 TRCE incoming request, uri: /extent/13/files, method: GET, req_id: d8906cf3-83f7-487f-8e73-b93bf4f67bcc, remote_addr: 127.0.0.1:42474, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.112 INFO request completed, latency_us: 193, response_code: 200, uri: /extent/13/files, method: GET, req_id: d8906cf3-83f7-487f-8e73-b93bf4f67bcc, remote_addr: 127.0.0.1:42474, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.113 INFO eid:13 Found repair files: ["00D", "00D.db"]
Sep 22 23:15:16.113 TRCE incoming request, uri: /newextent/13/data, method: GET, req_id: ffe2050c-b3e4-4268-a937-7fdca04098b2, remote_addr: 127.0.0.1:42474, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.113 INFO request completed, latency_us: 298, response_code: 200, uri: /newextent/13/data, method: GET, req_id: ffe2050c-b3e4-4268-a937-7fdca04098b2, remote_addr: 127.0.0.1:42474, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.118 TRCE incoming request, uri: /newextent/13/db, method: GET, req_id: 87953b4e-4d7e-497b-8c24-564593dd29fe, remote_addr: 127.0.0.1:42474, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.119 INFO request completed, latency_us: 293, response_code: 200, uri: /newextent/13/db, method: GET, req_id: 87953b4e-4d7e-497b-8c24-564593dd29fe, remote_addr: 127.0.0.1:42474, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.120 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/00D.copy" to "/tmp/downstairs-vrx8aK6L/00/000/00D.replace"
Sep 22 23:15:16.120 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:16.121 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/00D.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:16.121 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/00D"
Sep 22 23:15:16.121 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/00D.db"
Sep 22 23:15:16.121 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:16.121 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/00D.replace" to "/tmp/downstairs-vrx8aK6L/00/000/00D.completed"
Sep 22 23:15:16.121 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:16.121 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:16.121 DEBG [0] It's time to notify for 638
Sep 22 23:15:16.121 INFO Completion from [0] id:638 status:true
Sep 22 23:15:16.121 INFO [639/752] Repair commands completed
Sep 22 23:15:16.121 INFO Pop front: ReconcileIO { id: ReconciliationId(639), op: ExtentReopen { repair_id: ReconciliationId(639), extent_id: 13 }, state: ClientData([New, New, New]) }
Sep 22 23:15:16.122 INFO Sent repair work, now wait for resp
Sep 22 23:15:16.122 INFO [0] received reconcile message
Sep 22 23:15:16.122 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(639), op: ExtentReopen { repair_id: ReconciliationId(639), extent_id: 13 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:16.122 INFO [0] client ExtentReopen { repair_id: ReconciliationId(639), extent_id: 13 }
Sep 22 23:15:16.122 INFO [1] received reconcile message
Sep 22 23:15:16.122 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(639), op: ExtentReopen { repair_id: ReconciliationId(639), extent_id: 13 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:16.122 INFO [1] client ExtentReopen { repair_id: ReconciliationId(639), extent_id: 13 }
Sep 22 23:15:16.122 INFO [2] received reconcile message
Sep 22 23:15:16.122 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(639), op: ExtentReopen { repair_id: ReconciliationId(639), extent_id: 13 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:16.122 INFO [2] client ExtentReopen { repair_id: ReconciliationId(639), extent_id: 13 }
Sep 22 23:15:16.122 DEBG 639 Reopen extent 13
Sep 22 23:15:16.122 DEBG 639 Reopen extent 13
Sep 22 23:15:16.123 DEBG 639 Reopen extent 13
Sep 22 23:15:16.124 DEBG [2] It's time to notify for 639
Sep 22 23:15:16.124 INFO Completion from [2] id:639 status:true
Sep 22 23:15:16.124 INFO [640/752] Repair commands completed
Sep 22 23:15:16.124 INFO Pop front: ReconcileIO { id: ReconciliationId(640), op: ExtentFlush { repair_id: ReconciliationId(640), extent_id: 27, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:16.124 INFO Sent repair work, now wait for resp
Sep 22 23:15:16.124 INFO [0] received reconcile message
Sep 22 23:15:16.124 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(640), op: ExtentFlush { repair_id: ReconciliationId(640), extent_id: 27, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:16.124 INFO [0] client ExtentFlush { repair_id: ReconciliationId(640), extent_id: 27, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:16.124 INFO [1] received reconcile message
Sep 22 23:15:16.124 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(640), op: ExtentFlush { repair_id: ReconciliationId(640), extent_id: 27, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:16.124 INFO [1] client ExtentFlush { repair_id: ReconciliationId(640), extent_id: 27, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:16.124 INFO [2] received reconcile message
Sep 22 23:15:16.124 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(640), op: ExtentFlush { repair_id: ReconciliationId(640), extent_id: 27, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:16.124 INFO [2] client ExtentFlush { repair_id: ReconciliationId(640), extent_id: 27, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:16.124 DEBG 640 Flush extent 27 with f:2 g:2
Sep 22 23:15:16.124 DEBG Flush just extent 27 with f:2 and g:2
Sep 22 23:15:16.124 DEBG [1] It's time to notify for 640
Sep 22 23:15:16.124 INFO Completion from [1] id:640 status:true
Sep 22 23:15:16.124 INFO [641/752] Repair commands completed
Sep 22 23:15:16.124 INFO Pop front: ReconcileIO { id: ReconciliationId(641), op: ExtentClose { repair_id: ReconciliationId(641), extent_id: 27 }, state: ClientData([New, New, New]) }
Sep 22 23:15:16.124 INFO Sent repair work, now wait for resp
Sep 22 23:15:16.124 INFO [0] received reconcile message
Sep 22 23:15:16.124 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(641), op: ExtentClose { repair_id: ReconciliationId(641), extent_id: 27 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:16.124 INFO [0] client ExtentClose { repair_id: ReconciliationId(641), extent_id: 27 }
Sep 22 23:15:16.124 INFO [1] received reconcile message
Sep 22 23:15:16.124 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(641), op: ExtentClose { repair_id: ReconciliationId(641), extent_id: 27 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:16.124 INFO [1] client ExtentClose { repair_id: ReconciliationId(641), extent_id: 27 }
Sep 22 23:15:16.124 INFO [2] received reconcile message
Sep 22 23:15:16.124 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(641), op: ExtentClose { repair_id: ReconciliationId(641), extent_id: 27 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:16.124 INFO [2] client ExtentClose { repair_id: ReconciliationId(641), extent_id: 27 }
Sep 22 23:15:16.125 DEBG 641 Close extent 27
Sep 22 23:15:16.125 DEBG 641 Close extent 27
Sep 22 23:15:16.125 DEBG 641 Close extent 27
Sep 22 23:15:16.126 DEBG [2] It's time to notify for 641
Sep 22 23:15:16.126 INFO Completion from [2] id:641 status:true
Sep 22 23:15:16.126 INFO [642/752] Repair commands completed
Sep 22 23:15:16.126 INFO Pop front: ReconcileIO { id: ReconciliationId(642), op: ExtentRepair { repair_id: ReconciliationId(642), extent_id: 27, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:16.126 INFO Sent repair work, now wait for resp
Sep 22 23:15:16.126 INFO [0] received reconcile message
Sep 22 23:15:16.126 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(642), op: ExtentRepair { repair_id: ReconciliationId(642), extent_id: 27, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:16.126 INFO [0] client ExtentRepair { repair_id: ReconciliationId(642), extent_id: 27, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:16.126 INFO [0] Sending repair request ReconciliationId(642)
Sep 22 23:15:16.126 INFO [1] received reconcile message
Sep 22 23:15:16.126 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(642), op: ExtentRepair { repair_id: ReconciliationId(642), extent_id: 27, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:16.126 INFO [1] client ExtentRepair { repair_id: ReconciliationId(642), extent_id: 27, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:16.126 INFO [1] No action required ReconciliationId(642)
Sep 22 23:15:16.126 INFO [2] received reconcile message
Sep 22 23:15:16.126 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(642), op: ExtentRepair { repair_id: ReconciliationId(642), extent_id: 27, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:16.126 INFO [2] client ExtentRepair { repair_id: ReconciliationId(642), extent_id: 27, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:16.126 INFO [2] No action required ReconciliationId(642)
Sep 22 23:15:16.126 DEBG 642 Repair extent 27 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:16.126 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/01B.copy"
Sep 22 23:15:16.189 INFO accepted connection, remote_addr: 127.0.0.1:39512, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.189 TRCE incoming request, uri: /extent/27/files, method: GET, req_id: dd6c63a9-84bd-4916-bff6-6c6e90ec385a, remote_addr: 127.0.0.1:39512, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.190 INFO request completed, latency_us: 200, response_code: 200, uri: /extent/27/files, method: GET, req_id: dd6c63a9-84bd-4916-bff6-6c6e90ec385a, remote_addr: 127.0.0.1:39512, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.190 INFO eid:27 Found repair files: ["01B", "01B.db"]
Sep 22 23:15:16.190 TRCE incoming request, uri: /newextent/27/data, method: GET, req_id: 1e8f8524-a24b-40b3-82ae-aa1de4e0af17, remote_addr: 127.0.0.1:39512, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.190 INFO request completed, latency_us: 307, response_code: 200, uri: /newextent/27/data, method: GET, req_id: 1e8f8524-a24b-40b3-82ae-aa1de4e0af17, remote_addr: 127.0.0.1:39512, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.196 TRCE incoming request, uri: /newextent/27/db, method: GET, req_id: 75a84721-1acb-4dcf-9237-e95890348a2d, remote_addr: 127.0.0.1:39512, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.196 INFO request completed, latency_us: 294, response_code: 200, uri: /newextent/27/db, method: GET, req_id: 75a84721-1acb-4dcf-9237-e95890348a2d, remote_addr: 127.0.0.1:39512, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.197 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/01B.copy" to "/tmp/downstairs-vrx8aK6L/00/000/01B.replace"
Sep 22 23:15:16.197 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:16.198 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/01B.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:16.198 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/01B"
Sep 22 23:15:16.198 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/01B.db"
Sep 22 23:15:16.198 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:16.198 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/01B.replace" to "/tmp/downstairs-vrx8aK6L/00/000/01B.completed"
Sep 22 23:15:16.198 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:16.199 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:16.199 DEBG [0] It's time to notify for 642
Sep 22 23:15:16.199 INFO Completion from [0] id:642 status:true
Sep 22 23:15:16.199 INFO [643/752] Repair commands completed
Sep 22 23:15:16.199 INFO Pop front: ReconcileIO { id: ReconciliationId(643), op: ExtentReopen { repair_id: ReconciliationId(643), extent_id: 27 }, state: ClientData([New, New, New]) }
Sep 22 23:15:16.199 INFO Sent repair work, now wait for resp
Sep 22 23:15:16.199 INFO [0] received reconcile message
Sep 22 23:15:16.199 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(643), op: ExtentReopen { repair_id: ReconciliationId(643), extent_id: 27 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:16.199 INFO [0] client ExtentReopen { repair_id: ReconciliationId(643), extent_id: 27 }
Sep 22 23:15:16.199 INFO [1] received reconcile message
Sep 22 23:15:16.199 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(643), op: ExtentReopen { repair_id: ReconciliationId(643), extent_id: 27 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:16.199 INFO [1] client ExtentReopen { repair_id: ReconciliationId(643), extent_id: 27 }
Sep 22 23:15:16.199 INFO [2] received reconcile message
Sep 22 23:15:16.199 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(643), op: ExtentReopen { repair_id: ReconciliationId(643), extent_id: 27 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:16.199 INFO [2] client ExtentReopen { repair_id: ReconciliationId(643), extent_id: 27 }
Sep 22 23:15:16.199 DEBG 643 Reopen extent 27
Sep 22 23:15:16.200 DEBG 643 Reopen extent 27
Sep 22 23:15:16.200 DEBG 643 Reopen extent 27
Sep 22 23:15:16.201 DEBG [2] It's time to notify for 643
Sep 22 23:15:16.201 INFO Completion from [2] id:643 status:true
Sep 22 23:15:16.201 INFO [644/752] Repair commands completed
Sep 22 23:15:16.201 INFO Pop front: ReconcileIO { id: ReconciliationId(644), op: ExtentFlush { repair_id: ReconciliationId(644), extent_id: 8, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:16.201 INFO Sent repair work, now wait for resp
Sep 22 23:15:16.201 INFO [0] received reconcile message
Sep 22 23:15:16.201 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(644), op: ExtentFlush { repair_id: ReconciliationId(644), extent_id: 8, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:16.201 INFO [0] client ExtentFlush { repair_id: ReconciliationId(644), extent_id: 8, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:16.201 INFO [1] received reconcile message
Sep 22 23:15:16.201 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(644), op: ExtentFlush { repair_id: ReconciliationId(644), extent_id: 8, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:16.201 INFO [1] client ExtentFlush { repair_id: ReconciliationId(644), extent_id: 8, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:16.201 INFO [2] received reconcile message
Sep 22 23:15:16.201 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(644), op: ExtentFlush { repair_id: ReconciliationId(644), extent_id: 8, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:16.201 INFO [2] client ExtentFlush { repair_id: ReconciliationId(644), extent_id: 8, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:16.201 DEBG 644 Flush extent 8 with f:2 g:2
Sep 22 23:15:16.201 DEBG Flush just extent 8 with f:2 and g:2
Sep 22 23:15:16.202 DEBG [1] It's time to notify for 644
Sep 22 23:15:16.202 INFO Completion from [1] id:644 status:true
Sep 22 23:15:16.202 INFO [645/752] Repair commands completed
Sep 22 23:15:16.202 INFO Pop front: ReconcileIO { id: ReconciliationId(645), op: ExtentClose { repair_id: ReconciliationId(645), extent_id: 8 }, state: ClientData([New, New, New]) }
Sep 22 23:15:16.202 INFO Sent repair work, now wait for resp
Sep 22 23:15:16.202 INFO [0] received reconcile message
Sep 22 23:15:16.202 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(645), op: ExtentClose { repair_id: ReconciliationId(645), extent_id: 8 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:16.202 INFO [0] client ExtentClose { repair_id: ReconciliationId(645), extent_id: 8 }
Sep 22 23:15:16.202 INFO [1] received reconcile message
Sep 22 23:15:16.202 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(645), op: ExtentClose { repair_id: ReconciliationId(645), extent_id: 8 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:16.202 INFO [1] client ExtentClose { repair_id: ReconciliationId(645), extent_id: 8 }
Sep 22 23:15:16.202 INFO [2] received reconcile message
Sep 22 23:15:16.202 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(645), op: ExtentClose { repair_id: ReconciliationId(645), extent_id: 8 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:16.202 INFO [2] client ExtentClose { repair_id: ReconciliationId(645), extent_id: 8 }
Sep 22 23:15:16.202 DEBG 645 Close extent 8
Sep 22 23:15:16.202 DEBG 645 Close extent 8
Sep 22 23:15:16.203 DEBG 645 Close extent 8
Sep 22 23:15:16.203 DEBG [2] It's time to notify for 645
Sep 22 23:15:16.203 INFO Completion from [2] id:645 status:true
Sep 22 23:15:16.203 INFO [646/752] Repair commands completed
Sep 22 23:15:16.203 INFO Pop front: ReconcileIO { id: ReconciliationId(646), op: ExtentRepair { repair_id: ReconciliationId(646), extent_id: 8, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:16.203 INFO Sent repair work, now wait for resp
Sep 22 23:15:16.203 INFO [0] received reconcile message
Sep 22 23:15:16.203 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(646), op: ExtentRepair { repair_id: ReconciliationId(646), extent_id: 8, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:16.203 INFO [0] client ExtentRepair { repair_id: ReconciliationId(646), extent_id: 8, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:16.203 INFO [0] Sending repair request ReconciliationId(646)
Sep 22 23:15:16.203 INFO [1] received reconcile message
Sep 22 23:15:16.203 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(646), op: ExtentRepair { repair_id: ReconciliationId(646), extent_id: 8, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:16.203 INFO [1] client ExtentRepair { repair_id: ReconciliationId(646), extent_id: 8, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:16.203 INFO [1] No action required ReconciliationId(646)
Sep 22 23:15:16.203 INFO [2] received reconcile message
Sep 22 23:15:16.203 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(646), op: ExtentRepair { repair_id: ReconciliationId(646), extent_id: 8, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:16.203 INFO [2] client ExtentRepair { repair_id: ReconciliationId(646), extent_id: 8, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:16.203 INFO [2] No action required ReconciliationId(646)
Sep 22 23:15:16.203 DEBG 646 Repair extent 8 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:16.204 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/008.copy"
Sep 22 23:15:16.258 DEBG up_ds_listen was notified
Sep 22 23:15:16.258 DEBG up_ds_listen process 1089
Sep 22 23:15:16.258 DEBG [A] ack job 1089:90, : downstairs
Sep 22 23:15:16.258 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:16.258 DEBG Flush     :1089 extent_limit None deps:[JobId(1088), JobId(1087)] res:true f:34 g:1
Sep 22 23:15:16.259 WARN returning error on read!
Sep 22 23:15:16.259 DEBG Read      :1090 deps:[JobId(1089)] res:false
Sep 22 23:15:16.265 DEBG Read      :1090 deps:[JobId(1089)] res:true
Sep 22 23:15:16.266 INFO accepted connection, remote_addr: 127.0.0.1:49874, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.266 TRCE incoming request, uri: /extent/8/files, method: GET, req_id: 2e9ab566-b7c7-483d-9db8-babb4b21cd32, remote_addr: 127.0.0.1:49874, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.266 INFO request completed, latency_us: 197, response_code: 200, uri: /extent/8/files, method: GET, req_id: 2e9ab566-b7c7-483d-9db8-babb4b21cd32, remote_addr: 127.0.0.1:49874, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.267 INFO eid:8 Found repair files: ["008", "008.db"]
Sep 22 23:15:16.267 TRCE incoming request, uri: /newextent/8/data, method: GET, req_id: 08bdc375-501e-4b39-9a1d-3acfa8028999, remote_addr: 127.0.0.1:49874, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.267 INFO request completed, latency_us: 304, response_code: 200, uri: /newextent/8/data, method: GET, req_id: 08bdc375-501e-4b39-9a1d-3acfa8028999, remote_addr: 127.0.0.1:49874, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.272 TRCE incoming request, uri: /newextent/8/db, method: GET, req_id: 02cb42d2-67eb-466e-b9b5-a8e0cee4e3fa, remote_addr: 127.0.0.1:49874, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.273 INFO request completed, latency_us: 295, response_code: 200, uri: /newextent/8/db, method: GET, req_id: 02cb42d2-67eb-466e-b9b5-a8e0cee4e3fa, remote_addr: 127.0.0.1:49874, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.274 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/008.copy" to "/tmp/downstairs-vrx8aK6L/00/000/008.replace"
Sep 22 23:15:16.274 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:16.275 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/008.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:16.275 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/008"
Sep 22 23:15:16.275 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/008.db"
Sep 22 23:15:16.275 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:16.275 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/008.replace" to "/tmp/downstairs-vrx8aK6L/00/000/008.completed"
Sep 22 23:15:16.275 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:16.275 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:16.275 DEBG [0] It's time to notify for 646
Sep 22 23:15:16.275 INFO Completion from [0] id:646 status:true
Sep 22 23:15:16.276 INFO [647/752] Repair commands completed
Sep 22 23:15:16.276 INFO Pop front: ReconcileIO { id: ReconciliationId(647), op: ExtentReopen { repair_id: ReconciliationId(647), extent_id: 8 }, state: ClientData([New, New, New]) }
Sep 22 23:15:16.276 INFO Sent repair work, now wait for resp
Sep 22 23:15:16.276 INFO [0] received reconcile message
Sep 22 23:15:16.276 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(647), op: ExtentReopen { repair_id: ReconciliationId(647), extent_id: 8 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:16.276 INFO [0] client ExtentReopen { repair_id: ReconciliationId(647), extent_id: 8 }
Sep 22 23:15:16.276 INFO [1] received reconcile message
Sep 22 23:15:16.276 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(647), op: ExtentReopen { repair_id: ReconciliationId(647), extent_id: 8 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:16.276 INFO [1] client ExtentReopen { repair_id: ReconciliationId(647), extent_id: 8 }
Sep 22 23:15:16.276 INFO [2] received reconcile message
Sep 22 23:15:16.276 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(647), op: ExtentReopen { repair_id: ReconciliationId(647), extent_id: 8 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:16.276 INFO [2] client ExtentReopen { repair_id: ReconciliationId(647), extent_id: 8 }
Sep 22 23:15:16.276 DEBG 647 Reopen extent 8
Sep 22 23:15:16.277 DEBG 647 Reopen extent 8
Sep 22 23:15:16.277 DEBG 647 Reopen extent 8
Sep 22 23:15:16.278 DEBG [2] It's time to notify for 647
Sep 22 23:15:16.278 INFO Completion from [2] id:647 status:true
Sep 22 23:15:16.278 INFO [648/752] Repair commands completed
Sep 22 23:15:16.278 INFO Pop front: ReconcileIO { id: ReconciliationId(648), op: ExtentFlush { repair_id: ReconciliationId(648), extent_id: 1, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:16.278 INFO Sent repair work, now wait for resp
Sep 22 23:15:16.278 INFO [0] received reconcile message
Sep 22 23:15:16.278 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(648), op: ExtentFlush { repair_id: ReconciliationId(648), extent_id: 1, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:16.278 INFO [0] client ExtentFlush { repair_id: ReconciliationId(648), extent_id: 1, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:16.278 INFO [1] received reconcile message
Sep 22 23:15:16.278 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(648), op: ExtentFlush { repair_id: ReconciliationId(648), extent_id: 1, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:16.278 INFO [1] client ExtentFlush { repair_id: ReconciliationId(648), extent_id: 1, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:16.278 INFO [2] received reconcile message
Sep 22 23:15:16.278 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(648), op: ExtentFlush { repair_id: ReconciliationId(648), extent_id: 1, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:16.278 INFO [2] client ExtentFlush { repair_id: ReconciliationId(648), extent_id: 1, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:16.278 DEBG 648 Flush extent 1 with f:2 g:2
Sep 22 23:15:16.278 DEBG Flush just extent 1 with f:2 and g:2
Sep 22 23:15:16.278 DEBG [1] It's time to notify for 648
Sep 22 23:15:16.278 INFO Completion from [1] id:648 status:true
Sep 22 23:15:16.278 INFO [649/752] Repair commands completed
Sep 22 23:15:16.278 INFO Pop front: ReconcileIO { id: ReconciliationId(649), op: ExtentClose { repair_id: ReconciliationId(649), extent_id: 1 }, state: ClientData([New, New, New]) }
Sep 22 23:15:16.278 INFO Sent repair work, now wait for resp
Sep 22 23:15:16.278 INFO [0] received reconcile message
Sep 22 23:15:16.278 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(649), op: ExtentClose { repair_id: ReconciliationId(649), extent_id: 1 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:16.278 INFO [0] client ExtentClose { repair_id: ReconciliationId(649), extent_id: 1 }
Sep 22 23:15:16.278 INFO [1] received reconcile message
Sep 22 23:15:16.278 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(649), op: ExtentClose { repair_id: ReconciliationId(649), extent_id: 1 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:16.278 INFO [1] client ExtentClose { repair_id: ReconciliationId(649), extent_id: 1 }
Sep 22 23:15:16.279 INFO [2] received reconcile message
Sep 22 23:15:16.279 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(649), op: ExtentClose { repair_id: ReconciliationId(649), extent_id: 1 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:16.279 INFO [2] client ExtentClose { repair_id: ReconciliationId(649), extent_id: 1 }
Sep 22 23:15:16.279 DEBG 649 Close extent 1
Sep 22 23:15:16.279 DEBG 649 Close extent 1
Sep 22 23:15:16.279 DEBG 649 Close extent 1
Sep 22 23:15:16.280 DEBG [2] It's time to notify for 649
Sep 22 23:15:16.280 INFO Completion from [2] id:649 status:true
Sep 22 23:15:16.280 INFO [650/752] Repair commands completed
Sep 22 23:15:16.280 INFO Pop front: ReconcileIO { id: ReconciliationId(650), op: ExtentRepair { repair_id: ReconciliationId(650), extent_id: 1, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:16.280 INFO Sent repair work, now wait for resp
Sep 22 23:15:16.280 INFO [0] received reconcile message
Sep 22 23:15:16.280 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(650), op: ExtentRepair { repair_id: ReconciliationId(650), extent_id: 1, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:16.280 INFO [0] client ExtentRepair { repair_id: ReconciliationId(650), extent_id: 1, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:16.280 INFO [0] Sending repair request ReconciliationId(650)
Sep 22 23:15:16.280 INFO [1] received reconcile message
Sep 22 23:15:16.280 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(650), op: ExtentRepair { repair_id: ReconciliationId(650), extent_id: 1, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:16.280 INFO [1] client ExtentRepair { repair_id: ReconciliationId(650), extent_id: 1, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:16.280 INFO [1] No action required ReconciliationId(650)
Sep 22 23:15:16.280 INFO [2] received reconcile message
Sep 22 23:15:16.280 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(650), op: ExtentRepair { repair_id: ReconciliationId(650), extent_id: 1, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:16.280 INFO [2] client ExtentRepair { repair_id: ReconciliationId(650), extent_id: 1, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:16.280 INFO [2] No action required ReconciliationId(650)
Sep 22 23:15:16.280 DEBG 650 Repair extent 1 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:16.280 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/001.copy"
Sep 22 23:15:16.286 DEBG [rc] retire 1089 clears [JobId(1088), JobId(1089)], : downstairs
Sep 22 23:15:16.286 ERRO [2] job id 1090 saw error GenericError("test error")
Sep 22 23:15:16.289 DEBG Flush     :1091 extent_limit None deps:[JobId(1090), JobId(1089)] res:true f:35 g:1
Sep 22 23:15:16.289 INFO [lossy] sleeping 1 second
Sep 22 23:15:16.345 INFO accepted connection, remote_addr: 127.0.0.1:47943, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.345 TRCE incoming request, uri: /extent/1/files, method: GET, req_id: 21b0536f-f9f9-48fd-85f5-4d3a62dfb386, remote_addr: 127.0.0.1:47943, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.345 INFO request completed, latency_us: 257, response_code: 200, uri: /extent/1/files, method: GET, req_id: 21b0536f-f9f9-48fd-85f5-4d3a62dfb386, remote_addr: 127.0.0.1:47943, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.346 INFO eid:1 Found repair files: ["001", "001.db"]
Sep 22 23:15:16.346 TRCE incoming request, uri: /newextent/1/data, method: GET, req_id: 099aa364-6ce9-4744-b821-635d8c0c9c70, remote_addr: 127.0.0.1:47943, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.346 INFO request completed, latency_us: 340, response_code: 200, uri: /newextent/1/data, method: GET, req_id: 099aa364-6ce9-4744-b821-635d8c0c9c70, remote_addr: 127.0.0.1:47943, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.352 TRCE incoming request, uri: /newextent/1/db, method: GET, req_id: e94a52f8-8fb6-4a09-b878-a36d99780615, remote_addr: 127.0.0.1:47943, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.352 INFO request completed, latency_us: 294, response_code: 200, uri: /newextent/1/db, method: GET, req_id: e94a52f8-8fb6-4a09-b878-a36d99780615, remote_addr: 127.0.0.1:47943, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.353 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/001.copy" to "/tmp/downstairs-vrx8aK6L/00/000/001.replace"
Sep 22 23:15:16.353 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:16.354 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/001.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:16.354 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/001"
Sep 22 23:15:16.354 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/001.db"
Sep 22 23:15:16.354 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:16.354 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/001.replace" to "/tmp/downstairs-vrx8aK6L/00/000/001.completed"
Sep 22 23:15:16.354 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:16.355 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:16.355 DEBG [0] It's time to notify for 650
Sep 22 23:15:16.355 INFO Completion from [0] id:650 status:true
Sep 22 23:15:16.355 INFO [651/752] Repair commands completed
Sep 22 23:15:16.355 INFO Pop front: ReconcileIO { id: ReconciliationId(651), op: ExtentReopen { repair_id: ReconciliationId(651), extent_id: 1 }, state: ClientData([New, New, New]) }
Sep 22 23:15:16.355 INFO Sent repair work, now wait for resp
Sep 22 23:15:16.355 INFO [0] received reconcile message
Sep 22 23:15:16.355 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(651), op: ExtentReopen { repair_id: ReconciliationId(651), extent_id: 1 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:16.355 INFO [0] client ExtentReopen { repair_id: ReconciliationId(651), extent_id: 1 }
Sep 22 23:15:16.355 INFO [1] received reconcile message
Sep 22 23:15:16.355 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(651), op: ExtentReopen { repair_id: ReconciliationId(651), extent_id: 1 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:16.355 INFO [1] client ExtentReopen { repair_id: ReconciliationId(651), extent_id: 1 }
Sep 22 23:15:16.355 INFO [2] received reconcile message
Sep 22 23:15:16.355 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(651), op: ExtentReopen { repair_id: ReconciliationId(651), extent_id: 1 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:16.355 INFO [2] client ExtentReopen { repair_id: ReconciliationId(651), extent_id: 1 }
Sep 22 23:15:16.355 DEBG 651 Reopen extent 1
Sep 22 23:15:16.356 DEBG 651 Reopen extent 1
Sep 22 23:15:16.357 DEBG 651 Reopen extent 1
Sep 22 23:15:16.357 DEBG [2] It's time to notify for 651
Sep 22 23:15:16.357 INFO Completion from [2] id:651 status:true
Sep 22 23:15:16.357 INFO [652/752] Repair commands completed
Sep 22 23:15:16.357 INFO Pop front: ReconcileIO { id: ReconciliationId(652), op: ExtentFlush { repair_id: ReconciliationId(652), extent_id: 9, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:16.357 INFO Sent repair work, now wait for resp
Sep 22 23:15:16.357 INFO [0] received reconcile message
Sep 22 23:15:16.357 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(652), op: ExtentFlush { repair_id: ReconciliationId(652), extent_id: 9, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:16.357 INFO [0] client ExtentFlush { repair_id: ReconciliationId(652), extent_id: 9, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:16.357 INFO [1] received reconcile message
Sep 22 23:15:16.357 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(652), op: ExtentFlush { repair_id: ReconciliationId(652), extent_id: 9, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:16.357 INFO [1] client ExtentFlush { repair_id: ReconciliationId(652), extent_id: 9, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:16.358 INFO [2] received reconcile message
Sep 22 23:15:16.358 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(652), op: ExtentFlush { repair_id: ReconciliationId(652), extent_id: 9, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:16.358 INFO [2] client ExtentFlush { repair_id: ReconciliationId(652), extent_id: 9, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:16.358 DEBG 652 Flush extent 9 with f:2 g:2
Sep 22 23:15:16.358 DEBG Flush just extent 9 with f:2 and g:2
Sep 22 23:15:16.358 DEBG [1] It's time to notify for 652
Sep 22 23:15:16.358 INFO Completion from [1] id:652 status:true
Sep 22 23:15:16.358 INFO [653/752] Repair commands completed
Sep 22 23:15:16.358 INFO Pop front: ReconcileIO { id: ReconciliationId(653), op: ExtentClose { repair_id: ReconciliationId(653), extent_id: 9 }, state: ClientData([New, New, New]) }
Sep 22 23:15:16.358 INFO Sent repair work, now wait for resp
Sep 22 23:15:16.358 INFO [0] received reconcile message
Sep 22 23:15:16.358 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(653), op: ExtentClose { repair_id: ReconciliationId(653), extent_id: 9 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:16.358 INFO [0] client ExtentClose { repair_id: ReconciliationId(653), extent_id: 9 }
Sep 22 23:15:16.358 INFO [1] received reconcile message
Sep 22 23:15:16.358 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(653), op: ExtentClose { repair_id: ReconciliationId(653), extent_id: 9 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:16.358 INFO [1] client ExtentClose { repair_id: ReconciliationId(653), extent_id: 9 }
Sep 22 23:15:16.358 INFO [2] received reconcile message
Sep 22 23:15:16.358 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(653), op: ExtentClose { repair_id: ReconciliationId(653), extent_id: 9 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:16.358 INFO [2] client ExtentClose { repair_id: ReconciliationId(653), extent_id: 9 }
Sep 22 23:15:16.358 DEBG 653 Close extent 9
Sep 22 23:15:16.359 DEBG 653 Close extent 9
Sep 22 23:15:16.359 DEBG 653 Close extent 9
Sep 22 23:15:16.359 DEBG [2] It's time to notify for 653
Sep 22 23:15:16.359 INFO Completion from [2] id:653 status:true
Sep 22 23:15:16.359 INFO [654/752] Repair commands completed
Sep 22 23:15:16.359 INFO Pop front: ReconcileIO { id: ReconciliationId(654), op: ExtentRepair { repair_id: ReconciliationId(654), extent_id: 9, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:16.359 INFO Sent repair work, now wait for resp
Sep 22 23:15:16.359 INFO [0] received reconcile message
Sep 22 23:15:16.359 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(654), op: ExtentRepair { repair_id: ReconciliationId(654), extent_id: 9, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:16.359 INFO [0] client ExtentRepair { repair_id: ReconciliationId(654), extent_id: 9, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:16.359 INFO [0] Sending repair request ReconciliationId(654)
Sep 22 23:15:16.359 INFO [1] received reconcile message
Sep 22 23:15:16.359 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(654), op: ExtentRepair { repair_id: ReconciliationId(654), extent_id: 9, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:16.360 INFO [1] client ExtentRepair { repair_id: ReconciliationId(654), extent_id: 9, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:16.360 INFO [1] No action required ReconciliationId(654)
Sep 22 23:15:16.360 INFO [2] received reconcile message
Sep 22 23:15:16.360 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(654), op: ExtentRepair { repair_id: ReconciliationId(654), extent_id: 9, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:16.360 INFO [2] client ExtentRepair { repair_id: ReconciliationId(654), extent_id: 9, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:16.360 INFO [2] No action required ReconciliationId(654)
Sep 22 23:15:16.360 DEBG 654 Repair extent 9 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:16.360 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/009.copy"
Sep 22 23:15:16.425 INFO accepted connection, remote_addr: 127.0.0.1:46226, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.425 TRCE incoming request, uri: /extent/9/files, method: GET, req_id: 13bddf51-ee2a-4be0-9242-83821ad227eb, remote_addr: 127.0.0.1:46226, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.425 INFO request completed, latency_us: 272, response_code: 200, uri: /extent/9/files, method: GET, req_id: 13bddf51-ee2a-4be0-9242-83821ad227eb, remote_addr: 127.0.0.1:46226, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.426 INFO eid:9 Found repair files: ["009", "009.db"]
Sep 22 23:15:16.426 TRCE incoming request, uri: /newextent/9/data, method: GET, req_id: cd5418d7-9c7c-4a7e-9f2e-ffd837f2787c, remote_addr: 127.0.0.1:46226, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.426 INFO request completed, latency_us: 351, response_code: 200, uri: /newextent/9/data, method: GET, req_id: cd5418d7-9c7c-4a7e-9f2e-ffd837f2787c, remote_addr: 127.0.0.1:46226, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.432 TRCE incoming request, uri: /newextent/9/db, method: GET, req_id: 5e6097a3-0d66-4724-96dd-18f20bc7e176, remote_addr: 127.0.0.1:46226, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.432 INFO request completed, latency_us: 298, response_code: 200, uri: /newextent/9/db, method: GET, req_id: 5e6097a3-0d66-4724-96dd-18f20bc7e176, remote_addr: 127.0.0.1:46226, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.433 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/009.copy" to "/tmp/downstairs-vrx8aK6L/00/000/009.replace"
Sep 22 23:15:16.433 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:16.435 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/009.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:16.435 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/009"
Sep 22 23:15:16.435 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/009.db"
Sep 22 23:15:16.435 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:16.435 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/009.replace" to "/tmp/downstairs-vrx8aK6L/00/000/009.completed"
Sep 22 23:15:16.435 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:16.435 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:16.435 DEBG [0] It's time to notify for 654
Sep 22 23:15:16.436 INFO Completion from [0] id:654 status:true
Sep 22 23:15:16.436 INFO [655/752] Repair commands completed
Sep 22 23:15:16.436 INFO Pop front: ReconcileIO { id: ReconciliationId(655), op: ExtentReopen { repair_id: ReconciliationId(655), extent_id: 9 }, state: ClientData([New, New, New]) }
Sep 22 23:15:16.436 INFO Sent repair work, now wait for resp
Sep 22 23:15:16.436 INFO [0] received reconcile message
Sep 22 23:15:16.436 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(655), op: ExtentReopen { repair_id: ReconciliationId(655), extent_id: 9 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:16.436 INFO [0] client ExtentReopen { repair_id: ReconciliationId(655), extent_id: 9 }
Sep 22 23:15:16.436 INFO [1] received reconcile message
Sep 22 23:15:16.436 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(655), op: ExtentReopen { repair_id: ReconciliationId(655), extent_id: 9 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:16.436 INFO [1] client ExtentReopen { repair_id: ReconciliationId(655), extent_id: 9 }
Sep 22 23:15:16.436 INFO [2] received reconcile message
Sep 22 23:15:16.436 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(655), op: ExtentReopen { repair_id: ReconciliationId(655), extent_id: 9 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:16.436 INFO [2] client ExtentReopen { repair_id: ReconciliationId(655), extent_id: 9 }
Sep 22 23:15:16.436 DEBG 655 Reopen extent 9
Sep 22 23:15:16.437 DEBG 655 Reopen extent 9
Sep 22 23:15:16.437 DEBG 655 Reopen extent 9
Sep 22 23:15:16.438 DEBG [2] It's time to notify for 655
Sep 22 23:15:16.438 INFO Completion from [2] id:655 status:true
Sep 22 23:15:16.438 INFO [656/752] Repair commands completed
Sep 22 23:15:16.438 INFO Pop front: ReconcileIO { id: ReconciliationId(656), op: ExtentFlush { repair_id: ReconciliationId(656), extent_id: 116, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:16.438 INFO Sent repair work, now wait for resp
Sep 22 23:15:16.438 INFO [0] received reconcile message
Sep 22 23:15:16.438 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(656), op: ExtentFlush { repair_id: ReconciliationId(656), extent_id: 116, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:16.438 INFO [0] client ExtentFlush { repair_id: ReconciliationId(656), extent_id: 116, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:16.438 INFO [1] received reconcile message
Sep 22 23:15:16.438 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(656), op: ExtentFlush { repair_id: ReconciliationId(656), extent_id: 116, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:16.438 INFO [1] client ExtentFlush { repair_id: ReconciliationId(656), extent_id: 116, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:16.438 INFO [2] received reconcile message
Sep 22 23:15:16.438 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(656), op: ExtentFlush { repair_id: ReconciliationId(656), extent_id: 116, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:16.438 INFO [2] client ExtentFlush { repair_id: ReconciliationId(656), extent_id: 116, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:16.438 DEBG 656 Flush extent 116 with f:2 g:2
Sep 22 23:15:16.438 DEBG Flush just extent 116 with f:2 and g:2
Sep 22 23:15:16.439 DEBG [1] It's time to notify for 656
Sep 22 23:15:16.439 INFO Completion from [1] id:656 status:true
Sep 22 23:15:16.439 INFO [657/752] Repair commands completed
Sep 22 23:15:16.439 INFO Pop front: ReconcileIO { id: ReconciliationId(657), op: ExtentClose { repair_id: ReconciliationId(657), extent_id: 116 }, state: ClientData([New, New, New]) }
Sep 22 23:15:16.439 INFO Sent repair work, now wait for resp
Sep 22 23:15:16.439 INFO [0] received reconcile message
Sep 22 23:15:16.439 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(657), op: ExtentClose { repair_id: ReconciliationId(657), extent_id: 116 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:16.439 INFO [0] client ExtentClose { repair_id: ReconciliationId(657), extent_id: 116 }
Sep 22 23:15:16.439 INFO [1] received reconcile message
Sep 22 23:15:16.439 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(657), op: ExtentClose { repair_id: ReconciliationId(657), extent_id: 116 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:16.439 INFO [1] client ExtentClose { repair_id: ReconciliationId(657), extent_id: 116 }
Sep 22 23:15:16.439 INFO [2] received reconcile message
Sep 22 23:15:16.439 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(657), op: ExtentClose { repair_id: ReconciliationId(657), extent_id: 116 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:16.439 INFO [2] client ExtentClose { repair_id: ReconciliationId(657), extent_id: 116 }
Sep 22 23:15:16.439 DEBG 657 Close extent 116
Sep 22 23:15:16.439 DEBG 657 Close extent 116
Sep 22 23:15:16.440 DEBG 657 Close extent 116
Sep 22 23:15:16.440 DEBG [2] It's time to notify for 657
Sep 22 23:15:16.440 INFO Completion from [2] id:657 status:true
Sep 22 23:15:16.440 INFO [658/752] Repair commands completed
Sep 22 23:15:16.440 INFO Pop front: ReconcileIO { id: ReconciliationId(658), op: ExtentRepair { repair_id: ReconciliationId(658), extent_id: 116, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:16.440 INFO Sent repair work, now wait for resp
Sep 22 23:15:16.440 INFO [0] received reconcile message
Sep 22 23:15:16.440 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(658), op: ExtentRepair { repair_id: ReconciliationId(658), extent_id: 116, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:16.440 INFO [0] client ExtentRepair { repair_id: ReconciliationId(658), extent_id: 116, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:16.440 INFO [0] Sending repair request ReconciliationId(658)
Sep 22 23:15:16.440 INFO [1] received reconcile message
Sep 22 23:15:16.440 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(658), op: ExtentRepair { repair_id: ReconciliationId(658), extent_id: 116, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:16.440 INFO [1] client ExtentRepair { repair_id: ReconciliationId(658), extent_id: 116, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:16.440 INFO [1] No action required ReconciliationId(658)
Sep 22 23:15:16.440 INFO [2] received reconcile message
Sep 22 23:15:16.440 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(658), op: ExtentRepair { repair_id: ReconciliationId(658), extent_id: 116, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:16.440 INFO [2] client ExtentRepair { repair_id: ReconciliationId(658), extent_id: 116, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:16.440 INFO [2] No action required ReconciliationId(658)
Sep 22 23:15:16.441 DEBG 658 Repair extent 116 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:16.441 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/074.copy"
Sep 22 23:15:16.506 INFO accepted connection, remote_addr: 127.0.0.1:61137, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.506 TRCE incoming request, uri: /extent/116/files, method: GET, req_id: f65f8e95-a997-4331-a23f-8621bf7a91f5, remote_addr: 127.0.0.1:61137, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.506 INFO request completed, latency_us: 274, response_code: 200, uri: /extent/116/files, method: GET, req_id: f65f8e95-a997-4331-a23f-8621bf7a91f5, remote_addr: 127.0.0.1:61137, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.507 INFO eid:116 Found repair files: ["074", "074.db"]
Sep 22 23:15:16.507 TRCE incoming request, uri: /newextent/116/data, method: GET, req_id: 65cd57c7-dad1-4a99-915b-24a616fd0215, remote_addr: 127.0.0.1:61137, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.507 INFO request completed, latency_us: 372, response_code: 200, uri: /newextent/116/data, method: GET, req_id: 65cd57c7-dad1-4a99-915b-24a616fd0215, remote_addr: 127.0.0.1:61137, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.512 TRCE incoming request, uri: /newextent/116/db, method: GET, req_id: d2d407ce-d434-4429-8976-4789d4f6c234, remote_addr: 127.0.0.1:61137, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.513 INFO request completed, latency_us: 294, response_code: 200, uri: /newextent/116/db, method: GET, req_id: d2d407ce-d434-4429-8976-4789d4f6c234, remote_addr: 127.0.0.1:61137, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.514 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/074.copy" to "/tmp/downstairs-vrx8aK6L/00/000/074.replace"
Sep 22 23:15:16.514 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:16.515 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/074.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:16.516 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/074"
Sep 22 23:15:16.516 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/074.db"
Sep 22 23:15:16.516 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:16.516 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/074.replace" to "/tmp/downstairs-vrx8aK6L/00/000/074.completed"
Sep 22 23:15:16.516 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:16.516 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:16.516 DEBG [0] It's time to notify for 658
Sep 22 23:15:16.516 INFO Completion from [0] id:658 status:true
Sep 22 23:15:16.516 INFO [659/752] Repair commands completed
Sep 22 23:15:16.516 INFO Pop front: ReconcileIO { id: ReconciliationId(659), op: ExtentReopen { repair_id: ReconciliationId(659), extent_id: 116 }, state: ClientData([New, New, New]) }
Sep 22 23:15:16.516 INFO Sent repair work, now wait for resp
Sep 22 23:15:16.516 INFO [0] received reconcile message
Sep 22 23:15:16.516 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(659), op: ExtentReopen { repair_id: ReconciliationId(659), extent_id: 116 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:16.516 INFO [0] client ExtentReopen { repair_id: ReconciliationId(659), extent_id: 116 }
Sep 22 23:15:16.516 INFO [1] received reconcile message
Sep 22 23:15:16.516 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(659), op: ExtentReopen { repair_id: ReconciliationId(659), extent_id: 116 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:16.516 INFO [1] client ExtentReopen { repair_id: ReconciliationId(659), extent_id: 116 }
Sep 22 23:15:16.516 INFO [2] received reconcile message
Sep 22 23:15:16.517 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(659), op: ExtentReopen { repair_id: ReconciliationId(659), extent_id: 116 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:16.517 INFO [2] client ExtentReopen { repair_id: ReconciliationId(659), extent_id: 116 }
Sep 22 23:15:16.517 DEBG 659 Reopen extent 116
Sep 22 23:15:16.517 DEBG 659 Reopen extent 116
Sep 22 23:15:16.518 DEBG 659 Reopen extent 116
Sep 22 23:15:16.519 DEBG [2] It's time to notify for 659
Sep 22 23:15:16.519 INFO Completion from [2] id:659 status:true
Sep 22 23:15:16.519 INFO [660/752] Repair commands completed
Sep 22 23:15:16.519 INFO Pop front: ReconcileIO { id: ReconciliationId(660), op: ExtentFlush { repair_id: ReconciliationId(660), extent_id: 131, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:16.519 INFO Sent repair work, now wait for resp
Sep 22 23:15:16.519 INFO [0] received reconcile message
Sep 22 23:15:16.519 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(660), op: ExtentFlush { repair_id: ReconciliationId(660), extent_id: 131, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:16.519 INFO [0] client ExtentFlush { repair_id: ReconciliationId(660), extent_id: 131, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:16.519 INFO [1] received reconcile message
Sep 22 23:15:16.519 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(660), op: ExtentFlush { repair_id: ReconciliationId(660), extent_id: 131, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:16.519 INFO [1] client ExtentFlush { repair_id: ReconciliationId(660), extent_id: 131, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:16.519 INFO [2] received reconcile message
Sep 22 23:15:16.519 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(660), op: ExtentFlush { repair_id: ReconciliationId(660), extent_id: 131, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:16.519 INFO [2] client ExtentFlush { repair_id: ReconciliationId(660), extent_id: 131, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:16.519 DEBG 660 Flush extent 131 with f:2 g:2
Sep 22 23:15:16.519 DEBG Flush just extent 131 with f:2 and g:2
Sep 22 23:15:16.519 DEBG [1] It's time to notify for 660
Sep 22 23:15:16.519 INFO Completion from [1] id:660 status:true
Sep 22 23:15:16.519 INFO [661/752] Repair commands completed
Sep 22 23:15:16.519 INFO Pop front: ReconcileIO { id: ReconciliationId(661), op: ExtentClose { repair_id: ReconciliationId(661), extent_id: 131 }, state: ClientData([New, New, New]) }
Sep 22 23:15:16.519 INFO Sent repair work, now wait for resp
Sep 22 23:15:16.519 INFO [0] received reconcile message
Sep 22 23:15:16.519 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(661), op: ExtentClose { repair_id: ReconciliationId(661), extent_id: 131 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:16.519 INFO [0] client ExtentClose { repair_id: ReconciliationId(661), extent_id: 131 }
Sep 22 23:15:16.519 INFO [1] received reconcile message
Sep 22 23:15:16.519 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(661), op: ExtentClose { repair_id: ReconciliationId(661), extent_id: 131 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:16.519 INFO [1] client ExtentClose { repair_id: ReconciliationId(661), extent_id: 131 }
Sep 22 23:15:16.520 INFO [2] received reconcile message
Sep 22 23:15:16.520 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(661), op: ExtentClose { repair_id: ReconciliationId(661), extent_id: 131 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:16.520 INFO [2] client ExtentClose { repair_id: ReconciliationId(661), extent_id: 131 }
Sep 22 23:15:16.520 DEBG 661 Close extent 131
Sep 22 23:15:16.520 DEBG 661 Close extent 131
Sep 22 23:15:16.520 DEBG 661 Close extent 131
Sep 22 23:15:16.521 DEBG [2] It's time to notify for 661
Sep 22 23:15:16.521 INFO Completion from [2] id:661 status:true
Sep 22 23:15:16.521 INFO [662/752] Repair commands completed
Sep 22 23:15:16.521 INFO Pop front: ReconcileIO { id: ReconciliationId(662), op: ExtentRepair { repair_id: ReconciliationId(662), extent_id: 131, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:16.521 INFO Sent repair work, now wait for resp
Sep 22 23:15:16.521 INFO [0] received reconcile message
Sep 22 23:15:16.521 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(662), op: ExtentRepair { repair_id: ReconciliationId(662), extent_id: 131, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:16.521 INFO [0] client ExtentRepair { repair_id: ReconciliationId(662), extent_id: 131, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:16.521 INFO [0] Sending repair request ReconciliationId(662)
Sep 22 23:15:16.521 INFO [1] received reconcile message
Sep 22 23:15:16.521 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(662), op: ExtentRepair { repair_id: ReconciliationId(662), extent_id: 131, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:16.521 INFO [1] client ExtentRepair { repair_id: ReconciliationId(662), extent_id: 131, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:16.521 INFO [1] No action required ReconciliationId(662)
Sep 22 23:15:16.521 INFO [2] received reconcile message
Sep 22 23:15:16.521 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(662), op: ExtentRepair { repair_id: ReconciliationId(662), extent_id: 131, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:16.521 INFO [2] client ExtentRepair { repair_id: ReconciliationId(662), extent_id: 131, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:16.521 INFO [2] No action required ReconciliationId(662)
Sep 22 23:15:16.521 DEBG 662 Repair extent 131 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:16.521 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/083.copy"
Sep 22 23:15:16.584 INFO accepted connection, remote_addr: 127.0.0.1:47908, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.584 TRCE incoming request, uri: /extent/131/files, method: GET, req_id: 45630ca0-b216-4e09-a8bd-a521d00056bb, remote_addr: 127.0.0.1:47908, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.584 INFO request completed, latency_us: 236, response_code: 200, uri: /extent/131/files, method: GET, req_id: 45630ca0-b216-4e09-a8bd-a521d00056bb, remote_addr: 127.0.0.1:47908, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.585 INFO eid:131 Found repair files: ["083", "083.db"]
Sep 22 23:15:16.585 TRCE incoming request, uri: /newextent/131/data, method: GET, req_id: a4572ee0-c61e-4751-b2f4-a3b49da5d8bd, remote_addr: 127.0.0.1:47908, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.585 INFO request completed, latency_us: 326, response_code: 200, uri: /newextent/131/data, method: GET, req_id: a4572ee0-c61e-4751-b2f4-a3b49da5d8bd, remote_addr: 127.0.0.1:47908, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.590 TRCE incoming request, uri: /newextent/131/db, method: GET, req_id: 464778a9-df73-4e67-910d-0a71101a33fb, remote_addr: 127.0.0.1:47908, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.591 INFO request completed, latency_us: 295, response_code: 200, uri: /newextent/131/db, method: GET, req_id: 464778a9-df73-4e67-910d-0a71101a33fb, remote_addr: 127.0.0.1:47908, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.592 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/083.copy" to "/tmp/downstairs-vrx8aK6L/00/000/083.replace"
Sep 22 23:15:16.592 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:16.593 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/083.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:16.593 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/083"
Sep 22 23:15:16.593 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/083.db"
Sep 22 23:15:16.593 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:16.593 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/083.replace" to "/tmp/downstairs-vrx8aK6L/00/000/083.completed"
Sep 22 23:15:16.593 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:16.593 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:16.593 DEBG [0] It's time to notify for 662
Sep 22 23:15:16.594 INFO Completion from [0] id:662 status:true
Sep 22 23:15:16.594 INFO [663/752] Repair commands completed
Sep 22 23:15:16.594 INFO Pop front: ReconcileIO { id: ReconciliationId(663), op: ExtentReopen { repair_id: ReconciliationId(663), extent_id: 131 }, state: ClientData([New, New, New]) }
Sep 22 23:15:16.594 INFO Sent repair work, now wait for resp
Sep 22 23:15:16.594 INFO [0] received reconcile message
Sep 22 23:15:16.594 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(663), op: ExtentReopen { repair_id: ReconciliationId(663), extent_id: 131 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:16.594 INFO [0] client ExtentReopen { repair_id: ReconciliationId(663), extent_id: 131 }
Sep 22 23:15:16.594 INFO [1] received reconcile message
Sep 22 23:15:16.594 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(663), op: ExtentReopen { repair_id: ReconciliationId(663), extent_id: 131 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:16.594 INFO [1] client ExtentReopen { repair_id: ReconciliationId(663), extent_id: 131 }
Sep 22 23:15:16.594 INFO [2] received reconcile message
Sep 22 23:15:16.594 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(663), op: ExtentReopen { repair_id: ReconciliationId(663), extent_id: 131 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:16.594 INFO [2] client ExtentReopen { repair_id: ReconciliationId(663), extent_id: 131 }
Sep 22 23:15:16.594 DEBG 663 Reopen extent 131
Sep 22 23:15:16.595 DEBG 663 Reopen extent 131
Sep 22 23:15:16.595 DEBG 663 Reopen extent 131
Sep 22 23:15:16.596 DEBG [2] It's time to notify for 663
Sep 22 23:15:16.596 INFO Completion from [2] id:663 status:true
Sep 22 23:15:16.596 INFO [664/752] Repair commands completed
Sep 22 23:15:16.596 INFO Pop front: ReconcileIO { id: ReconciliationId(664), op: ExtentFlush { repair_id: ReconciliationId(664), extent_id: 17, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:16.596 INFO Sent repair work, now wait for resp
Sep 22 23:15:16.596 INFO [0] received reconcile message
Sep 22 23:15:16.596 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(664), op: ExtentFlush { repair_id: ReconciliationId(664), extent_id: 17, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:16.596 INFO [0] client ExtentFlush { repair_id: ReconciliationId(664), extent_id: 17, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:16.596 INFO [1] received reconcile message
Sep 22 23:15:16.596 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(664), op: ExtentFlush { repair_id: ReconciliationId(664), extent_id: 17, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:16.596 INFO [1] client ExtentFlush { repair_id: ReconciliationId(664), extent_id: 17, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:16.596 INFO [2] received reconcile message
Sep 22 23:15:16.596 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(664), op: ExtentFlush { repair_id: ReconciliationId(664), extent_id: 17, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:16.596 INFO [2] client ExtentFlush { repair_id: ReconciliationId(664), extent_id: 17, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:16.596 DEBG 664 Flush extent 17 with f:2 g:2
Sep 22 23:15:16.596 DEBG Flush just extent 17 with f:2 and g:2
Sep 22 23:15:16.596 DEBG [1] It's time to notify for 664
Sep 22 23:15:16.596 INFO Completion from [1] id:664 status:true
Sep 22 23:15:16.596 INFO [665/752] Repair commands completed
Sep 22 23:15:16.596 INFO Pop front: ReconcileIO { id: ReconciliationId(665), op: ExtentClose { repair_id: ReconciliationId(665), extent_id: 17 }, state: ClientData([New, New, New]) }
Sep 22 23:15:16.596 INFO Sent repair work, now wait for resp
Sep 22 23:15:16.596 INFO [0] received reconcile message
Sep 22 23:15:16.596 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(665), op: ExtentClose { repair_id: ReconciliationId(665), extent_id: 17 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:16.596 INFO [0] client ExtentClose { repair_id: ReconciliationId(665), extent_id: 17 }
Sep 22 23:15:16.597 INFO [1] received reconcile message
Sep 22 23:15:16.597 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(665), op: ExtentClose { repair_id: ReconciliationId(665), extent_id: 17 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:16.597 INFO [1] client ExtentClose { repair_id: ReconciliationId(665), extent_id: 17 }
Sep 22 23:15:16.597 INFO [2] received reconcile message
Sep 22 23:15:16.597 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(665), op: ExtentClose { repair_id: ReconciliationId(665), extent_id: 17 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:16.597 INFO [2] client ExtentClose { repair_id: ReconciliationId(665), extent_id: 17 }
Sep 22 23:15:16.597 DEBG 665 Close extent 17
Sep 22 23:15:16.597 DEBG 665 Close extent 17
Sep 22 23:15:16.597 DEBG 665 Close extent 17
Sep 22 23:15:16.598 DEBG [2] It's time to notify for 665
Sep 22 23:15:16.598 INFO Completion from [2] id:665 status:true
Sep 22 23:15:16.598 INFO [666/752] Repair commands completed
Sep 22 23:15:16.598 INFO Pop front: ReconcileIO { id: ReconciliationId(666), op: ExtentRepair { repair_id: ReconciliationId(666), extent_id: 17, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:16.598 INFO Sent repair work, now wait for resp
Sep 22 23:15:16.598 INFO [0] received reconcile message
Sep 22 23:15:16.598 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(666), op: ExtentRepair { repair_id: ReconciliationId(666), extent_id: 17, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:16.598 INFO [0] client ExtentRepair { repair_id: ReconciliationId(666), extent_id: 17, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:16.598 INFO [0] Sending repair request ReconciliationId(666)
Sep 22 23:15:16.598 INFO [1] received reconcile message
Sep 22 23:15:16.598 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(666), op: ExtentRepair { repair_id: ReconciliationId(666), extent_id: 17, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:16.598 INFO [1] client ExtentRepair { repair_id: ReconciliationId(666), extent_id: 17, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:16.598 INFO [1] No action required ReconciliationId(666)
Sep 22 23:15:16.598 INFO [2] received reconcile message
Sep 22 23:15:16.598 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(666), op: ExtentRepair { repair_id: ReconciliationId(666), extent_id: 17, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:16.598 INFO [2] client ExtentRepair { repair_id: ReconciliationId(666), extent_id: 17, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:16.598 INFO [2] No action required ReconciliationId(666)
Sep 22 23:15:16.598 DEBG 666 Repair extent 17 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:16.598 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/011.copy"
Sep 22 23:15:16.662 INFO accepted connection, remote_addr: 127.0.0.1:40101, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.662 TRCE incoming request, uri: /extent/17/files, method: GET, req_id: 14fcb968-9e6e-4647-aa7d-59390b0c41e5, remote_addr: 127.0.0.1:40101, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.662 INFO request completed, latency_us: 192, response_code: 200, uri: /extent/17/files, method: GET, req_id: 14fcb968-9e6e-4647-aa7d-59390b0c41e5, remote_addr: 127.0.0.1:40101, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.662 INFO eid:17 Found repair files: ["011", "011.db"]
Sep 22 23:15:16.662 TRCE incoming request, uri: /newextent/17/data, method: GET, req_id: 69007d68-50c4-45af-924e-12869dfc0279, remote_addr: 127.0.0.1:40101, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.663 INFO request completed, latency_us: 251, response_code: 200, uri: /newextent/17/data, method: GET, req_id: 69007d68-50c4-45af-924e-12869dfc0279, remote_addr: 127.0.0.1:40101, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.667 DEBG [0] Read AckReady 1090, : downstairs
Sep 22 23:15:16.668 TRCE incoming request, uri: /newextent/17/db, method: GET, req_id: 8bedfc34-b26b-4721-9294-78ea2f3adcc1, remote_addr: 127.0.0.1:40101, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.668 INFO request completed, latency_us: 297, response_code: 200, uri: /newextent/17/db, method: GET, req_id: 8bedfc34-b26b-4721-9294-78ea2f3adcc1, remote_addr: 127.0.0.1:40101, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.669 DEBG up_ds_listen was notified
Sep 22 23:15:16.669 DEBG up_ds_listen process 1090
Sep 22 23:15:16.669 DEBG [A] ack job 1090:91, : downstairs
Sep 22 23:15:16.669 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/011.copy" to "/tmp/downstairs-vrx8aK6L/00/000/011.replace"
Sep 22 23:15:16.669 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:16.670 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/011.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:16.670 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/011"
Sep 22 23:15:16.670 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/011.db"
Sep 22 23:15:16.670 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:16.670 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/011.replace" to "/tmp/downstairs-vrx8aK6L/00/000/011.completed"
Sep 22 23:15:16.670 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:16.670 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:16.671 DEBG [0] It's time to notify for 666
Sep 22 23:15:16.671 INFO Completion from [0] id:666 status:true
Sep 22 23:15:16.671 INFO [667/752] Repair commands completed
Sep 22 23:15:16.671 INFO Pop front: ReconcileIO { id: ReconciliationId(667), op: ExtentReopen { repair_id: ReconciliationId(667), extent_id: 17 }, state: ClientData([New, New, New]) }
Sep 22 23:15:16.671 INFO Sent repair work, now wait for resp
Sep 22 23:15:16.671 INFO [0] received reconcile message
Sep 22 23:15:16.671 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(667), op: ExtentReopen { repair_id: ReconciliationId(667), extent_id: 17 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:16.671 INFO [0] client ExtentReopen { repair_id: ReconciliationId(667), extent_id: 17 }
Sep 22 23:15:16.671 INFO [1] received reconcile message
Sep 22 23:15:16.671 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(667), op: ExtentReopen { repair_id: ReconciliationId(667), extent_id: 17 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:16.671 INFO [1] client ExtentReopen { repair_id: ReconciliationId(667), extent_id: 17 }
Sep 22 23:15:16.671 INFO [2] received reconcile message
Sep 22 23:15:16.671 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(667), op: ExtentReopen { repair_id: ReconciliationId(667), extent_id: 17 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:16.671 INFO [2] client ExtentReopen { repair_id: ReconciliationId(667), extent_id: 17 }
Sep 22 23:15:16.671 DEBG 667 Reopen extent 17
Sep 22 23:15:16.672 DEBG 667 Reopen extent 17
Sep 22 23:15:16.672 DEBG 667 Reopen extent 17
Sep 22 23:15:16.673 DEBG [2] It's time to notify for 667
Sep 22 23:15:16.673 INFO Completion from [2] id:667 status:true
Sep 22 23:15:16.673 INFO [668/752] Repair commands completed
Sep 22 23:15:16.673 INFO Pop front: ReconcileIO { id: ReconciliationId(668), op: ExtentFlush { repair_id: ReconciliationId(668), extent_id: 182, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:16.673 INFO Sent repair work, now wait for resp
Sep 22 23:15:16.673 INFO [0] received reconcile message
Sep 22 23:15:16.673 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(668), op: ExtentFlush { repair_id: ReconciliationId(668), extent_id: 182, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:16.673 INFO [0] client ExtentFlush { repair_id: ReconciliationId(668), extent_id: 182, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:16.673 INFO [1] received reconcile message
Sep 22 23:15:16.673 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(668), op: ExtentFlush { repair_id: ReconciliationId(668), extent_id: 182, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:16.673 INFO [1] client ExtentFlush { repair_id: ReconciliationId(668), extent_id: 182, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:16.673 INFO [2] received reconcile message
Sep 22 23:15:16.673 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(668), op: ExtentFlush { repair_id: ReconciliationId(668), extent_id: 182, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:16.673 INFO [2] client ExtentFlush { repair_id: ReconciliationId(668), extent_id: 182, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:16.673 DEBG 668 Flush extent 182 with f:2 g:2
Sep 22 23:15:16.673 DEBG Flush just extent 182 with f:2 and g:2
Sep 22 23:15:16.673 DEBG [1] It's time to notify for 668
Sep 22 23:15:16.673 INFO Completion from [1] id:668 status:true
Sep 22 23:15:16.673 INFO [669/752] Repair commands completed
Sep 22 23:15:16.674 INFO Pop front: ReconcileIO { id: ReconciliationId(669), op: ExtentClose { repair_id: ReconciliationId(669), extent_id: 182 }, state: ClientData([New, New, New]) }
Sep 22 23:15:16.674 INFO Sent repair work, now wait for resp
Sep 22 23:15:16.674 INFO [0] received reconcile message
Sep 22 23:15:16.674 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(669), op: ExtentClose { repair_id: ReconciliationId(669), extent_id: 182 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:16.674 INFO [0] client ExtentClose { repair_id: ReconciliationId(669), extent_id: 182 }
Sep 22 23:15:16.674 INFO [1] received reconcile message
Sep 22 23:15:16.674 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(669), op: ExtentClose { repair_id: ReconciliationId(669), extent_id: 182 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:16.674 INFO [1] client ExtentClose { repair_id: ReconciliationId(669), extent_id: 182 }
Sep 22 23:15:16.674 INFO [2] received reconcile message
Sep 22 23:15:16.674 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(669), op: ExtentClose { repair_id: ReconciliationId(669), extent_id: 182 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:16.674 INFO [2] client ExtentClose { repair_id: ReconciliationId(669), extent_id: 182 }
Sep 22 23:15:16.674 DEBG 669 Close extent 182
Sep 22 23:15:16.674 DEBG 669 Close extent 182
Sep 22 23:15:16.674 DEBG 669 Close extent 182
Sep 22 23:15:16.675 DEBG [2] It's time to notify for 669
Sep 22 23:15:16.675 INFO Completion from [2] id:669 status:true
Sep 22 23:15:16.675 INFO [670/752] Repair commands completed
Sep 22 23:15:16.675 INFO Pop front: ReconcileIO { id: ReconciliationId(670), op: ExtentRepair { repair_id: ReconciliationId(670), extent_id: 182, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:16.675 INFO Sent repair work, now wait for resp
Sep 22 23:15:16.675 INFO [0] received reconcile message
Sep 22 23:15:16.675 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(670), op: ExtentRepair { repair_id: ReconciliationId(670), extent_id: 182, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:16.675 INFO [0] client ExtentRepair { repair_id: ReconciliationId(670), extent_id: 182, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:16.675 INFO [0] Sending repair request ReconciliationId(670)
Sep 22 23:15:16.675 INFO [1] received reconcile message
Sep 22 23:15:16.675 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(670), op: ExtentRepair { repair_id: ReconciliationId(670), extent_id: 182, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:16.675 INFO [1] client ExtentRepair { repair_id: ReconciliationId(670), extent_id: 182, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:16.675 INFO [1] No action required ReconciliationId(670)
Sep 22 23:15:16.675 INFO [2] received reconcile message
Sep 22 23:15:16.675 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(670), op: ExtentRepair { repair_id: ReconciliationId(670), extent_id: 182, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:16.675 INFO [2] client ExtentRepair { repair_id: ReconciliationId(670), extent_id: 182, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:16.675 INFO [2] No action required ReconciliationId(670)
Sep 22 23:15:16.675 DEBG 670 Repair extent 182 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:16.675 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/0B6.copy"
Sep 22 23:15:16.722 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:16.724 DEBG Flush     :1091 extent_limit None deps:[JobId(1090), JobId(1089)] res:true f:35 g:1
Sep 22 23:15:16.724 INFO [lossy] sleeping 1 second
Sep 22 23:15:16.724 INFO [lossy] skipping 1090
Sep 22 23:15:16.730 DEBG Read      :1090 deps:[JobId(1089)] res:true
Sep 22 23:15:16.739 INFO accepted connection, remote_addr: 127.0.0.1:52509, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.740 TRCE incoming request, uri: /extent/182/files, method: GET, req_id: 9da082b9-3bb5-4b5d-b108-b999f01798eb, remote_addr: 127.0.0.1:52509, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.740 INFO request completed, latency_us: 192, response_code: 200, uri: /extent/182/files, method: GET, req_id: 9da082b9-3bb5-4b5d-b108-b999f01798eb, remote_addr: 127.0.0.1:52509, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.740 INFO eid:182 Found repair files: ["0B6", "0B6.db"]
Sep 22 23:15:16.740 TRCE incoming request, uri: /newextent/182/data, method: GET, req_id: dc869527-e495-48b8-8c6f-6f606a5a36da, remote_addr: 127.0.0.1:52509, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.741 INFO request completed, latency_us: 252, response_code: 200, uri: /newextent/182/data, method: GET, req_id: dc869527-e495-48b8-8c6f-6f606a5a36da, remote_addr: 127.0.0.1:52509, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.746 TRCE incoming request, uri: /newextent/182/db, method: GET, req_id: 070fe0da-36ce-497e-88d9-577a28ff67d5, remote_addr: 127.0.0.1:52509, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.746 INFO request completed, latency_us: 295, response_code: 200, uri: /newextent/182/db, method: GET, req_id: 070fe0da-36ce-497e-88d9-577a28ff67d5, remote_addr: 127.0.0.1:52509, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.747 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/0B6.copy" to "/tmp/downstairs-vrx8aK6L/00/000/0B6.replace"
Sep 22 23:15:16.747 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:16.748 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/0B6.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:16.748 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/0B6"
Sep 22 23:15:16.748 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/0B6.db"
Sep 22 23:15:16.748 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:16.749 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/0B6.replace" to "/tmp/downstairs-vrx8aK6L/00/000/0B6.completed"
Sep 22 23:15:16.749 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:16.749 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:16.749 DEBG [0] It's time to notify for 670
Sep 22 23:15:16.749 INFO Completion from [0] id:670 status:true
Sep 22 23:15:16.749 INFO [671/752] Repair commands completed
Sep 22 23:15:16.749 INFO Pop front: ReconcileIO { id: ReconciliationId(671), op: ExtentReopen { repair_id: ReconciliationId(671), extent_id: 182 }, state: ClientData([New, New, New]) }
Sep 22 23:15:16.749 INFO Sent repair work, now wait for resp
Sep 22 23:15:16.749 INFO [0] received reconcile message
Sep 22 23:15:16.749 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(671), op: ExtentReopen { repair_id: ReconciliationId(671), extent_id: 182 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:16.749 INFO [0] client ExtentReopen { repair_id: ReconciliationId(671), extent_id: 182 }
Sep 22 23:15:16.749 INFO [1] received reconcile message
Sep 22 23:15:16.749 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(671), op: ExtentReopen { repair_id: ReconciliationId(671), extent_id: 182 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:16.749 INFO [1] client ExtentReopen { repair_id: ReconciliationId(671), extent_id: 182 }
Sep 22 23:15:16.749 INFO [2] received reconcile message
Sep 22 23:15:16.749 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(671), op: ExtentReopen { repair_id: ReconciliationId(671), extent_id: 182 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:16.749 INFO [2] client ExtentReopen { repair_id: ReconciliationId(671), extent_id: 182 }
Sep 22 23:15:16.749 DEBG 671 Reopen extent 182
Sep 22 23:15:16.750 DEBG 671 Reopen extent 182
Sep 22 23:15:16.750 DEBG 671 Reopen extent 182
Sep 22 23:15:16.751 DEBG [2] It's time to notify for 671
Sep 22 23:15:16.751 INFO Completion from [2] id:671 status:true
Sep 22 23:15:16.751 INFO [672/752] Repair commands completed
Sep 22 23:15:16.751 INFO Pop front: ReconcileIO { id: ReconciliationId(672), op: ExtentFlush { repair_id: ReconciliationId(672), extent_id: 183, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:16.751 INFO Sent repair work, now wait for resp
Sep 22 23:15:16.751 INFO [0] received reconcile message
Sep 22 23:15:16.751 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(672), op: ExtentFlush { repair_id: ReconciliationId(672), extent_id: 183, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:16.751 INFO [0] client ExtentFlush { repair_id: ReconciliationId(672), extent_id: 183, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:16.751 INFO [1] received reconcile message
Sep 22 23:15:16.751 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(672), op: ExtentFlush { repair_id: ReconciliationId(672), extent_id: 183, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:16.751 INFO [1] client ExtentFlush { repair_id: ReconciliationId(672), extent_id: 183, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:16.751 INFO [2] received reconcile message
Sep 22 23:15:16.751 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(672), op: ExtentFlush { repair_id: ReconciliationId(672), extent_id: 183, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:16.751 INFO [2] client ExtentFlush { repair_id: ReconciliationId(672), extent_id: 183, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:16.751 DEBG 672 Flush extent 183 with f:2 g:2
Sep 22 23:15:16.751 DEBG Flush just extent 183 with f:2 and g:2
Sep 22 23:15:16.752 DEBG [1] It's time to notify for 672
Sep 22 23:15:16.752 INFO Completion from [1] id:672 status:true
Sep 22 23:15:16.752 INFO [673/752] Repair commands completed
Sep 22 23:15:16.752 INFO Pop front: ReconcileIO { id: ReconciliationId(673), op: ExtentClose { repair_id: ReconciliationId(673), extent_id: 183 }, state: ClientData([New, New, New]) }
Sep 22 23:15:16.752 INFO Sent repair work, now wait for resp
Sep 22 23:15:16.752 INFO [0] received reconcile message
Sep 22 23:15:16.752 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(673), op: ExtentClose { repair_id: ReconciliationId(673), extent_id: 183 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:16.752 INFO [0] client ExtentClose { repair_id: ReconciliationId(673), extent_id: 183 }
Sep 22 23:15:16.752 INFO [1] received reconcile message
Sep 22 23:15:16.752 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(673), op: ExtentClose { repair_id: ReconciliationId(673), extent_id: 183 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:16.752 INFO [1] client ExtentClose { repair_id: ReconciliationId(673), extent_id: 183 }
Sep 22 23:15:16.752 INFO [2] received reconcile message
Sep 22 23:15:16.752 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(673), op: ExtentClose { repair_id: ReconciliationId(673), extent_id: 183 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:16.752 INFO [2] client ExtentClose { repair_id: ReconciliationId(673), extent_id: 183 }
Sep 22 23:15:16.752 DEBG 673 Close extent 183
Sep 22 23:15:16.752 DEBG 673 Close extent 183
Sep 22 23:15:16.753 DEBG 673 Close extent 183
Sep 22 23:15:16.753 DEBG [2] It's time to notify for 673
Sep 22 23:15:16.753 INFO Completion from [2] id:673 status:true
Sep 22 23:15:16.753 INFO [674/752] Repair commands completed
Sep 22 23:15:16.753 INFO Pop front: ReconcileIO { id: ReconciliationId(674), op: ExtentRepair { repair_id: ReconciliationId(674), extent_id: 183, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:16.753 INFO Sent repair work, now wait for resp
Sep 22 23:15:16.753 INFO [0] received reconcile message
Sep 22 23:15:16.753 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(674), op: ExtentRepair { repair_id: ReconciliationId(674), extent_id: 183, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:16.753 INFO [0] client ExtentRepair { repair_id: ReconciliationId(674), extent_id: 183, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:16.753 INFO [0] Sending repair request ReconciliationId(674)
Sep 22 23:15:16.753 INFO [1] received reconcile message
Sep 22 23:15:16.753 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(674), op: ExtentRepair { repair_id: ReconciliationId(674), extent_id: 183, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:16.753 INFO [1] client ExtentRepair { repair_id: ReconciliationId(674), extent_id: 183, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:16.753 INFO [1] No action required ReconciliationId(674)
Sep 22 23:15:16.753 INFO [2] received reconcile message
Sep 22 23:15:16.753 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(674), op: ExtentRepair { repair_id: ReconciliationId(674), extent_id: 183, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:16.753 INFO [2] client ExtentRepair { repair_id: ReconciliationId(674), extent_id: 183, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:16.753 INFO [2] No action required ReconciliationId(674)
Sep 22 23:15:16.753 DEBG 674 Repair extent 183 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:16.753 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/0B7.copy"
Sep 22 23:15:16.800 DEBG IO Read  1092 has deps [JobId(1091)]
Sep 22 23:15:16.818 INFO accepted connection, remote_addr: 127.0.0.1:46103, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.818 TRCE incoming request, uri: /extent/183/files, method: GET, req_id: 94291c13-e9d2-40ec-b6c5-cf879ae548a4, remote_addr: 127.0.0.1:46103, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.818 INFO request completed, latency_us: 198, response_code: 200, uri: /extent/183/files, method: GET, req_id: 94291c13-e9d2-40ec-b6c5-cf879ae548a4, remote_addr: 127.0.0.1:46103, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.818 INFO eid:183 Found repair files: ["0B7", "0B7.db"]
Sep 22 23:15:16.818 TRCE incoming request, uri: /newextent/183/data, method: GET, req_id: e08f7204-495f-4d24-945a-03fcc08f64a9, remote_addr: 127.0.0.1:46103, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.819 INFO request completed, latency_us: 331, response_code: 200, uri: /newextent/183/data, method: GET, req_id: e08f7204-495f-4d24-945a-03fcc08f64a9, remote_addr: 127.0.0.1:46103, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.824 TRCE incoming request, uri: /newextent/183/db, method: GET, req_id: 596256a6-0480-480b-8bde-456ec1e40267, remote_addr: 127.0.0.1:46103, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.824 INFO request completed, latency_us: 298, response_code: 200, uri: /newextent/183/db, method: GET, req_id: 596256a6-0480-480b-8bde-456ec1e40267, remote_addr: 127.0.0.1:46103, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.825 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/0B7.copy" to "/tmp/downstairs-vrx8aK6L/00/000/0B7.replace"
Sep 22 23:15:16.825 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:16.826 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/0B7.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:16.826 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/0B7"
Sep 22 23:15:16.826 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/0B7.db"
Sep 22 23:15:16.826 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:16.827 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/0B7.replace" to "/tmp/downstairs-vrx8aK6L/00/000/0B7.completed"
Sep 22 23:15:16.827 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:16.827 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:16.827 DEBG [0] It's time to notify for 674
Sep 22 23:15:16.827 INFO Completion from [0] id:674 status:true
Sep 22 23:15:16.827 INFO [675/752] Repair commands completed
Sep 22 23:15:16.827 INFO Pop front: ReconcileIO { id: ReconciliationId(675), op: ExtentReopen { repair_id: ReconciliationId(675), extent_id: 183 }, state: ClientData([New, New, New]) }
Sep 22 23:15:16.827 INFO Sent repair work, now wait for resp
Sep 22 23:15:16.827 INFO [0] received reconcile message
Sep 22 23:15:16.827 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(675), op: ExtentReopen { repair_id: ReconciliationId(675), extent_id: 183 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:16.827 INFO [0] client ExtentReopen { repair_id: ReconciliationId(675), extent_id: 183 }
Sep 22 23:15:16.827 INFO [1] received reconcile message
Sep 22 23:15:16.827 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(675), op: ExtentReopen { repair_id: ReconciliationId(675), extent_id: 183 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:16.827 INFO [1] client ExtentReopen { repair_id: ReconciliationId(675), extent_id: 183 }
Sep 22 23:15:16.827 INFO [2] received reconcile message
Sep 22 23:15:16.827 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(675), op: ExtentReopen { repair_id: ReconciliationId(675), extent_id: 183 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:16.827 INFO [2] client ExtentReopen { repair_id: ReconciliationId(675), extent_id: 183 }
Sep 22 23:15:16.827 DEBG 675 Reopen extent 183
Sep 22 23:15:16.828 DEBG 675 Reopen extent 183
Sep 22 23:15:16.828 DEBG 675 Reopen extent 183
Sep 22 23:15:16.829 DEBG [2] It's time to notify for 675
Sep 22 23:15:16.829 INFO Completion from [2] id:675 status:true
Sep 22 23:15:16.829 INFO [676/752] Repair commands completed
Sep 22 23:15:16.829 INFO Pop front: ReconcileIO { id: ReconciliationId(676), op: ExtentFlush { repair_id: ReconciliationId(676), extent_id: 20, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:16.829 INFO Sent repair work, now wait for resp
Sep 22 23:15:16.829 INFO [0] received reconcile message
Sep 22 23:15:16.829 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(676), op: ExtentFlush { repair_id: ReconciliationId(676), extent_id: 20, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:16.829 INFO [0] client ExtentFlush { repair_id: ReconciliationId(676), extent_id: 20, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:16.829 INFO [1] received reconcile message
Sep 22 23:15:16.829 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(676), op: ExtentFlush { repair_id: ReconciliationId(676), extent_id: 20, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:16.829 INFO [1] client ExtentFlush { repair_id: ReconciliationId(676), extent_id: 20, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:16.829 INFO [2] received reconcile message
Sep 22 23:15:16.829 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(676), op: ExtentFlush { repair_id: ReconciliationId(676), extent_id: 20, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:16.829 INFO [2] client ExtentFlush { repair_id: ReconciliationId(676), extent_id: 20, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:16.829 DEBG 676 Flush extent 20 with f:2 g:2
Sep 22 23:15:16.829 DEBG Flush just extent 20 with f:2 and g:2
Sep 22 23:15:16.830 DEBG [1] It's time to notify for 676
Sep 22 23:15:16.830 INFO Completion from [1] id:676 status:true
Sep 22 23:15:16.830 INFO [677/752] Repair commands completed
Sep 22 23:15:16.830 INFO Pop front: ReconcileIO { id: ReconciliationId(677), op: ExtentClose { repair_id: ReconciliationId(677), extent_id: 20 }, state: ClientData([New, New, New]) }
Sep 22 23:15:16.830 INFO Sent repair work, now wait for resp
Sep 22 23:15:16.830 INFO [0] received reconcile message
Sep 22 23:15:16.830 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(677), op: ExtentClose { repair_id: ReconciliationId(677), extent_id: 20 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:16.830 INFO [0] client ExtentClose { repair_id: ReconciliationId(677), extent_id: 20 }
Sep 22 23:15:16.830 INFO [1] received reconcile message
Sep 22 23:15:16.830 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(677), op: ExtentClose { repair_id: ReconciliationId(677), extent_id: 20 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:16.830 INFO [1] client ExtentClose { repair_id: ReconciliationId(677), extent_id: 20 }
Sep 22 23:15:16.830 INFO [2] received reconcile message
Sep 22 23:15:16.830 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(677), op: ExtentClose { repair_id: ReconciliationId(677), extent_id: 20 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:16.830 INFO [2] client ExtentClose { repair_id: ReconciliationId(677), extent_id: 20 }
Sep 22 23:15:16.830 DEBG 677 Close extent 20
Sep 22 23:15:16.830 DEBG 677 Close extent 20
Sep 22 23:15:16.831 DEBG 677 Close extent 20
Sep 22 23:15:16.831 DEBG [2] It's time to notify for 677
Sep 22 23:15:16.831 INFO Completion from [2] id:677 status:true
Sep 22 23:15:16.831 INFO [678/752] Repair commands completed
Sep 22 23:15:16.831 INFO Pop front: ReconcileIO { id: ReconciliationId(678), op: ExtentRepair { repair_id: ReconciliationId(678), extent_id: 20, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:16.831 INFO Sent repair work, now wait for resp
Sep 22 23:15:16.831 INFO [0] received reconcile message
Sep 22 23:15:16.831 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(678), op: ExtentRepair { repair_id: ReconciliationId(678), extent_id: 20, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:16.831 INFO [0] client ExtentRepair { repair_id: ReconciliationId(678), extent_id: 20, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:16.831 INFO [0] Sending repair request ReconciliationId(678)
Sep 22 23:15:16.831 INFO [1] received reconcile message
Sep 22 23:15:16.831 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(678), op: ExtentRepair { repair_id: ReconciliationId(678), extent_id: 20, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:16.831 INFO [1] client ExtentRepair { repair_id: ReconciliationId(678), extent_id: 20, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:16.831 INFO [1] No action required ReconciliationId(678)
Sep 22 23:15:16.831 INFO [2] received reconcile message
Sep 22 23:15:16.831 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(678), op: ExtentRepair { repair_id: ReconciliationId(678), extent_id: 20, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:16.831 INFO [2] client ExtentRepair { repair_id: ReconciliationId(678), extent_id: 20, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:16.831 INFO [2] No action required ReconciliationId(678)
Sep 22 23:15:16.831 DEBG 678 Repair extent 20 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:16.832 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/014.copy"
Sep 22 23:15:16.893 INFO accepted connection, remote_addr: 127.0.0.1:57550, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.894 TRCE incoming request, uri: /extent/20/files, method: GET, req_id: 4e92c872-8ce0-463f-b09a-29f24eca1511, remote_addr: 127.0.0.1:57550, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.894 INFO request completed, latency_us: 202, response_code: 200, uri: /extent/20/files, method: GET, req_id: 4e92c872-8ce0-463f-b09a-29f24eca1511, remote_addr: 127.0.0.1:57550, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.894 INFO eid:20 Found repair files: ["014", "014.db"]
Sep 22 23:15:16.894 TRCE incoming request, uri: /newextent/20/data, method: GET, req_id: 86e68e0c-a5ff-4ed9-8062-37a6b6e231ef, remote_addr: 127.0.0.1:57550, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.895 INFO request completed, latency_us: 256, response_code: 200, uri: /newextent/20/data, method: GET, req_id: 86e68e0c-a5ff-4ed9-8062-37a6b6e231ef, remote_addr: 127.0.0.1:57550, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.900 TRCE incoming request, uri: /newextent/20/db, method: GET, req_id: dc6f4d51-f6ad-4187-8df1-9c1f24e72387, remote_addr: 127.0.0.1:57550, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.900 INFO request completed, latency_us: 298, response_code: 200, uri: /newextent/20/db, method: GET, req_id: dc6f4d51-f6ad-4187-8df1-9c1f24e72387, remote_addr: 127.0.0.1:57550, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.901 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/014.copy" to "/tmp/downstairs-vrx8aK6L/00/000/014.replace"
Sep 22 23:15:16.901 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:16.902 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/014.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:16.902 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/014"
Sep 22 23:15:16.902 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/014.db"
Sep 22 23:15:16.902 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:16.902 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/014.replace" to "/tmp/downstairs-vrx8aK6L/00/000/014.completed"
Sep 22 23:15:16.902 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:16.902 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:16.903 DEBG [0] It's time to notify for 678
Sep 22 23:15:16.903 INFO Completion from [0] id:678 status:true
Sep 22 23:15:16.903 INFO [679/752] Repair commands completed
Sep 22 23:15:16.903 INFO Pop front: ReconcileIO { id: ReconciliationId(679), op: ExtentReopen { repair_id: ReconciliationId(679), extent_id: 20 }, state: ClientData([New, New, New]) }
Sep 22 23:15:16.903 INFO Sent repair work, now wait for resp
Sep 22 23:15:16.903 INFO [0] received reconcile message
Sep 22 23:15:16.903 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(679), op: ExtentReopen { repair_id: ReconciliationId(679), extent_id: 20 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:16.903 INFO [0] client ExtentReopen { repair_id: ReconciliationId(679), extent_id: 20 }
Sep 22 23:15:16.903 INFO [1] received reconcile message
Sep 22 23:15:16.903 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(679), op: ExtentReopen { repair_id: ReconciliationId(679), extent_id: 20 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:16.903 INFO [1] client ExtentReopen { repair_id: ReconciliationId(679), extent_id: 20 }
Sep 22 23:15:16.903 INFO [2] received reconcile message
Sep 22 23:15:16.903 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(679), op: ExtentReopen { repair_id: ReconciliationId(679), extent_id: 20 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:16.903 INFO [2] client ExtentReopen { repair_id: ReconciliationId(679), extent_id: 20 }
Sep 22 23:15:16.903 DEBG 679 Reopen extent 20
Sep 22 23:15:16.904 DEBG 679 Reopen extent 20
Sep 22 23:15:16.904 DEBG 679 Reopen extent 20
Sep 22 23:15:16.905 DEBG [2] It's time to notify for 679
Sep 22 23:15:16.905 INFO Completion from [2] id:679 status:true
Sep 22 23:15:16.905 INFO [680/752] Repair commands completed
Sep 22 23:15:16.905 INFO Pop front: ReconcileIO { id: ReconciliationId(680), op: ExtentFlush { repair_id: ReconciliationId(680), extent_id: 63, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:16.905 INFO Sent repair work, now wait for resp
Sep 22 23:15:16.905 INFO [0] received reconcile message
Sep 22 23:15:16.905 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(680), op: ExtentFlush { repair_id: ReconciliationId(680), extent_id: 63, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:16.905 INFO [0] client ExtentFlush { repair_id: ReconciliationId(680), extent_id: 63, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:16.905 INFO [1] received reconcile message
Sep 22 23:15:16.905 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(680), op: ExtentFlush { repair_id: ReconciliationId(680), extent_id: 63, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:16.905 INFO [1] client ExtentFlush { repair_id: ReconciliationId(680), extent_id: 63, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:16.905 INFO [2] received reconcile message
Sep 22 23:15:16.905 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(680), op: ExtentFlush { repair_id: ReconciliationId(680), extent_id: 63, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:16.905 INFO [2] client ExtentFlush { repair_id: ReconciliationId(680), extent_id: 63, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:16.905 DEBG 680 Flush extent 63 with f:2 g:2
Sep 22 23:15:16.905 DEBG Flush just extent 63 with f:2 and g:2
Sep 22 23:15:16.905 DEBG [1] It's time to notify for 680
Sep 22 23:15:16.905 INFO Completion from [1] id:680 status:true
Sep 22 23:15:16.905 INFO [681/752] Repair commands completed
Sep 22 23:15:16.906 INFO Pop front: ReconcileIO { id: ReconciliationId(681), op: ExtentClose { repair_id: ReconciliationId(681), extent_id: 63 }, state: ClientData([New, New, New]) }
Sep 22 23:15:16.906 INFO Sent repair work, now wait for resp
Sep 22 23:15:16.906 INFO [0] received reconcile message
Sep 22 23:15:16.906 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(681), op: ExtentClose { repair_id: ReconciliationId(681), extent_id: 63 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:16.906 INFO [0] client ExtentClose { repair_id: ReconciliationId(681), extent_id: 63 }
Sep 22 23:15:16.906 INFO [1] received reconcile message
Sep 22 23:15:16.906 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(681), op: ExtentClose { repair_id: ReconciliationId(681), extent_id: 63 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:16.906 INFO [1] client ExtentClose { repair_id: ReconciliationId(681), extent_id: 63 }
Sep 22 23:15:16.906 INFO [2] received reconcile message
Sep 22 23:15:16.906 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(681), op: ExtentClose { repair_id: ReconciliationId(681), extent_id: 63 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:16.906 INFO [2] client ExtentClose { repair_id: ReconciliationId(681), extent_id: 63 }
Sep 22 23:15:16.906 DEBG 681 Close extent 63
Sep 22 23:15:16.906 DEBG 681 Close extent 63
Sep 22 23:15:16.906 DEBG 681 Close extent 63
Sep 22 23:15:16.907 DEBG [2] It's time to notify for 681
Sep 22 23:15:16.907 INFO Completion from [2] id:681 status:true
Sep 22 23:15:16.907 INFO [682/752] Repair commands completed
Sep 22 23:15:16.907 INFO Pop front: ReconcileIO { id: ReconciliationId(682), op: ExtentRepair { repair_id: ReconciliationId(682), extent_id: 63, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:16.907 INFO Sent repair work, now wait for resp
Sep 22 23:15:16.907 INFO [0] received reconcile message
Sep 22 23:15:16.907 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(682), op: ExtentRepair { repair_id: ReconciliationId(682), extent_id: 63, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:16.907 INFO [0] client ExtentRepair { repair_id: ReconciliationId(682), extent_id: 63, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:16.907 INFO [0] Sending repair request ReconciliationId(682)
Sep 22 23:15:16.907 INFO [1] received reconcile message
Sep 22 23:15:16.907 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(682), op: ExtentRepair { repair_id: ReconciliationId(682), extent_id: 63, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:16.907 INFO [1] client ExtentRepair { repair_id: ReconciliationId(682), extent_id: 63, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:16.907 INFO [1] No action required ReconciliationId(682)
Sep 22 23:15:16.907 INFO [2] received reconcile message
Sep 22 23:15:16.907 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(682), op: ExtentRepair { repair_id: ReconciliationId(682), extent_id: 63, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:16.907 INFO [2] client ExtentRepair { repair_id: ReconciliationId(682), extent_id: 63, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:16.907 INFO [2] No action required ReconciliationId(682)
Sep 22 23:15:16.907 DEBG 682 Repair extent 63 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:16.907 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/03F.copy"
Sep 22 23:15:16.971 INFO accepted connection, remote_addr: 127.0.0.1:43683, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.971 TRCE incoming request, uri: /extent/63/files, method: GET, req_id: a34b3e28-2b8e-48a1-9d9d-fc87b52e8d04, remote_addr: 127.0.0.1:43683, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.971 INFO request completed, latency_us: 189, response_code: 200, uri: /extent/63/files, method: GET, req_id: a34b3e28-2b8e-48a1-9d9d-fc87b52e8d04, remote_addr: 127.0.0.1:43683, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.971 INFO eid:63 Found repair files: ["03F", "03F.db"]
Sep 22 23:15:16.971 TRCE incoming request, uri: /newextent/63/data, method: GET, req_id: 676a94eb-4898-44c0-9f47-369eed37da50, remote_addr: 127.0.0.1:43683, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.972 INFO request completed, latency_us: 249, response_code: 200, uri: /newextent/63/data, method: GET, req_id: 676a94eb-4898-44c0-9f47-369eed37da50, remote_addr: 127.0.0.1:43683, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.977 TRCE incoming request, uri: /newextent/63/db, method: GET, req_id: 382c490f-e811-4ff8-bd2a-ccfadadbb678, remote_addr: 127.0.0.1:43683, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.977 INFO request completed, latency_us: 294, response_code: 200, uri: /newextent/63/db, method: GET, req_id: 382c490f-e811-4ff8-bd2a-ccfadadbb678, remote_addr: 127.0.0.1:43683, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:16.978 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/03F.copy" to "/tmp/downstairs-vrx8aK6L/00/000/03F.replace"
Sep 22 23:15:16.978 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:16.979 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/03F.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:16.979 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/03F"
Sep 22 23:15:16.979 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/03F.db"
Sep 22 23:15:16.979 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:16.979 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/03F.replace" to "/tmp/downstairs-vrx8aK6L/00/000/03F.completed"
Sep 22 23:15:16.979 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:16.979 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:16.980 DEBG [0] It's time to notify for 682
Sep 22 23:15:16.980 INFO Completion from [0] id:682 status:true
Sep 22 23:15:16.980 INFO [683/752] Repair commands completed
Sep 22 23:15:16.980 INFO Pop front: ReconcileIO { id: ReconciliationId(683), op: ExtentReopen { repair_id: ReconciliationId(683), extent_id: 63 }, state: ClientData([New, New, New]) }
Sep 22 23:15:16.980 INFO Sent repair work, now wait for resp
Sep 22 23:15:16.980 INFO [0] received reconcile message
Sep 22 23:15:16.980 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(683), op: ExtentReopen { repair_id: ReconciliationId(683), extent_id: 63 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:16.980 INFO [0] client ExtentReopen { repair_id: ReconciliationId(683), extent_id: 63 }
Sep 22 23:15:16.980 INFO [1] received reconcile message
Sep 22 23:15:16.980 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(683), op: ExtentReopen { repair_id: ReconciliationId(683), extent_id: 63 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:16.980 INFO [1] client ExtentReopen { repair_id: ReconciliationId(683), extent_id: 63 }
Sep 22 23:15:16.980 INFO [2] received reconcile message
Sep 22 23:15:16.980 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(683), op: ExtentReopen { repair_id: ReconciliationId(683), extent_id: 63 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:16.980 INFO [2] client ExtentReopen { repair_id: ReconciliationId(683), extent_id: 63 }
Sep 22 23:15:16.980 DEBG 683 Reopen extent 63
Sep 22 23:15:16.981 DEBG 683 Reopen extent 63
Sep 22 23:15:16.981 DEBG 683 Reopen extent 63
Sep 22 23:15:16.982 DEBG [2] It's time to notify for 683
Sep 22 23:15:16.982 INFO Completion from [2] id:683 status:true
Sep 22 23:15:16.982 INFO [684/752] Repair commands completed
Sep 22 23:15:16.982 INFO Pop front: ReconcileIO { id: ReconciliationId(684), op: ExtentFlush { repair_id: ReconciliationId(684), extent_id: 166, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:16.982 INFO Sent repair work, now wait for resp
Sep 22 23:15:16.982 INFO [0] received reconcile message
Sep 22 23:15:16.982 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(684), op: ExtentFlush { repair_id: ReconciliationId(684), extent_id: 166, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:16.982 INFO [0] client ExtentFlush { repair_id: ReconciliationId(684), extent_id: 166, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:16.982 INFO [1] received reconcile message
Sep 22 23:15:16.982 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(684), op: ExtentFlush { repair_id: ReconciliationId(684), extent_id: 166, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:16.982 INFO [1] client ExtentFlush { repair_id: ReconciliationId(684), extent_id: 166, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:16.982 INFO [2] received reconcile message
Sep 22 23:15:16.982 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(684), op: ExtentFlush { repair_id: ReconciliationId(684), extent_id: 166, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:16.982 INFO [2] client ExtentFlush { repair_id: ReconciliationId(684), extent_id: 166, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:16.982 DEBG 684 Flush extent 166 with f:2 g:2
Sep 22 23:15:16.982 DEBG Flush just extent 166 with f:2 and g:2
Sep 22 23:15:16.982 DEBG [1] It's time to notify for 684
Sep 22 23:15:16.982 INFO Completion from [1] id:684 status:true
Sep 22 23:15:16.982 INFO [685/752] Repair commands completed
Sep 22 23:15:16.983 INFO Pop front: ReconcileIO { id: ReconciliationId(685), op: ExtentClose { repair_id: ReconciliationId(685), extent_id: 166 }, state: ClientData([New, New, New]) }
Sep 22 23:15:16.983 INFO Sent repair work, now wait for resp
Sep 22 23:15:16.983 INFO [0] received reconcile message
Sep 22 23:15:16.983 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(685), op: ExtentClose { repair_id: ReconciliationId(685), extent_id: 166 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:16.983 INFO [0] client ExtentClose { repair_id: ReconciliationId(685), extent_id: 166 }
Sep 22 23:15:16.983 INFO [1] received reconcile message
Sep 22 23:15:16.983 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(685), op: ExtentClose { repair_id: ReconciliationId(685), extent_id: 166 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:16.983 INFO [1] client ExtentClose { repair_id: ReconciliationId(685), extent_id: 166 }
Sep 22 23:15:16.983 INFO [2] received reconcile message
Sep 22 23:15:16.983 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(685), op: ExtentClose { repair_id: ReconciliationId(685), extent_id: 166 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:16.983 INFO [2] client ExtentClose { repair_id: ReconciliationId(685), extent_id: 166 }
Sep 22 23:15:16.983 DEBG 685 Close extent 166
Sep 22 23:15:16.983 DEBG 685 Close extent 166
Sep 22 23:15:16.983 DEBG 685 Close extent 166
Sep 22 23:15:16.984 DEBG [2] It's time to notify for 685
Sep 22 23:15:16.984 INFO Completion from [2] id:685 status:true
Sep 22 23:15:16.984 INFO [686/752] Repair commands completed
Sep 22 23:15:16.984 INFO Pop front: ReconcileIO { id: ReconciliationId(686), op: ExtentRepair { repair_id: ReconciliationId(686), extent_id: 166, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:16.984 INFO Sent repair work, now wait for resp
Sep 22 23:15:16.984 INFO [0] received reconcile message
Sep 22 23:15:16.984 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(686), op: ExtentRepair { repair_id: ReconciliationId(686), extent_id: 166, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:16.984 INFO [0] client ExtentRepair { repair_id: ReconciliationId(686), extent_id: 166, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:16.984 INFO [0] Sending repair request ReconciliationId(686)
Sep 22 23:15:16.984 INFO [1] received reconcile message
Sep 22 23:15:16.984 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(686), op: ExtentRepair { repair_id: ReconciliationId(686), extent_id: 166, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:16.984 INFO [1] client ExtentRepair { repair_id: ReconciliationId(686), extent_id: 166, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:16.984 INFO [1] No action required ReconciliationId(686)
Sep 22 23:15:16.984 INFO [2] received reconcile message
Sep 22 23:15:16.984 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(686), op: ExtentRepair { repair_id: ReconciliationId(686), extent_id: 166, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:16.984 INFO [2] client ExtentRepair { repair_id: ReconciliationId(686), extent_id: 166, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:16.984 INFO [2] No action required ReconciliationId(686)
Sep 22 23:15:16.984 DEBG 686 Repair extent 166 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:16.984 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/0A6.copy"
Sep 22 23:15:17.046 INFO accepted connection, remote_addr: 127.0.0.1:55473, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.046 TRCE incoming request, uri: /extent/166/files, method: GET, req_id: 50c1b6e8-d5f1-4e0f-a263-1fdf3c0e1d9e, remote_addr: 127.0.0.1:55473, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.046 INFO request completed, latency_us: 202, response_code: 200, uri: /extent/166/files, method: GET, req_id: 50c1b6e8-d5f1-4e0f-a263-1fdf3c0e1d9e, remote_addr: 127.0.0.1:55473, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.047 INFO eid:166 Found repair files: ["0A6", "0A6.db"]
Sep 22 23:15:17.047 TRCE incoming request, uri: /newextent/166/data, method: GET, req_id: 9d86f1d9-4be9-428f-9aa6-fb31783fd65c, remote_addr: 127.0.0.1:55473, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.047 INFO request completed, latency_us: 312, response_code: 200, uri: /newextent/166/data, method: GET, req_id: 9d86f1d9-4be9-428f-9aa6-fb31783fd65c, remote_addr: 127.0.0.1:55473, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.053 TRCE incoming request, uri: /newextent/166/db, method: GET, req_id: 1c41826b-3af3-4b7b-86da-df7435f086f0, remote_addr: 127.0.0.1:55473, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.053 INFO request completed, latency_us: 292, response_code: 200, uri: /newextent/166/db, method: GET, req_id: 1c41826b-3af3-4b7b-86da-df7435f086f0, remote_addr: 127.0.0.1:55473, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.054 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/0A6.copy" to "/tmp/downstairs-vrx8aK6L/00/000/0A6.replace"
Sep 22 23:15:17.054 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:17.055 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/0A6.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:17.055 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/0A6"
Sep 22 23:15:17.055 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/0A6.db"
Sep 22 23:15:17.055 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:17.055 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/0A6.replace" to "/tmp/downstairs-vrx8aK6L/00/000/0A6.completed"
Sep 22 23:15:17.055 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:17.055 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:17.056 DEBG [0] It's time to notify for 686
Sep 22 23:15:17.056 INFO Completion from [0] id:686 status:true
Sep 22 23:15:17.056 INFO [687/752] Repair commands completed
Sep 22 23:15:17.056 INFO Pop front: ReconcileIO { id: ReconciliationId(687), op: ExtentReopen { repair_id: ReconciliationId(687), extent_id: 166 }, state: ClientData([New, New, New]) }
Sep 22 23:15:17.056 INFO Sent repair work, now wait for resp
Sep 22 23:15:17.056 INFO [0] received reconcile message
Sep 22 23:15:17.056 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(687), op: ExtentReopen { repair_id: ReconciliationId(687), extent_id: 166 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:17.056 INFO [0] client ExtentReopen { repair_id: ReconciliationId(687), extent_id: 166 }
Sep 22 23:15:17.056 INFO [1] received reconcile message
Sep 22 23:15:17.056 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(687), op: ExtentReopen { repair_id: ReconciliationId(687), extent_id: 166 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:17.056 INFO [1] client ExtentReopen { repair_id: ReconciliationId(687), extent_id: 166 }
Sep 22 23:15:17.056 INFO [2] received reconcile message
Sep 22 23:15:17.056 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(687), op: ExtentReopen { repair_id: ReconciliationId(687), extent_id: 166 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:17.056 INFO [2] client ExtentReopen { repair_id: ReconciliationId(687), extent_id: 166 }
Sep 22 23:15:17.056 DEBG 687 Reopen extent 166
Sep 22 23:15:17.057 DEBG 687 Reopen extent 166
Sep 22 23:15:17.057 DEBG 687 Reopen extent 166
Sep 22 23:15:17.058 DEBG [2] It's time to notify for 687
Sep 22 23:15:17.058 INFO Completion from [2] id:687 status:true
Sep 22 23:15:17.058 INFO [688/752] Repair commands completed
Sep 22 23:15:17.058 INFO Pop front: ReconcileIO { id: ReconciliationId(688), op: ExtentFlush { repair_id: ReconciliationId(688), extent_id: 14, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:17.058 INFO Sent repair work, now wait for resp
Sep 22 23:15:17.058 INFO [0] received reconcile message
Sep 22 23:15:17.058 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(688), op: ExtentFlush { repair_id: ReconciliationId(688), extent_id: 14, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:17.058 INFO [0] client ExtentFlush { repair_id: ReconciliationId(688), extent_id: 14, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:17.058 INFO [1] received reconcile message
Sep 22 23:15:17.058 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(688), op: ExtentFlush { repair_id: ReconciliationId(688), extent_id: 14, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:17.058 INFO [1] client ExtentFlush { repair_id: ReconciliationId(688), extent_id: 14, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:17.058 INFO [2] received reconcile message
Sep 22 23:15:17.058 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(688), op: ExtentFlush { repair_id: ReconciliationId(688), extent_id: 14, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:17.058 INFO [2] client ExtentFlush { repair_id: ReconciliationId(688), extent_id: 14, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:17.058 DEBG 688 Flush extent 14 with f:2 g:2
Sep 22 23:15:17.058 DEBG Flush just extent 14 with f:2 and g:2
Sep 22 23:15:17.058 DEBG [1] It's time to notify for 688
Sep 22 23:15:17.058 INFO Completion from [1] id:688 status:true
Sep 22 23:15:17.058 INFO [689/752] Repair commands completed
Sep 22 23:15:17.058 INFO Pop front: ReconcileIO { id: ReconciliationId(689), op: ExtentClose { repair_id: ReconciliationId(689), extent_id: 14 }, state: ClientData([New, New, New]) }
Sep 22 23:15:17.059 INFO Sent repair work, now wait for resp
Sep 22 23:15:17.059 INFO [0] received reconcile message
Sep 22 23:15:17.059 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(689), op: ExtentClose { repair_id: ReconciliationId(689), extent_id: 14 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:17.059 INFO [0] client ExtentClose { repair_id: ReconciliationId(689), extent_id: 14 }
Sep 22 23:15:17.059 INFO [1] received reconcile message
Sep 22 23:15:17.059 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(689), op: ExtentClose { repair_id: ReconciliationId(689), extent_id: 14 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:17.059 INFO [1] client ExtentClose { repair_id: ReconciliationId(689), extent_id: 14 }
Sep 22 23:15:17.059 INFO [2] received reconcile message
Sep 22 23:15:17.059 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(689), op: ExtentClose { repair_id: ReconciliationId(689), extent_id: 14 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:17.059 INFO [2] client ExtentClose { repair_id: ReconciliationId(689), extent_id: 14 }
Sep 22 23:15:17.059 DEBG 689 Close extent 14
Sep 22 23:15:17.059 DEBG 689 Close extent 14
Sep 22 23:15:17.059 DEBG 689 Close extent 14
Sep 22 23:15:17.060 DEBG [2] It's time to notify for 689
Sep 22 23:15:17.060 INFO Completion from [2] id:689 status:true
Sep 22 23:15:17.060 INFO [690/752] Repair commands completed
Sep 22 23:15:17.060 INFO Pop front: ReconcileIO { id: ReconciliationId(690), op: ExtentRepair { repair_id: ReconciliationId(690), extent_id: 14, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:17.060 INFO Sent repair work, now wait for resp
Sep 22 23:15:17.060 INFO [0] received reconcile message
Sep 22 23:15:17.060 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(690), op: ExtentRepair { repair_id: ReconciliationId(690), extent_id: 14, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:17.060 INFO [0] client ExtentRepair { repair_id: ReconciliationId(690), extent_id: 14, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:17.060 INFO [0] Sending repair request ReconciliationId(690)
Sep 22 23:15:17.060 INFO [1] received reconcile message
Sep 22 23:15:17.060 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(690), op: ExtentRepair { repair_id: ReconciliationId(690), extent_id: 14, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:17.060 INFO [1] client ExtentRepair { repair_id: ReconciliationId(690), extent_id: 14, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:17.060 INFO [1] No action required ReconciliationId(690)
Sep 22 23:15:17.060 INFO [2] received reconcile message
Sep 22 23:15:17.060 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(690), op: ExtentRepair { repair_id: ReconciliationId(690), extent_id: 14, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:17.060 INFO [2] client ExtentRepair { repair_id: ReconciliationId(690), extent_id: 14, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:17.060 INFO [2] No action required ReconciliationId(690)
Sep 22 23:15:17.060 DEBG 690 Repair extent 14 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:17.060 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/00E.copy"
Sep 22 23:15:17.125 INFO accepted connection, remote_addr: 127.0.0.1:45501, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.125 TRCE incoming request, uri: /extent/14/files, method: GET, req_id: f437eab1-9f72-48ea-8ad0-d672907049db, remote_addr: 127.0.0.1:45501, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.125 INFO request completed, latency_us: 198, response_code: 200, uri: /extent/14/files, method: GET, req_id: f437eab1-9f72-48ea-8ad0-d672907049db, remote_addr: 127.0.0.1:45501, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.126 INFO eid:14 Found repair files: ["00E", "00E.db"]
Sep 22 23:15:17.126 TRCE incoming request, uri: /newextent/14/data, method: GET, req_id: 6428d5a9-da5c-498c-ba01-2a00663843c1, remote_addr: 127.0.0.1:45501, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.126 INFO request completed, latency_us: 315, response_code: 200, uri: /newextent/14/data, method: GET, req_id: 6428d5a9-da5c-498c-ba01-2a00663843c1, remote_addr: 127.0.0.1:45501, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.131 TRCE incoming request, uri: /newextent/14/db, method: GET, req_id: b7a6950c-f770-4cec-8c01-1f4402e80a26, remote_addr: 127.0.0.1:45501, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.131 INFO request completed, latency_us: 295, response_code: 200, uri: /newextent/14/db, method: GET, req_id: b7a6950c-f770-4cec-8c01-1f4402e80a26, remote_addr: 127.0.0.1:45501, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.132 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/00E.copy" to "/tmp/downstairs-vrx8aK6L/00/000/00E.replace"
Sep 22 23:15:17.132 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:17.133 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/00E.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:17.133 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/00E"
Sep 22 23:15:17.134 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/00E.db"
Sep 22 23:15:17.134 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:17.134 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/00E.replace" to "/tmp/downstairs-vrx8aK6L/00/000/00E.completed"
Sep 22 23:15:17.134 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:17.134 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:17.134 DEBG [0] It's time to notify for 690
Sep 22 23:15:17.134 INFO Completion from [0] id:690 status:true
Sep 22 23:15:17.134 INFO [691/752] Repair commands completed
Sep 22 23:15:17.134 INFO Pop front: ReconcileIO { id: ReconciliationId(691), op: ExtentReopen { repair_id: ReconciliationId(691), extent_id: 14 }, state: ClientData([New, New, New]) }
Sep 22 23:15:17.134 INFO Sent repair work, now wait for resp
Sep 22 23:15:17.134 INFO [0] received reconcile message
Sep 22 23:15:17.134 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(691), op: ExtentReopen { repair_id: ReconciliationId(691), extent_id: 14 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:17.134 INFO [0] client ExtentReopen { repair_id: ReconciliationId(691), extent_id: 14 }
Sep 22 23:15:17.134 INFO [1] received reconcile message
Sep 22 23:15:17.134 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(691), op: ExtentReopen { repair_id: ReconciliationId(691), extent_id: 14 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:17.134 INFO [1] client ExtentReopen { repair_id: ReconciliationId(691), extent_id: 14 }
Sep 22 23:15:17.134 INFO [2] received reconcile message
Sep 22 23:15:17.134 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(691), op: ExtentReopen { repair_id: ReconciliationId(691), extent_id: 14 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:17.134 INFO [2] client ExtentReopen { repair_id: ReconciliationId(691), extent_id: 14 }
Sep 22 23:15:17.134 DEBG 691 Reopen extent 14
Sep 22 23:15:17.135 DEBG 691 Reopen extent 14
Sep 22 23:15:17.136 DEBG 691 Reopen extent 14
Sep 22 23:15:17.136 DEBG [2] It's time to notify for 691
Sep 22 23:15:17.136 INFO Completion from [2] id:691 status:true
Sep 22 23:15:17.136 INFO [692/752] Repair commands completed
Sep 22 23:15:17.136 INFO Pop front: ReconcileIO { id: ReconciliationId(692), op: ExtentFlush { repair_id: ReconciliationId(692), extent_id: 37, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:17.136 INFO Sent repair work, now wait for resp
Sep 22 23:15:17.136 INFO [0] received reconcile message
Sep 22 23:15:17.136 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(692), op: ExtentFlush { repair_id: ReconciliationId(692), extent_id: 37, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:17.136 INFO [0] client ExtentFlush { repair_id: ReconciliationId(692), extent_id: 37, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:17.136 INFO [1] received reconcile message
Sep 22 23:15:17.136 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(692), op: ExtentFlush { repair_id: ReconciliationId(692), extent_id: 37, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:17.136 INFO [1] client ExtentFlush { repair_id: ReconciliationId(692), extent_id: 37, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:17.136 INFO [2] received reconcile message
Sep 22 23:15:17.136 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(692), op: ExtentFlush { repair_id: ReconciliationId(692), extent_id: 37, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:17.136 INFO [2] client ExtentFlush { repair_id: ReconciliationId(692), extent_id: 37, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:17.137 DEBG 692 Flush extent 37 with f:2 g:2
Sep 22 23:15:17.137 DEBG Flush just extent 37 with f:2 and g:2
Sep 22 23:15:17.137 DEBG [1] It's time to notify for 692
Sep 22 23:15:17.137 INFO Completion from [1] id:692 status:true
Sep 22 23:15:17.137 INFO [693/752] Repair commands completed
Sep 22 23:15:17.137 INFO Pop front: ReconcileIO { id: ReconciliationId(693), op: ExtentClose { repair_id: ReconciliationId(693), extent_id: 37 }, state: ClientData([New, New, New]) }
Sep 22 23:15:17.137 INFO Sent repair work, now wait for resp
Sep 22 23:15:17.137 INFO [0] received reconcile message
Sep 22 23:15:17.137 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(693), op: ExtentClose { repair_id: ReconciliationId(693), extent_id: 37 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:17.137 INFO [0] client ExtentClose { repair_id: ReconciliationId(693), extent_id: 37 }
Sep 22 23:15:17.137 INFO [1] received reconcile message
Sep 22 23:15:17.137 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(693), op: ExtentClose { repair_id: ReconciliationId(693), extent_id: 37 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:17.137 INFO [1] client ExtentClose { repair_id: ReconciliationId(693), extent_id: 37 }
Sep 22 23:15:17.137 INFO [2] received reconcile message
Sep 22 23:15:17.137 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(693), op: ExtentClose { repair_id: ReconciliationId(693), extent_id: 37 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:17.137 INFO [2] client ExtentClose { repair_id: ReconciliationId(693), extent_id: 37 }
Sep 22 23:15:17.137 DEBG 693 Close extent 37
Sep 22 23:15:17.137 DEBG 693 Close extent 37
Sep 22 23:15:17.138 DEBG 693 Close extent 37
Sep 22 23:15:17.138 DEBG up_ds_listen was notified
Sep 22 23:15:17.138 DEBG up_ds_listen process 1091
Sep 22 23:15:17.138 DEBG [2] It's time to notify for 693
Sep 22 23:15:17.138 DEBG [A] ack job 1091:92, : downstairs
Sep 22 23:15:17.138 INFO Completion from [2] id:693 status:true
Sep 22 23:15:17.138 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:17.138 INFO [694/752] Repair commands completed
Sep 22 23:15:17.138 INFO Pop front: ReconcileIO { id: ReconciliationId(694), op: ExtentRepair { repair_id: ReconciliationId(694), extent_id: 37, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:17.138 INFO Sent repair work, now wait for resp
Sep 22 23:15:17.138 INFO [0] received reconcile message
Sep 22 23:15:17.138 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(694), op: ExtentRepair { repair_id: ReconciliationId(694), extent_id: 37, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:17.138 INFO [0] client ExtentRepair { repair_id: ReconciliationId(694), extent_id: 37, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:17.138 INFO [0] Sending repair request ReconciliationId(694)
Sep 22 23:15:17.138 INFO [1] received reconcile message
Sep 22 23:15:17.138 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(694), op: ExtentRepair { repair_id: ReconciliationId(694), extent_id: 37, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:17.138 INFO [1] client ExtentRepair { repair_id: ReconciliationId(694), extent_id: 37, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:17.138 INFO [1] No action required ReconciliationId(694)
Sep 22 23:15:17.138 INFO [2] received reconcile message
Sep 22 23:15:17.138 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(694), op: ExtentRepair { repair_id: ReconciliationId(694), extent_id: 37, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:17.138 INFO [2] client ExtentRepair { repair_id: ReconciliationId(694), extent_id: 37, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:17.138 INFO [2] No action required ReconciliationId(694)
Sep 22 23:15:17.139 DEBG 694 Repair extent 37 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:17.139 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/025.copy"
Sep 22 23:15:17.147 DEBG Flush     :1091 extent_limit None deps:[JobId(1090), JobId(1089)] res:true f:35 g:1
Sep 22 23:15:17.148 WARN returning error on read!
Sep 22 23:15:17.148 DEBG Read      :1092 deps:[JobId(1091)] res:false
Sep 22 23:15:17.154 DEBG Read      :1092 deps:[JobId(1091)] res:true
Sep 22 23:15:17.201 INFO accepted connection, remote_addr: 127.0.0.1:42680, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.201 TRCE incoming request, uri: /extent/37/files, method: GET, req_id: fb00f139-631a-4b41-b4d8-ffa7c81158f3, remote_addr: 127.0.0.1:42680, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.201 INFO request completed, latency_us: 209, response_code: 200, uri: /extent/37/files, method: GET, req_id: fb00f139-631a-4b41-b4d8-ffa7c81158f3, remote_addr: 127.0.0.1:42680, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.202 INFO eid:37 Found repair files: ["025", "025.db"]
Sep 22 23:15:17.202 TRCE incoming request, uri: /newextent/37/data, method: GET, req_id: a9846241-9110-4081-b82f-ffd28e60a457, remote_addr: 127.0.0.1:42680, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.202 INFO request completed, latency_us: 304, response_code: 200, uri: /newextent/37/data, method: GET, req_id: a9846241-9110-4081-b82f-ffd28e60a457, remote_addr: 127.0.0.1:42680, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.207 TRCE incoming request, uri: /newextent/37/db, method: GET, req_id: c19470f0-a39c-4700-886d-f7c2acd0a5ea, remote_addr: 127.0.0.1:42680, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.208 INFO request completed, latency_us: 292, response_code: 200, uri: /newextent/37/db, method: GET, req_id: c19470f0-a39c-4700-886d-f7c2acd0a5ea, remote_addr: 127.0.0.1:42680, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.209 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/025.copy" to "/tmp/downstairs-vrx8aK6L/00/000/025.replace"
Sep 22 23:15:17.209 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:17.210 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/025.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:17.210 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/025"
Sep 22 23:15:17.210 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/025.db"
Sep 22 23:15:17.210 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:17.210 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/025.replace" to "/tmp/downstairs-vrx8aK6L/00/000/025.completed"
Sep 22 23:15:17.210 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:17.210 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:17.210 DEBG [0] It's time to notify for 694
Sep 22 23:15:17.211 INFO Completion from [0] id:694 status:true
Sep 22 23:15:17.211 INFO [695/752] Repair commands completed
Sep 22 23:15:17.211 INFO Pop front: ReconcileIO { id: ReconciliationId(695), op: ExtentReopen { repair_id: ReconciliationId(695), extent_id: 37 }, state: ClientData([New, New, New]) }
Sep 22 23:15:17.211 INFO Sent repair work, now wait for resp
Sep 22 23:15:17.211 INFO [0] received reconcile message
Sep 22 23:15:17.211 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(695), op: ExtentReopen { repair_id: ReconciliationId(695), extent_id: 37 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:17.211 INFO [0] client ExtentReopen { repair_id: ReconciliationId(695), extent_id: 37 }
Sep 22 23:15:17.211 INFO [1] received reconcile message
Sep 22 23:15:17.211 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(695), op: ExtentReopen { repair_id: ReconciliationId(695), extent_id: 37 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:17.211 INFO [1] client ExtentReopen { repair_id: ReconciliationId(695), extent_id: 37 }
Sep 22 23:15:17.211 INFO [2] received reconcile message
Sep 22 23:15:17.211 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(695), op: ExtentReopen { repair_id: ReconciliationId(695), extent_id: 37 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:17.211 INFO [2] client ExtentReopen { repair_id: ReconciliationId(695), extent_id: 37 }
Sep 22 23:15:17.211 DEBG 695 Reopen extent 37
Sep 22 23:15:17.211 DEBG 695 Reopen extent 37
Sep 22 23:15:17.212 DEBG 695 Reopen extent 37
Sep 22 23:15:17.213 DEBG [2] It's time to notify for 695
Sep 22 23:15:17.213 INFO Completion from [2] id:695 status:true
Sep 22 23:15:17.213 INFO [696/752] Repair commands completed
Sep 22 23:15:17.213 INFO Pop front: ReconcileIO { id: ReconciliationId(696), op: ExtentFlush { repair_id: ReconciliationId(696), extent_id: 124, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:17.213 INFO Sent repair work, now wait for resp
Sep 22 23:15:17.213 INFO [0] received reconcile message
Sep 22 23:15:17.213 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(696), op: ExtentFlush { repair_id: ReconciliationId(696), extent_id: 124, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:17.213 INFO [0] client ExtentFlush { repair_id: ReconciliationId(696), extent_id: 124, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:17.213 INFO [1] received reconcile message
Sep 22 23:15:17.213 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(696), op: ExtentFlush { repair_id: ReconciliationId(696), extent_id: 124, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:17.213 INFO [1] client ExtentFlush { repair_id: ReconciliationId(696), extent_id: 124, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:17.213 INFO [2] received reconcile message
Sep 22 23:15:17.213 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(696), op: ExtentFlush { repair_id: ReconciliationId(696), extent_id: 124, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:17.213 INFO [2] client ExtentFlush { repair_id: ReconciliationId(696), extent_id: 124, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:17.213 DEBG 696 Flush extent 124 with f:2 g:2
Sep 22 23:15:17.213 DEBG Flush just extent 124 with f:2 and g:2
Sep 22 23:15:17.213 DEBG [1] It's time to notify for 696
Sep 22 23:15:17.213 INFO Completion from [1] id:696 status:true
Sep 22 23:15:17.213 INFO [697/752] Repair commands completed
Sep 22 23:15:17.213 INFO Pop front: ReconcileIO { id: ReconciliationId(697), op: ExtentClose { repair_id: ReconciliationId(697), extent_id: 124 }, state: ClientData([New, New, New]) }
Sep 22 23:15:17.213 INFO Sent repair work, now wait for resp
Sep 22 23:15:17.213 INFO [0] received reconcile message
Sep 22 23:15:17.213 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(697), op: ExtentClose { repair_id: ReconciliationId(697), extent_id: 124 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:17.213 INFO [0] client ExtentClose { repair_id: ReconciliationId(697), extent_id: 124 }
Sep 22 23:15:17.213 INFO [1] received reconcile message
Sep 22 23:15:17.213 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(697), op: ExtentClose { repair_id: ReconciliationId(697), extent_id: 124 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:17.213 INFO [1] client ExtentClose { repair_id: ReconciliationId(697), extent_id: 124 }
Sep 22 23:15:17.213 INFO [2] received reconcile message
Sep 22 23:15:17.213 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(697), op: ExtentClose { repair_id: ReconciliationId(697), extent_id: 124 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:17.214 INFO [2] client ExtentClose { repair_id: ReconciliationId(697), extent_id: 124 }
Sep 22 23:15:17.214 DEBG 697 Close extent 124
Sep 22 23:15:17.214 DEBG 697 Close extent 124
Sep 22 23:15:17.214 DEBG 697 Close extent 124
Sep 22 23:15:17.215 DEBG [2] It's time to notify for 697
Sep 22 23:15:17.215 INFO Completion from [2] id:697 status:true
Sep 22 23:15:17.215 INFO [698/752] Repair commands completed
Sep 22 23:15:17.215 INFO Pop front: ReconcileIO { id: ReconciliationId(698), op: ExtentRepair { repair_id: ReconciliationId(698), extent_id: 124, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:17.215 INFO Sent repair work, now wait for resp
Sep 22 23:15:17.215 INFO [0] received reconcile message
Sep 22 23:15:17.215 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(698), op: ExtentRepair { repair_id: ReconciliationId(698), extent_id: 124, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:17.215 INFO [0] client ExtentRepair { repair_id: ReconciliationId(698), extent_id: 124, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:17.215 INFO [0] Sending repair request ReconciliationId(698)
Sep 22 23:15:17.215 INFO [1] received reconcile message
Sep 22 23:15:17.215 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(698), op: ExtentRepair { repair_id: ReconciliationId(698), extent_id: 124, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:17.215 INFO [1] client ExtentRepair { repair_id: ReconciliationId(698), extent_id: 124, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:17.215 INFO [1] No action required ReconciliationId(698)
Sep 22 23:15:17.215 INFO [2] received reconcile message
Sep 22 23:15:17.215 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(698), op: ExtentRepair { repair_id: ReconciliationId(698), extent_id: 124, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:17.215 INFO [2] client ExtentRepair { repair_id: ReconciliationId(698), extent_id: 124, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:17.215 INFO [2] No action required ReconciliationId(698)
Sep 22 23:15:17.215 DEBG 698 Repair extent 124 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:17.215 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/07C.copy"
Sep 22 23:15:17.277 INFO accepted connection, remote_addr: 127.0.0.1:42905, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.277 TRCE incoming request, uri: /extent/124/files, method: GET, req_id: e38070f0-4181-4fd0-9726-8c979008bbae, remote_addr: 127.0.0.1:42905, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.277 INFO request completed, latency_us: 191, response_code: 200, uri: /extent/124/files, method: GET, req_id: e38070f0-4181-4fd0-9726-8c979008bbae, remote_addr: 127.0.0.1:42905, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.278 INFO eid:124 Found repair files: ["07C", "07C.db"]
Sep 22 23:15:17.278 TRCE incoming request, uri: /newextent/124/data, method: GET, req_id: 1ca4752a-ccd0-4cae-a693-ea8b95f59e57, remote_addr: 127.0.0.1:42905, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.278 INFO request completed, latency_us: 298, response_code: 200, uri: /newextent/124/data, method: GET, req_id: 1ca4752a-ccd0-4cae-a693-ea8b95f59e57, remote_addr: 127.0.0.1:42905, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.283 TRCE incoming request, uri: /newextent/124/db, method: GET, req_id: 2702736f-6b60-4da8-ad5d-952c90d1b43f, remote_addr: 127.0.0.1:42905, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.284 INFO request completed, latency_us: 294, response_code: 200, uri: /newextent/124/db, method: GET, req_id: 2702736f-6b60-4da8-ad5d-952c90d1b43f, remote_addr: 127.0.0.1:42905, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.285 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/07C.copy" to "/tmp/downstairs-vrx8aK6L/00/000/07C.replace"
Sep 22 23:15:17.285 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:17.286 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/07C.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:17.286 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/07C"
Sep 22 23:15:17.286 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/07C.db"
Sep 22 23:15:17.286 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:17.286 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/07C.replace" to "/tmp/downstairs-vrx8aK6L/00/000/07C.completed"
Sep 22 23:15:17.286 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:17.286 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:17.286 DEBG [0] It's time to notify for 698
Sep 22 23:15:17.286 INFO Completion from [0] id:698 status:true
Sep 22 23:15:17.286 INFO [699/752] Repair commands completed
Sep 22 23:15:17.286 INFO Pop front: ReconcileIO { id: ReconciliationId(699), op: ExtentReopen { repair_id: ReconciliationId(699), extent_id: 124 }, state: ClientData([New, New, New]) }
Sep 22 23:15:17.287 INFO Sent repair work, now wait for resp
Sep 22 23:15:17.287 INFO [0] received reconcile message
Sep 22 23:15:17.287 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(699), op: ExtentReopen { repair_id: ReconciliationId(699), extent_id: 124 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:17.287 INFO [0] client ExtentReopen { repair_id: ReconciliationId(699), extent_id: 124 }
Sep 22 23:15:17.287 INFO [1] received reconcile message
Sep 22 23:15:17.287 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(699), op: ExtentReopen { repair_id: ReconciliationId(699), extent_id: 124 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:17.287 INFO [1] client ExtentReopen { repair_id: ReconciliationId(699), extent_id: 124 }
Sep 22 23:15:17.287 INFO [2] received reconcile message
Sep 22 23:15:17.287 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(699), op: ExtentReopen { repair_id: ReconciliationId(699), extent_id: 124 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:17.287 INFO [2] client ExtentReopen { repair_id: ReconciliationId(699), extent_id: 124 }
Sep 22 23:15:17.287 DEBG 699 Reopen extent 124
Sep 22 23:15:17.287 DEBG 699 Reopen extent 124
Sep 22 23:15:17.288 DEBG 699 Reopen extent 124
Sep 22 23:15:17.288 DEBG [2] It's time to notify for 699
Sep 22 23:15:17.289 INFO Completion from [2] id:699 status:true
Sep 22 23:15:17.289 INFO [700/752] Repair commands completed
Sep 22 23:15:17.289 INFO Pop front: ReconcileIO { id: ReconciliationId(700), op: ExtentFlush { repair_id: ReconciliationId(700), extent_id: 5, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:17.289 INFO Sent repair work, now wait for resp
Sep 22 23:15:17.289 INFO [0] received reconcile message
Sep 22 23:15:17.289 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(700), op: ExtentFlush { repair_id: ReconciliationId(700), extent_id: 5, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:17.289 INFO [0] client ExtentFlush { repair_id: ReconciliationId(700), extent_id: 5, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:17.289 INFO [1] received reconcile message
Sep 22 23:15:17.289 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(700), op: ExtentFlush { repair_id: ReconciliationId(700), extent_id: 5, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:17.289 INFO [1] client ExtentFlush { repair_id: ReconciliationId(700), extent_id: 5, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:17.289 INFO [2] received reconcile message
Sep 22 23:15:17.289 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(700), op: ExtentFlush { repair_id: ReconciliationId(700), extent_id: 5, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:17.289 INFO [2] client ExtentFlush { repair_id: ReconciliationId(700), extent_id: 5, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:17.289 DEBG 700 Flush extent 5 with f:2 g:2
Sep 22 23:15:17.289 DEBG Flush just extent 5 with f:2 and g:2
Sep 22 23:15:17.289 DEBG [1] It's time to notify for 700
Sep 22 23:15:17.289 INFO Completion from [1] id:700 status:true
Sep 22 23:15:17.289 INFO [701/752] Repair commands completed
Sep 22 23:15:17.289 INFO Pop front: ReconcileIO { id: ReconciliationId(701), op: ExtentClose { repair_id: ReconciliationId(701), extent_id: 5 }, state: ClientData([New, New, New]) }
Sep 22 23:15:17.289 INFO Sent repair work, now wait for resp
Sep 22 23:15:17.289 INFO [0] received reconcile message
Sep 22 23:15:17.289 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(701), op: ExtentClose { repair_id: ReconciliationId(701), extent_id: 5 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:17.289 INFO [0] client ExtentClose { repair_id: ReconciliationId(701), extent_id: 5 }
Sep 22 23:15:17.289 INFO [1] received reconcile message
Sep 22 23:15:17.289 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(701), op: ExtentClose { repair_id: ReconciliationId(701), extent_id: 5 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:17.289 INFO [1] client ExtentClose { repair_id: ReconciliationId(701), extent_id: 5 }
Sep 22 23:15:17.289 INFO [2] received reconcile message
Sep 22 23:15:17.289 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(701), op: ExtentClose { repair_id: ReconciliationId(701), extent_id: 5 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:17.289 INFO [2] client ExtentClose { repair_id: ReconciliationId(701), extent_id: 5 }
Sep 22 23:15:17.290 DEBG 701 Close extent 5
Sep 22 23:15:17.290 DEBG 701 Close extent 5
Sep 22 23:15:17.290 DEBG 701 Close extent 5
Sep 22 23:15:17.290 DEBG [2] It's time to notify for 701
Sep 22 23:15:17.291 INFO Completion from [2] id:701 status:true
Sep 22 23:15:17.291 INFO [702/752] Repair commands completed
Sep 22 23:15:17.291 INFO Pop front: ReconcileIO { id: ReconciliationId(702), op: ExtentRepair { repair_id: ReconciliationId(702), extent_id: 5, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:17.291 INFO Sent repair work, now wait for resp
Sep 22 23:15:17.291 INFO [0] received reconcile message
Sep 22 23:15:17.291 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(702), op: ExtentRepair { repair_id: ReconciliationId(702), extent_id: 5, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:17.291 INFO [0] client ExtentRepair { repair_id: ReconciliationId(702), extent_id: 5, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:17.291 INFO [0] Sending repair request ReconciliationId(702)
Sep 22 23:15:17.291 INFO [1] received reconcile message
Sep 22 23:15:17.291 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(702), op: ExtentRepair { repair_id: ReconciliationId(702), extent_id: 5, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:17.291 INFO [1] client ExtentRepair { repair_id: ReconciliationId(702), extent_id: 5, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:17.291 INFO [1] No action required ReconciliationId(702)
Sep 22 23:15:17.291 INFO [2] received reconcile message
Sep 22 23:15:17.291 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(702), op: ExtentRepair { repair_id: ReconciliationId(702), extent_id: 5, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:17.291 INFO [2] client ExtentRepair { repair_id: ReconciliationId(702), extent_id: 5, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:17.291 INFO [2] No action required ReconciliationId(702)
Sep 22 23:15:17.291 DEBG 702 Repair extent 5 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:17.291 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/005.copy"
Sep 22 23:15:17.355 INFO accepted connection, remote_addr: 127.0.0.1:46810, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.356 TRCE incoming request, uri: /extent/5/files, method: GET, req_id: 15bae0bb-9d3e-4857-85dc-d5533c6d3fe7, remote_addr: 127.0.0.1:46810, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.356 INFO request completed, latency_us: 194, response_code: 200, uri: /extent/5/files, method: GET, req_id: 15bae0bb-9d3e-4857-85dc-d5533c6d3fe7, remote_addr: 127.0.0.1:46810, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.356 INFO eid:5 Found repair files: ["005", "005.db"]
Sep 22 23:15:17.356 TRCE incoming request, uri: /newextent/5/data, method: GET, req_id: e6e8cc4b-41dc-4c6c-a083-101970c835a0, remote_addr: 127.0.0.1:46810, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.357 INFO request completed, latency_us: 258, response_code: 200, uri: /newextent/5/data, method: GET, req_id: e6e8cc4b-41dc-4c6c-a083-101970c835a0, remote_addr: 127.0.0.1:46810, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.361 TRCE incoming request, uri: /newextent/5/db, method: GET, req_id: 395127cf-397c-420b-8648-28cd8d1ba6b4, remote_addr: 127.0.0.1:46810, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.362 INFO request completed, latency_us: 294, response_code: 200, uri: /newextent/5/db, method: GET, req_id: 395127cf-397c-420b-8648-28cd8d1ba6b4, remote_addr: 127.0.0.1:46810, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.363 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/005.copy" to "/tmp/downstairs-vrx8aK6L/00/000/005.replace"
Sep 22 23:15:17.363 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:17.364 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/005.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:17.364 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/005"
Sep 22 23:15:17.364 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/005.db"
Sep 22 23:15:17.364 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:17.364 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/005.replace" to "/tmp/downstairs-vrx8aK6L/00/000/005.completed"
Sep 22 23:15:17.364 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:17.364 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:17.364 DEBG [0] It's time to notify for 702
Sep 22 23:15:17.364 INFO Completion from [0] id:702 status:true
Sep 22 23:15:17.364 INFO [703/752] Repair commands completed
Sep 22 23:15:17.364 INFO Pop front: ReconcileIO { id: ReconciliationId(703), op: ExtentReopen { repair_id: ReconciliationId(703), extent_id: 5 }, state: ClientData([New, New, New]) }
Sep 22 23:15:17.364 INFO Sent repair work, now wait for resp
Sep 22 23:15:17.365 INFO [0] received reconcile message
Sep 22 23:15:17.365 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(703), op: ExtentReopen { repair_id: ReconciliationId(703), extent_id: 5 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:17.365 INFO [0] client ExtentReopen { repair_id: ReconciliationId(703), extent_id: 5 }
Sep 22 23:15:17.365 INFO [1] received reconcile message
Sep 22 23:15:17.365 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(703), op: ExtentReopen { repair_id: ReconciliationId(703), extent_id: 5 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:17.365 INFO [1] client ExtentReopen { repair_id: ReconciliationId(703), extent_id: 5 }
Sep 22 23:15:17.365 INFO [2] received reconcile message
Sep 22 23:15:17.365 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(703), op: ExtentReopen { repair_id: ReconciliationId(703), extent_id: 5 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:17.365 INFO [2] client ExtentReopen { repair_id: ReconciliationId(703), extent_id: 5 }
Sep 22 23:15:17.365 DEBG 703 Reopen extent 5
Sep 22 23:15:17.365 DEBG 703 Reopen extent 5
Sep 22 23:15:17.366 DEBG 703 Reopen extent 5
Sep 22 23:15:17.366 DEBG [2] It's time to notify for 703
Sep 22 23:15:17.367 INFO Completion from [2] id:703 status:true
Sep 22 23:15:17.367 INFO [704/752] Repair commands completed
Sep 22 23:15:17.367 INFO Pop front: ReconcileIO { id: ReconciliationId(704), op: ExtentFlush { repair_id: ReconciliationId(704), extent_id: 55, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:17.367 INFO Sent repair work, now wait for resp
Sep 22 23:15:17.367 INFO [0] received reconcile message
Sep 22 23:15:17.367 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(704), op: ExtentFlush { repair_id: ReconciliationId(704), extent_id: 55, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:17.367 INFO [0] client ExtentFlush { repair_id: ReconciliationId(704), extent_id: 55, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:17.367 INFO [1] received reconcile message
Sep 22 23:15:17.367 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(704), op: ExtentFlush { repair_id: ReconciliationId(704), extent_id: 55, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:17.367 INFO [1] client ExtentFlush { repair_id: ReconciliationId(704), extent_id: 55, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:17.367 INFO [2] received reconcile message
Sep 22 23:15:17.367 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(704), op: ExtentFlush { repair_id: ReconciliationId(704), extent_id: 55, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:17.367 INFO [2] client ExtentFlush { repair_id: ReconciliationId(704), extent_id: 55, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:17.367 DEBG 704 Flush extent 55 with f:2 g:2
Sep 22 23:15:17.367 DEBG Flush just extent 55 with f:2 and g:2
Sep 22 23:15:17.367 DEBG [1] It's time to notify for 704
Sep 22 23:15:17.367 INFO Completion from [1] id:704 status:true
Sep 22 23:15:17.367 INFO [705/752] Repair commands completed
Sep 22 23:15:17.367 INFO Pop front: ReconcileIO { id: ReconciliationId(705), op: ExtentClose { repair_id: ReconciliationId(705), extent_id: 55 }, state: ClientData([New, New, New]) }
Sep 22 23:15:17.367 INFO Sent repair work, now wait for resp
Sep 22 23:15:17.367 INFO [0] received reconcile message
Sep 22 23:15:17.367 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(705), op: ExtentClose { repair_id: ReconciliationId(705), extent_id: 55 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:17.367 INFO [0] client ExtentClose { repair_id: ReconciliationId(705), extent_id: 55 }
Sep 22 23:15:17.367 INFO [1] received reconcile message
Sep 22 23:15:17.367 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(705), op: ExtentClose { repair_id: ReconciliationId(705), extent_id: 55 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:17.367 INFO [1] client ExtentClose { repair_id: ReconciliationId(705), extent_id: 55 }
Sep 22 23:15:17.367 INFO [2] received reconcile message
Sep 22 23:15:17.367 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(705), op: ExtentClose { repair_id: ReconciliationId(705), extent_id: 55 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:17.367 INFO [2] client ExtentClose { repair_id: ReconciliationId(705), extent_id: 55 }
Sep 22 23:15:17.368 DEBG 705 Close extent 55
Sep 22 23:15:17.368 DEBG 705 Close extent 55
Sep 22 23:15:17.368 DEBG 705 Close extent 55
Sep 22 23:15:17.368 DEBG [2] It's time to notify for 705
Sep 22 23:15:17.368 INFO Completion from [2] id:705 status:true
Sep 22 23:15:17.368 INFO [706/752] Repair commands completed
Sep 22 23:15:17.369 INFO Pop front: ReconcileIO { id: ReconciliationId(706), op: ExtentRepair { repair_id: ReconciliationId(706), extent_id: 55, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:17.369 INFO Sent repair work, now wait for resp
Sep 22 23:15:17.369 INFO [0] received reconcile message
Sep 22 23:15:17.369 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(706), op: ExtentRepair { repair_id: ReconciliationId(706), extent_id: 55, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:17.369 INFO [0] client ExtentRepair { repair_id: ReconciliationId(706), extent_id: 55, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:17.369 INFO [0] Sending repair request ReconciliationId(706)
Sep 22 23:15:17.369 INFO [1] received reconcile message
Sep 22 23:15:17.369 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(706), op: ExtentRepair { repair_id: ReconciliationId(706), extent_id: 55, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:17.369 INFO [1] client ExtentRepair { repair_id: ReconciliationId(706), extent_id: 55, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:17.369 INFO [1] No action required ReconciliationId(706)
Sep 22 23:15:17.369 INFO [2] received reconcile message
Sep 22 23:15:17.369 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(706), op: ExtentRepair { repair_id: ReconciliationId(706), extent_id: 55, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:17.369 INFO [2] client ExtentRepair { repair_id: ReconciliationId(706), extent_id: 55, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:17.369 INFO [2] No action required ReconciliationId(706)
Sep 22 23:15:17.369 DEBG 706 Repair extent 55 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:17.369 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/037.copy"
Sep 22 23:15:17.434 INFO accepted connection, remote_addr: 127.0.0.1:57552, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.434 TRCE incoming request, uri: /extent/55/files, method: GET, req_id: 6454ead8-cc50-4d05-8409-96505b307855, remote_addr: 127.0.0.1:57552, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.434 INFO request completed, latency_us: 189, response_code: 200, uri: /extent/55/files, method: GET, req_id: 6454ead8-cc50-4d05-8409-96505b307855, remote_addr: 127.0.0.1:57552, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.434 INFO eid:55 Found repair files: ["037", "037.db"]
Sep 22 23:15:17.435 TRCE incoming request, uri: /newextent/55/data, method: GET, req_id: e494a712-8bd2-428f-8d5d-2a10cb6714ad, remote_addr: 127.0.0.1:57552, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.435 INFO request completed, latency_us: 319, response_code: 200, uri: /newextent/55/data, method: GET, req_id: e494a712-8bd2-428f-8d5d-2a10cb6714ad, remote_addr: 127.0.0.1:57552, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.440 TRCE incoming request, uri: /newextent/55/db, method: GET, req_id: 81f8e990-a4e9-4df1-8c3b-4a4a25fbdb4e, remote_addr: 127.0.0.1:57552, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.440 INFO request completed, latency_us: 294, response_code: 200, uri: /newextent/55/db, method: GET, req_id: 81f8e990-a4e9-4df1-8c3b-4a4a25fbdb4e, remote_addr: 127.0.0.1:57552, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.441 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/037.copy" to "/tmp/downstairs-vrx8aK6L/00/000/037.replace"
Sep 22 23:15:17.441 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:17.442 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/037.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:17.442 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/037"
Sep 22 23:15:17.442 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/037.db"
Sep 22 23:15:17.442 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:17.442 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/037.replace" to "/tmp/downstairs-vrx8aK6L/00/000/037.completed"
Sep 22 23:15:17.442 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:17.443 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:17.443 DEBG [0] It's time to notify for 706
Sep 22 23:15:17.443 INFO Completion from [0] id:706 status:true
Sep 22 23:15:17.443 INFO [707/752] Repair commands completed
Sep 22 23:15:17.443 INFO Pop front: ReconcileIO { id: ReconciliationId(707), op: ExtentReopen { repair_id: ReconciliationId(707), extent_id: 55 }, state: ClientData([New, New, New]) }
Sep 22 23:15:17.443 INFO Sent repair work, now wait for resp
Sep 22 23:15:17.443 INFO [0] received reconcile message
Sep 22 23:15:17.443 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(707), op: ExtentReopen { repair_id: ReconciliationId(707), extent_id: 55 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:17.443 INFO [0] client ExtentReopen { repair_id: ReconciliationId(707), extent_id: 55 }
Sep 22 23:15:17.443 INFO [1] received reconcile message
Sep 22 23:15:17.443 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(707), op: ExtentReopen { repair_id: ReconciliationId(707), extent_id: 55 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:17.443 INFO [1] client ExtentReopen { repair_id: ReconciliationId(707), extent_id: 55 }
Sep 22 23:15:17.443 INFO [2] received reconcile message
Sep 22 23:15:17.443 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(707), op: ExtentReopen { repair_id: ReconciliationId(707), extent_id: 55 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:17.443 INFO [2] client ExtentReopen { repair_id: ReconciliationId(707), extent_id: 55 }
Sep 22 23:15:17.443 DEBG 707 Reopen extent 55
Sep 22 23:15:17.444 DEBG 707 Reopen extent 55
Sep 22 23:15:17.444 DEBG 707 Reopen extent 55
Sep 22 23:15:17.445 DEBG [2] It's time to notify for 707
Sep 22 23:15:17.445 INFO Completion from [2] id:707 status:true
Sep 22 23:15:17.445 INFO [708/752] Repair commands completed
Sep 22 23:15:17.445 INFO Pop front: ReconcileIO { id: ReconciliationId(708), op: ExtentFlush { repair_id: ReconciliationId(708), extent_id: 62, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:17.445 INFO Sent repair work, now wait for resp
Sep 22 23:15:17.445 INFO [0] received reconcile message
Sep 22 23:15:17.445 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(708), op: ExtentFlush { repair_id: ReconciliationId(708), extent_id: 62, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:17.445 INFO [0] client ExtentFlush { repair_id: ReconciliationId(708), extent_id: 62, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:17.445 INFO [1] received reconcile message
Sep 22 23:15:17.445 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(708), op: ExtentFlush { repair_id: ReconciliationId(708), extent_id: 62, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:17.445 INFO [1] client ExtentFlush { repair_id: ReconciliationId(708), extent_id: 62, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:17.445 INFO [2] received reconcile message
Sep 22 23:15:17.445 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(708), op: ExtentFlush { repair_id: ReconciliationId(708), extent_id: 62, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:17.445 INFO [2] client ExtentFlush { repair_id: ReconciliationId(708), extent_id: 62, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:17.445 DEBG 708 Flush extent 62 with f:2 g:2
Sep 22 23:15:17.445 DEBG Flush just extent 62 with f:2 and g:2
Sep 22 23:15:17.445 DEBG [1] It's time to notify for 708
Sep 22 23:15:17.446 INFO Completion from [1] id:708 status:true
Sep 22 23:15:17.446 INFO [709/752] Repair commands completed
Sep 22 23:15:17.446 INFO Pop front: ReconcileIO { id: ReconciliationId(709), op: ExtentClose { repair_id: ReconciliationId(709), extent_id: 62 }, state: ClientData([New, New, New]) }
Sep 22 23:15:17.446 INFO Sent repair work, now wait for resp
Sep 22 23:15:17.446 INFO [0] received reconcile message
Sep 22 23:15:17.446 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(709), op: ExtentClose { repair_id: ReconciliationId(709), extent_id: 62 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:17.446 INFO [0] client ExtentClose { repair_id: ReconciliationId(709), extent_id: 62 }
Sep 22 23:15:17.446 INFO [1] received reconcile message
Sep 22 23:15:17.446 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(709), op: ExtentClose { repair_id: ReconciliationId(709), extent_id: 62 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:17.446 INFO [1] client ExtentClose { repair_id: ReconciliationId(709), extent_id: 62 }
Sep 22 23:15:17.446 INFO [2] received reconcile message
Sep 22 23:15:17.446 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(709), op: ExtentClose { repair_id: ReconciliationId(709), extent_id: 62 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:17.446 INFO [2] client ExtentClose { repair_id: ReconciliationId(709), extent_id: 62 }
Sep 22 23:15:17.446 DEBG 709 Close extent 62
Sep 22 23:15:17.446 DEBG 709 Close extent 62
Sep 22 23:15:17.446 DEBG 709 Close extent 62
Sep 22 23:15:17.447 DEBG [2] It's time to notify for 709
Sep 22 23:15:17.447 INFO Completion from [2] id:709 status:true
Sep 22 23:15:17.447 INFO [710/752] Repair commands completed
Sep 22 23:15:17.447 INFO Pop front: ReconcileIO { id: ReconciliationId(710), op: ExtentRepair { repair_id: ReconciliationId(710), extent_id: 62, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:17.447 INFO Sent repair work, now wait for resp
Sep 22 23:15:17.447 INFO [0] received reconcile message
Sep 22 23:15:17.447 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(710), op: ExtentRepair { repair_id: ReconciliationId(710), extent_id: 62, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:17.447 INFO [0] client ExtentRepair { repair_id: ReconciliationId(710), extent_id: 62, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:17.447 INFO [0] Sending repair request ReconciliationId(710)
Sep 22 23:15:17.447 INFO [1] received reconcile message
Sep 22 23:15:17.447 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(710), op: ExtentRepair { repair_id: ReconciliationId(710), extent_id: 62, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:17.447 INFO [1] client ExtentRepair { repair_id: ReconciliationId(710), extent_id: 62, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:17.447 INFO [1] No action required ReconciliationId(710)
Sep 22 23:15:17.447 INFO [2] received reconcile message
Sep 22 23:15:17.447 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(710), op: ExtentRepair { repair_id: ReconciliationId(710), extent_id: 62, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:17.447 INFO [2] client ExtentRepair { repair_id: ReconciliationId(710), extent_id: 62, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:17.447 INFO [2] No action required ReconciliationId(710)
Sep 22 23:15:17.447 DEBG 710 Repair extent 62 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:17.447 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/03E.copy"
Sep 22 23:15:17.510 INFO accepted connection, remote_addr: 127.0.0.1:59015, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.510 TRCE incoming request, uri: /extent/62/files, method: GET, req_id: 44d53c19-c569-41bb-846a-b2daca637e48, remote_addr: 127.0.0.1:59015, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.510 INFO request completed, latency_us: 188, response_code: 200, uri: /extent/62/files, method: GET, req_id: 44d53c19-c569-41bb-846a-b2daca637e48, remote_addr: 127.0.0.1:59015, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.511 INFO eid:62 Found repair files: ["03E", "03E.db"]
Sep 22 23:15:17.511 TRCE incoming request, uri: /newextent/62/data, method: GET, req_id: b031140b-6e94-4e5c-ae32-b65d58b4dd01, remote_addr: 127.0.0.1:59015, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.511 INFO request completed, latency_us: 296, response_code: 200, uri: /newextent/62/data, method: GET, req_id: b031140b-6e94-4e5c-ae32-b65d58b4dd01, remote_addr: 127.0.0.1:59015, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.516 TRCE incoming request, uri: /newextent/62/db, method: GET, req_id: 59e2250e-7b86-4391-a8c7-f8f7e812e62f, remote_addr: 127.0.0.1:59015, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.517 INFO request completed, latency_us: 293, response_code: 200, uri: /newextent/62/db, method: GET, req_id: 59e2250e-7b86-4391-a8c7-f8f7e812e62f, remote_addr: 127.0.0.1:59015, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.518 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/03E.copy" to "/tmp/downstairs-vrx8aK6L/00/000/03E.replace"
Sep 22 23:15:17.518 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:17.519 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/03E.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:17.519 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/03E"
Sep 22 23:15:17.519 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/03E.db"
Sep 22 23:15:17.519 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:17.519 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/03E.replace" to "/tmp/downstairs-vrx8aK6L/00/000/03E.completed"
Sep 22 23:15:17.519 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:17.519 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:17.519 DEBG [0] It's time to notify for 710
Sep 22 23:15:17.519 INFO Completion from [0] id:710 status:true
Sep 22 23:15:17.519 INFO [711/752] Repair commands completed
Sep 22 23:15:17.519 INFO Pop front: ReconcileIO { id: ReconciliationId(711), op: ExtentReopen { repair_id: ReconciliationId(711), extent_id: 62 }, state: ClientData([New, New, New]) }
Sep 22 23:15:17.519 INFO Sent repair work, now wait for resp
Sep 22 23:15:17.519 INFO [0] received reconcile message
Sep 22 23:15:17.519 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(711), op: ExtentReopen { repair_id: ReconciliationId(711), extent_id: 62 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:17.519 INFO [0] client ExtentReopen { repair_id: ReconciliationId(711), extent_id: 62 }
Sep 22 23:15:17.520 INFO [1] received reconcile message
Sep 22 23:15:17.520 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(711), op: ExtentReopen { repair_id: ReconciliationId(711), extent_id: 62 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:17.520 INFO [1] client ExtentReopen { repair_id: ReconciliationId(711), extent_id: 62 }
Sep 22 23:15:17.520 INFO [2] received reconcile message
Sep 22 23:15:17.520 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(711), op: ExtentReopen { repair_id: ReconciliationId(711), extent_id: 62 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:17.520 INFO [2] client ExtentReopen { repair_id: ReconciliationId(711), extent_id: 62 }
Sep 22 23:15:17.520 DEBG 711 Reopen extent 62
Sep 22 23:15:17.520 DEBG 711 Reopen extent 62
Sep 22 23:15:17.521 DEBG 711 Reopen extent 62
Sep 22 23:15:17.521 DEBG [2] It's time to notify for 711
Sep 22 23:15:17.521 INFO Completion from [2] id:711 status:true
Sep 22 23:15:17.521 INFO [712/752] Repair commands completed
Sep 22 23:15:17.521 INFO Pop front: ReconcileIO { id: ReconciliationId(712), op: ExtentFlush { repair_id: ReconciliationId(712), extent_id: 89, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:17.522 INFO Sent repair work, now wait for resp
Sep 22 23:15:17.522 INFO [0] received reconcile message
Sep 22 23:15:17.522 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(712), op: ExtentFlush { repair_id: ReconciliationId(712), extent_id: 89, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:17.522 INFO [0] client ExtentFlush { repair_id: ReconciliationId(712), extent_id: 89, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:17.522 INFO [1] received reconcile message
Sep 22 23:15:17.522 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(712), op: ExtentFlush { repair_id: ReconciliationId(712), extent_id: 89, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:17.522 INFO [1] client ExtentFlush { repair_id: ReconciliationId(712), extent_id: 89, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:17.522 INFO [2] received reconcile message
Sep 22 23:15:17.522 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(712), op: ExtentFlush { repair_id: ReconciliationId(712), extent_id: 89, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:17.522 INFO [2] client ExtentFlush { repair_id: ReconciliationId(712), extent_id: 89, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:17.522 DEBG 712 Flush extent 89 with f:2 g:2
Sep 22 23:15:17.522 DEBG Flush just extent 89 with f:2 and g:2
Sep 22 23:15:17.522 DEBG [1] It's time to notify for 712
Sep 22 23:15:17.522 INFO Completion from [1] id:712 status:true
Sep 22 23:15:17.522 INFO [713/752] Repair commands completed
Sep 22 23:15:17.522 INFO Pop front: ReconcileIO { id: ReconciliationId(713), op: ExtentClose { repair_id: ReconciliationId(713), extent_id: 89 }, state: ClientData([New, New, New]) }
Sep 22 23:15:17.522 INFO Sent repair work, now wait for resp
Sep 22 23:15:17.522 INFO [0] received reconcile message
Sep 22 23:15:17.522 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(713), op: ExtentClose { repair_id: ReconciliationId(713), extent_id: 89 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:17.522 INFO [0] client ExtentClose { repair_id: ReconciliationId(713), extent_id: 89 }
Sep 22 23:15:17.522 INFO [1] received reconcile message
Sep 22 23:15:17.522 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(713), op: ExtentClose { repair_id: ReconciliationId(713), extent_id: 89 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:17.522 INFO [1] client ExtentClose { repair_id: ReconciliationId(713), extent_id: 89 }
Sep 22 23:15:17.522 INFO [2] received reconcile message
Sep 22 23:15:17.522 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(713), op: ExtentClose { repair_id: ReconciliationId(713), extent_id: 89 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:17.522 INFO [2] client ExtentClose { repair_id: ReconciliationId(713), extent_id: 89 }
Sep 22 23:15:17.522 DEBG 713 Close extent 89
Sep 22 23:15:17.523 DEBG 713 Close extent 89
Sep 22 23:15:17.523 DEBG 713 Close extent 89
Sep 22 23:15:17.523 DEBG [2] It's time to notify for 713
Sep 22 23:15:17.523 INFO Completion from [2] id:713 status:true
Sep 22 23:15:17.523 INFO [714/752] Repair commands completed
Sep 22 23:15:17.523 INFO Pop front: ReconcileIO { id: ReconciliationId(714), op: ExtentRepair { repair_id: ReconciliationId(714), extent_id: 89, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:17.523 INFO Sent repair work, now wait for resp
Sep 22 23:15:17.523 INFO [0] received reconcile message
Sep 22 23:15:17.524 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(714), op: ExtentRepair { repair_id: ReconciliationId(714), extent_id: 89, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:17.524 INFO [0] client ExtentRepair { repair_id: ReconciliationId(714), extent_id: 89, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:17.524 INFO [0] Sending repair request ReconciliationId(714)
Sep 22 23:15:17.524 INFO [1] received reconcile message
Sep 22 23:15:17.524 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(714), op: ExtentRepair { repair_id: ReconciliationId(714), extent_id: 89, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:17.524 INFO [1] client ExtentRepair { repair_id: ReconciliationId(714), extent_id: 89, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:17.524 INFO [1] No action required ReconciliationId(714)
Sep 22 23:15:17.524 INFO [2] received reconcile message
Sep 22 23:15:17.524 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(714), op: ExtentRepair { repair_id: ReconciliationId(714), extent_id: 89, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:17.524 INFO [2] client ExtentRepair { repair_id: ReconciliationId(714), extent_id: 89, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:17.524 INFO [2] No action required ReconciliationId(714)
Sep 22 23:15:17.524 DEBG 714 Repair extent 89 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:17.524 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/059.copy"
Sep 22 23:15:17.554 DEBG [rc] retire 1091 clears [JobId(1090), JobId(1091)], : downstairs
Sep 22 23:15:17.554 ERRO [1] job id 1092 saw error GenericError("test error")
Sep 22 23:15:17.554 DEBG IO Flush 1093 has deps [JobId(1092)]
Sep 22 23:15:17.554 WARN returning error on read!
Sep 22 23:15:17.554 DEBG Read      :1092 deps:[JobId(1091)] res:false
Sep 22 23:15:17.560 DEBG Read      :1092 deps:[JobId(1091)] res:true
Sep 22 23:15:17.582 ERRO [0] job id 1092 saw error GenericError("test error")
Sep 22 23:15:17.584 DEBG Flush     :1093 extent_limit None deps:[JobId(1092)] res:true f:36 g:1
Sep 22 23:15:17.584 INFO [lossy] sleeping 1 second
Sep 22 23:15:17.588 INFO accepted connection, remote_addr: 127.0.0.1:33732, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.588 TRCE incoming request, uri: /extent/89/files, method: GET, req_id: ec39fe56-1327-4046-8ad5-a9420b9458a9, remote_addr: 127.0.0.1:33732, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.588 INFO request completed, latency_us: 193, response_code: 200, uri: /extent/89/files, method: GET, req_id: ec39fe56-1327-4046-8ad5-a9420b9458a9, remote_addr: 127.0.0.1:33732, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.588 INFO eid:89 Found repair files: ["059", "059.db"]
Sep 22 23:15:17.589 TRCE incoming request, uri: /newextent/89/data, method: GET, req_id: 8bd54bd4-e07b-4b71-a72f-c4c152e56ecd, remote_addr: 127.0.0.1:33732, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.589 INFO request completed, latency_us: 319, response_code: 200, uri: /newextent/89/data, method: GET, req_id: 8bd54bd4-e07b-4b71-a72f-c4c152e56ecd, remote_addr: 127.0.0.1:33732, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.594 TRCE incoming request, uri: /newextent/89/db, method: GET, req_id: 4f56a5f6-da06-4545-9695-5b7a865585eb, remote_addr: 127.0.0.1:33732, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.594 INFO request completed, latency_us: 299, response_code: 200, uri: /newextent/89/db, method: GET, req_id: 4f56a5f6-da06-4545-9695-5b7a865585eb, remote_addr: 127.0.0.1:33732, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.596 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/059.copy" to "/tmp/downstairs-vrx8aK6L/00/000/059.replace"
Sep 22 23:15:17.596 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:17.597 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/059.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:17.597 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/059"
Sep 22 23:15:17.597 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/059.db"
Sep 22 23:15:17.597 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:17.597 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/059.replace" to "/tmp/downstairs-vrx8aK6L/00/000/059.completed"
Sep 22 23:15:17.597 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:17.597 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:17.597 DEBG [0] It's time to notify for 714
Sep 22 23:15:17.597 INFO Completion from [0] id:714 status:true
Sep 22 23:15:17.597 INFO [715/752] Repair commands completed
Sep 22 23:15:17.597 INFO Pop front: ReconcileIO { id: ReconciliationId(715), op: ExtentReopen { repair_id: ReconciliationId(715), extent_id: 89 }, state: ClientData([New, New, New]) }
Sep 22 23:15:17.597 INFO Sent repair work, now wait for resp
Sep 22 23:15:17.597 INFO [0] received reconcile message
Sep 22 23:15:17.597 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(715), op: ExtentReopen { repair_id: ReconciliationId(715), extent_id: 89 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:17.597 INFO [0] client ExtentReopen { repair_id: ReconciliationId(715), extent_id: 89 }
Sep 22 23:15:17.598 INFO [1] received reconcile message
Sep 22 23:15:17.598 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(715), op: ExtentReopen { repair_id: ReconciliationId(715), extent_id: 89 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:17.598 INFO [1] client ExtentReopen { repair_id: ReconciliationId(715), extent_id: 89 }
Sep 22 23:15:17.598 INFO [2] received reconcile message
Sep 22 23:15:17.598 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(715), op: ExtentReopen { repair_id: ReconciliationId(715), extent_id: 89 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:17.598 INFO [2] client ExtentReopen { repair_id: ReconciliationId(715), extent_id: 89 }
Sep 22 23:15:17.598 DEBG 715 Reopen extent 89
Sep 22 23:15:17.598 DEBG 715 Reopen extent 89
Sep 22 23:15:17.599 DEBG 715 Reopen extent 89
Sep 22 23:15:17.599 DEBG [2] It's time to notify for 715
Sep 22 23:15:17.599 INFO Completion from [2] id:715 status:true
Sep 22 23:15:17.599 INFO [716/752] Repair commands completed
Sep 22 23:15:17.599 INFO Pop front: ReconcileIO { id: ReconciliationId(716), op: ExtentFlush { repair_id: ReconciliationId(716), extent_id: 126, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:17.600 INFO Sent repair work, now wait for resp
Sep 22 23:15:17.600 INFO [0] received reconcile message
Sep 22 23:15:17.600 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(716), op: ExtentFlush { repair_id: ReconciliationId(716), extent_id: 126, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:17.600 INFO [0] client ExtentFlush { repair_id: ReconciliationId(716), extent_id: 126, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:17.600 INFO [1] received reconcile message
Sep 22 23:15:17.600 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(716), op: ExtentFlush { repair_id: ReconciliationId(716), extent_id: 126, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:17.600 INFO [1] client ExtentFlush { repair_id: ReconciliationId(716), extent_id: 126, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:17.600 INFO [2] received reconcile message
Sep 22 23:15:17.600 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(716), op: ExtentFlush { repair_id: ReconciliationId(716), extent_id: 126, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:17.600 INFO [2] client ExtentFlush { repair_id: ReconciliationId(716), extent_id: 126, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:17.600 DEBG 716 Flush extent 126 with f:2 g:2
Sep 22 23:15:17.600 DEBG Flush just extent 126 with f:2 and g:2
Sep 22 23:15:17.600 DEBG [1] It's time to notify for 716
Sep 22 23:15:17.600 INFO Completion from [1] id:716 status:true
Sep 22 23:15:17.600 INFO [717/752] Repair commands completed
Sep 22 23:15:17.600 INFO Pop front: ReconcileIO { id: ReconciliationId(717), op: ExtentClose { repair_id: ReconciliationId(717), extent_id: 126 }, state: ClientData([New, New, New]) }
Sep 22 23:15:17.600 INFO Sent repair work, now wait for resp
Sep 22 23:15:17.600 INFO [0] received reconcile message
Sep 22 23:15:17.600 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(717), op: ExtentClose { repair_id: ReconciliationId(717), extent_id: 126 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:17.600 INFO [0] client ExtentClose { repair_id: ReconciliationId(717), extent_id: 126 }
Sep 22 23:15:17.600 INFO [1] received reconcile message
Sep 22 23:15:17.600 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(717), op: ExtentClose { repair_id: ReconciliationId(717), extent_id: 126 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:17.600 INFO [1] client ExtentClose { repair_id: ReconciliationId(717), extent_id: 126 }
Sep 22 23:15:17.600 INFO [2] received reconcile message
Sep 22 23:15:17.600 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(717), op: ExtentClose { repair_id: ReconciliationId(717), extent_id: 126 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:17.600 INFO [2] client ExtentClose { repair_id: ReconciliationId(717), extent_id: 126 }
Sep 22 23:15:17.600 DEBG 717 Close extent 126
Sep 22 23:15:17.601 DEBG 717 Close extent 126
Sep 22 23:15:17.601 DEBG 717 Close extent 126
Sep 22 23:15:17.601 DEBG [2] It's time to notify for 717
Sep 22 23:15:17.601 INFO Completion from [2] id:717 status:true
Sep 22 23:15:17.601 INFO [718/752] Repair commands completed
Sep 22 23:15:17.601 INFO Pop front: ReconcileIO { id: ReconciliationId(718), op: ExtentRepair { repair_id: ReconciliationId(718), extent_id: 126, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:17.602 INFO Sent repair work, now wait for resp
Sep 22 23:15:17.602 INFO [0] received reconcile message
Sep 22 23:15:17.602 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(718), op: ExtentRepair { repair_id: ReconciliationId(718), extent_id: 126, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:17.602 INFO [0] client ExtentRepair { repair_id: ReconciliationId(718), extent_id: 126, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:17.602 INFO [0] Sending repair request ReconciliationId(718)
Sep 22 23:15:17.602 INFO [1] received reconcile message
Sep 22 23:15:17.602 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(718), op: ExtentRepair { repair_id: ReconciliationId(718), extent_id: 126, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:17.602 INFO [1] client ExtentRepair { repair_id: ReconciliationId(718), extent_id: 126, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:17.602 INFO [1] No action required ReconciliationId(718)
Sep 22 23:15:17.602 INFO [2] received reconcile message
Sep 22 23:15:17.602 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(718), op: ExtentRepair { repair_id: ReconciliationId(718), extent_id: 126, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:17.602 INFO [2] client ExtentRepair { repair_id: ReconciliationId(718), extent_id: 126, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:17.602 INFO [2] No action required ReconciliationId(718)
Sep 22 23:15:17.602 DEBG 718 Repair extent 126 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:17.602 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/07E.copy"
Sep 22 23:15:17.664 INFO accepted connection, remote_addr: 127.0.0.1:34817, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.664 TRCE incoming request, uri: /extent/126/files, method: GET, req_id: 6ad0c804-5c72-4e65-a940-a0aa1d4918d6, remote_addr: 127.0.0.1:34817, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.665 INFO request completed, latency_us: 191, response_code: 200, uri: /extent/126/files, method: GET, req_id: 6ad0c804-5c72-4e65-a940-a0aa1d4918d6, remote_addr: 127.0.0.1:34817, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.665 INFO eid:126 Found repair files: ["07E", "07E.db"]
Sep 22 23:15:17.665 TRCE incoming request, uri: /newextent/126/data, method: GET, req_id: a32e0d22-0263-4725-91c3-a8a6b0705f1d, remote_addr: 127.0.0.1:34817, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.665 INFO request completed, latency_us: 303, response_code: 200, uri: /newextent/126/data, method: GET, req_id: a32e0d22-0263-4725-91c3-a8a6b0705f1d, remote_addr: 127.0.0.1:34817, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.670 TRCE incoming request, uri: /newextent/126/db, method: GET, req_id: ad0256fe-142c-455e-92cf-ac92ed7fbca1, remote_addr: 127.0.0.1:34817, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.671 INFO request completed, latency_us: 294, response_code: 200, uri: /newextent/126/db, method: GET, req_id: ad0256fe-142c-455e-92cf-ac92ed7fbca1, remote_addr: 127.0.0.1:34817, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.672 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/07E.copy" to "/tmp/downstairs-vrx8aK6L/00/000/07E.replace"
Sep 22 23:15:17.672 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:17.673 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/07E.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:17.673 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/07E"
Sep 22 23:15:17.673 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/07E.db"
Sep 22 23:15:17.673 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:17.673 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/07E.replace" to "/tmp/downstairs-vrx8aK6L/00/000/07E.completed"
Sep 22 23:15:17.673 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:17.673 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:17.673 DEBG [0] It's time to notify for 718
Sep 22 23:15:17.673 INFO Completion from [0] id:718 status:true
Sep 22 23:15:17.674 INFO [719/752] Repair commands completed
Sep 22 23:15:17.674 INFO Pop front: ReconcileIO { id: ReconciliationId(719), op: ExtentReopen { repair_id: ReconciliationId(719), extent_id: 126 }, state: ClientData([New, New, New]) }
Sep 22 23:15:17.674 INFO Sent repair work, now wait for resp
Sep 22 23:15:17.674 INFO [0] received reconcile message
Sep 22 23:15:17.674 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(719), op: ExtentReopen { repair_id: ReconciliationId(719), extent_id: 126 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:17.674 INFO [0] client ExtentReopen { repair_id: ReconciliationId(719), extent_id: 126 }
Sep 22 23:15:17.674 INFO [1] received reconcile message
Sep 22 23:15:17.674 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(719), op: ExtentReopen { repair_id: ReconciliationId(719), extent_id: 126 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:17.674 INFO [1] client ExtentReopen { repair_id: ReconciliationId(719), extent_id: 126 }
Sep 22 23:15:17.674 INFO [2] received reconcile message
Sep 22 23:15:17.674 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(719), op: ExtentReopen { repair_id: ReconciliationId(719), extent_id: 126 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:17.674 INFO [2] client ExtentReopen { repair_id: ReconciliationId(719), extent_id: 126 }
Sep 22 23:15:17.674 DEBG 719 Reopen extent 126
Sep 22 23:15:17.674 DEBG 719 Reopen extent 126
Sep 22 23:15:17.675 DEBG 719 Reopen extent 126
Sep 22 23:15:17.676 DEBG [2] It's time to notify for 719
Sep 22 23:15:17.676 INFO Completion from [2] id:719 status:true
Sep 22 23:15:17.676 INFO [720/752] Repair commands completed
Sep 22 23:15:17.676 INFO Pop front: ReconcileIO { id: ReconciliationId(720), op: ExtentFlush { repair_id: ReconciliationId(720), extent_id: 186, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:17.676 INFO Sent repair work, now wait for resp
Sep 22 23:15:17.676 INFO [0] received reconcile message
Sep 22 23:15:17.676 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(720), op: ExtentFlush { repair_id: ReconciliationId(720), extent_id: 186, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:17.676 INFO [0] client ExtentFlush { repair_id: ReconciliationId(720), extent_id: 186, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:17.676 INFO [1] received reconcile message
Sep 22 23:15:17.676 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(720), op: ExtentFlush { repair_id: ReconciliationId(720), extent_id: 186, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:17.676 INFO [1] client ExtentFlush { repair_id: ReconciliationId(720), extent_id: 186, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:17.676 INFO [2] received reconcile message
Sep 22 23:15:17.676 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(720), op: ExtentFlush { repair_id: ReconciliationId(720), extent_id: 186, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:17.676 INFO [2] client ExtentFlush { repair_id: ReconciliationId(720), extent_id: 186, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:17.676 DEBG 720 Flush extent 186 with f:2 g:2
Sep 22 23:15:17.676 DEBG Flush just extent 186 with f:2 and g:2
Sep 22 23:15:17.676 DEBG [1] It's time to notify for 720
Sep 22 23:15:17.676 INFO Completion from [1] id:720 status:true
Sep 22 23:15:17.676 INFO [721/752] Repair commands completed
Sep 22 23:15:17.676 INFO Pop front: ReconcileIO { id: ReconciliationId(721), op: ExtentClose { repair_id: ReconciliationId(721), extent_id: 186 }, state: ClientData([New, New, New]) }
Sep 22 23:15:17.676 INFO Sent repair work, now wait for resp
Sep 22 23:15:17.676 INFO [0] received reconcile message
Sep 22 23:15:17.676 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(721), op: ExtentClose { repair_id: ReconciliationId(721), extent_id: 186 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:17.676 INFO [0] client ExtentClose { repair_id: ReconciliationId(721), extent_id: 186 }
Sep 22 23:15:17.676 INFO [1] received reconcile message
Sep 22 23:15:17.676 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(721), op: ExtentClose { repair_id: ReconciliationId(721), extent_id: 186 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:17.676 INFO [1] client ExtentClose { repair_id: ReconciliationId(721), extent_id: 186 }
Sep 22 23:15:17.676 INFO [2] received reconcile message
Sep 22 23:15:17.676 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(721), op: ExtentClose { repair_id: ReconciliationId(721), extent_id: 186 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:17.676 INFO [2] client ExtentClose { repair_id: ReconciliationId(721), extent_id: 186 }
Sep 22 23:15:17.677 DEBG 721 Close extent 186
Sep 22 23:15:17.677 DEBG 721 Close extent 186
Sep 22 23:15:17.677 DEBG 721 Close extent 186
Sep 22 23:15:17.678 DEBG [2] It's time to notify for 721
Sep 22 23:15:17.678 INFO Completion from [2] id:721 status:true
Sep 22 23:15:17.678 INFO [722/752] Repair commands completed
Sep 22 23:15:17.678 INFO Pop front: ReconcileIO { id: ReconciliationId(722), op: ExtentRepair { repair_id: ReconciliationId(722), extent_id: 186, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:17.678 INFO Sent repair work, now wait for resp
Sep 22 23:15:17.678 INFO [0] received reconcile message
Sep 22 23:15:17.678 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(722), op: ExtentRepair { repair_id: ReconciliationId(722), extent_id: 186, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:17.678 INFO [0] client ExtentRepair { repair_id: ReconciliationId(722), extent_id: 186, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:17.678 INFO [0] Sending repair request ReconciliationId(722)
Sep 22 23:15:17.678 INFO [1] received reconcile message
Sep 22 23:15:17.678 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(722), op: ExtentRepair { repair_id: ReconciliationId(722), extent_id: 186, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:17.678 INFO [1] client ExtentRepair { repair_id: ReconciliationId(722), extent_id: 186, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:17.678 INFO [1] No action required ReconciliationId(722)
Sep 22 23:15:17.678 INFO [2] received reconcile message
Sep 22 23:15:17.678 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(722), op: ExtentRepair { repair_id: ReconciliationId(722), extent_id: 186, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:17.678 INFO [2] client ExtentRepair { repair_id: ReconciliationId(722), extent_id: 186, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:17.678 INFO [2] No action required ReconciliationId(722)
Sep 22 23:15:17.678 DEBG 722 Repair extent 186 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:17.678 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/0BA.copy"
Sep 22 23:15:17.740 INFO accepted connection, remote_addr: 127.0.0.1:52544, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.741 TRCE incoming request, uri: /extent/186/files, method: GET, req_id: 11244d44-9860-4092-be8e-9b0f980528de, remote_addr: 127.0.0.1:52544, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.741 INFO request completed, latency_us: 197, response_code: 200, uri: /extent/186/files, method: GET, req_id: 11244d44-9860-4092-be8e-9b0f980528de, remote_addr: 127.0.0.1:52544, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.741 INFO eid:186 Found repair files: ["0BA", "0BA.db"]
Sep 22 23:15:17.741 TRCE incoming request, uri: /newextent/186/data, method: GET, req_id: 4a474cbb-2561-4299-909a-78ac2b46ee6e, remote_addr: 127.0.0.1:52544, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.742 INFO request completed, latency_us: 252, response_code: 200, uri: /newextent/186/data, method: GET, req_id: 4a474cbb-2561-4299-909a-78ac2b46ee6e, remote_addr: 127.0.0.1:52544, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.746 TRCE incoming request, uri: /newextent/186/db, method: GET, req_id: a0a99c1f-37da-421f-9c68-4c79fcc799a3, remote_addr: 127.0.0.1:52544, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.747 INFO request completed, latency_us: 292, response_code: 200, uri: /newextent/186/db, method: GET, req_id: a0a99c1f-37da-421f-9c68-4c79fcc799a3, remote_addr: 127.0.0.1:52544, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.748 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/0BA.copy" to "/tmp/downstairs-vrx8aK6L/00/000/0BA.replace"
Sep 22 23:15:17.748 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:17.749 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/0BA.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:17.749 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/0BA"
Sep 22 23:15:17.749 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/0BA.db"
Sep 22 23:15:17.749 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:17.749 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/0BA.replace" to "/tmp/downstairs-vrx8aK6L/00/000/0BA.completed"
Sep 22 23:15:17.749 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:17.749 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:17.749 DEBG [0] It's time to notify for 722
Sep 22 23:15:17.749 INFO Completion from [0] id:722 status:true
Sep 22 23:15:17.749 INFO [723/752] Repair commands completed
Sep 22 23:15:17.749 INFO Pop front: ReconcileIO { id: ReconciliationId(723), op: ExtentReopen { repair_id: ReconciliationId(723), extent_id: 186 }, state: ClientData([New, New, New]) }
Sep 22 23:15:17.750 INFO Sent repair work, now wait for resp
Sep 22 23:15:17.750 INFO [0] received reconcile message
Sep 22 23:15:17.750 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(723), op: ExtentReopen { repair_id: ReconciliationId(723), extent_id: 186 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:17.750 INFO [0] client ExtentReopen { repair_id: ReconciliationId(723), extent_id: 186 }
Sep 22 23:15:17.750 INFO [1] received reconcile message
Sep 22 23:15:17.750 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(723), op: ExtentReopen { repair_id: ReconciliationId(723), extent_id: 186 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:17.750 INFO [1] client ExtentReopen { repair_id: ReconciliationId(723), extent_id: 186 }
Sep 22 23:15:17.750 INFO [2] received reconcile message
Sep 22 23:15:17.750 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(723), op: ExtentReopen { repair_id: ReconciliationId(723), extent_id: 186 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:17.750 INFO [2] client ExtentReopen { repair_id: ReconciliationId(723), extent_id: 186 }
Sep 22 23:15:17.750 DEBG 723 Reopen extent 186
Sep 22 23:15:17.750 DEBG 723 Reopen extent 186
Sep 22 23:15:17.751 DEBG 723 Reopen extent 186
Sep 22 23:15:17.751 DEBG [2] It's time to notify for 723
Sep 22 23:15:17.752 INFO Completion from [2] id:723 status:true
Sep 22 23:15:17.752 INFO [724/752] Repair commands completed
Sep 22 23:15:17.752 INFO Pop front: ReconcileIO { id: ReconciliationId(724), op: ExtentFlush { repair_id: ReconciliationId(724), extent_id: 108, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:17.752 INFO Sent repair work, now wait for resp
Sep 22 23:15:17.752 INFO [0] received reconcile message
Sep 22 23:15:17.752 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(724), op: ExtentFlush { repair_id: ReconciliationId(724), extent_id: 108, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:17.752 INFO [0] client ExtentFlush { repair_id: ReconciliationId(724), extent_id: 108, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:17.752 INFO [1] received reconcile message
Sep 22 23:15:17.752 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(724), op: ExtentFlush { repair_id: ReconciliationId(724), extent_id: 108, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:17.752 INFO [1] client ExtentFlush { repair_id: ReconciliationId(724), extent_id: 108, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:17.752 INFO [2] received reconcile message
Sep 22 23:15:17.752 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(724), op: ExtentFlush { repair_id: ReconciliationId(724), extent_id: 108, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:17.752 INFO [2] client ExtentFlush { repair_id: ReconciliationId(724), extent_id: 108, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:17.752 DEBG 724 Flush extent 108 with f:2 g:2
Sep 22 23:15:17.752 DEBG Flush just extent 108 with f:2 and g:2
Sep 22 23:15:17.752 DEBG [1] It's time to notify for 724
Sep 22 23:15:17.752 INFO Completion from [1] id:724 status:true
Sep 22 23:15:17.752 INFO [725/752] Repair commands completed
Sep 22 23:15:17.752 INFO Pop front: ReconcileIO { id: ReconciliationId(725), op: ExtentClose { repair_id: ReconciliationId(725), extent_id: 108 }, state: ClientData([New, New, New]) }
Sep 22 23:15:17.752 INFO Sent repair work, now wait for resp
Sep 22 23:15:17.752 INFO [0] received reconcile message
Sep 22 23:15:17.752 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(725), op: ExtentClose { repair_id: ReconciliationId(725), extent_id: 108 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:17.752 INFO [0] client ExtentClose { repair_id: ReconciliationId(725), extent_id: 108 }
Sep 22 23:15:17.752 INFO [1] received reconcile message
Sep 22 23:15:17.752 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(725), op: ExtentClose { repair_id: ReconciliationId(725), extent_id: 108 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:17.752 INFO [1] client ExtentClose { repair_id: ReconciliationId(725), extent_id: 108 }
Sep 22 23:15:17.752 INFO [2] received reconcile message
Sep 22 23:15:17.752 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(725), op: ExtentClose { repair_id: ReconciliationId(725), extent_id: 108 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:17.752 INFO [2] client ExtentClose { repair_id: ReconciliationId(725), extent_id: 108 }
Sep 22 23:15:17.752 DEBG 725 Close extent 108
Sep 22 23:15:17.753 DEBG 725 Close extent 108
Sep 22 23:15:17.753 DEBG 725 Close extent 108
Sep 22 23:15:17.753 DEBG [2] It's time to notify for 725
Sep 22 23:15:17.753 INFO Completion from [2] id:725 status:true
Sep 22 23:15:17.753 INFO [726/752] Repair commands completed
Sep 22 23:15:17.754 INFO Pop front: ReconcileIO { id: ReconciliationId(726), op: ExtentRepair { repair_id: ReconciliationId(726), extent_id: 108, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:17.754 INFO Sent repair work, now wait for resp
Sep 22 23:15:17.754 INFO [0] received reconcile message
Sep 22 23:15:17.754 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(726), op: ExtentRepair { repair_id: ReconciliationId(726), extent_id: 108, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:17.754 INFO [0] client ExtentRepair { repair_id: ReconciliationId(726), extent_id: 108, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:17.754 INFO [0] Sending repair request ReconciliationId(726)
Sep 22 23:15:17.754 INFO [1] received reconcile message
Sep 22 23:15:17.754 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(726), op: ExtentRepair { repair_id: ReconciliationId(726), extent_id: 108, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:17.754 INFO [1] client ExtentRepair { repair_id: ReconciliationId(726), extent_id: 108, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:17.754 INFO [1] No action required ReconciliationId(726)
Sep 22 23:15:17.754 INFO [2] received reconcile message
Sep 22 23:15:17.754 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(726), op: ExtentRepair { repair_id: ReconciliationId(726), extent_id: 108, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:17.754 INFO [2] client ExtentRepair { repair_id: ReconciliationId(726), extent_id: 108, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:17.754 INFO [2] No action required ReconciliationId(726)
Sep 22 23:15:17.754 DEBG 726 Repair extent 108 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:17.754 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/06C.copy"
Sep 22 23:15:17.817 INFO accepted connection, remote_addr: 127.0.0.1:60392, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.817 TRCE incoming request, uri: /extent/108/files, method: GET, req_id: 90e0864b-9dea-4701-b38f-ca91371dc188, remote_addr: 127.0.0.1:60392, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.817 INFO request completed, latency_us: 191, response_code: 200, uri: /extent/108/files, method: GET, req_id: 90e0864b-9dea-4701-b38f-ca91371dc188, remote_addr: 127.0.0.1:60392, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.817 INFO eid:108 Found repair files: ["06C", "06C.db"]
Sep 22 23:15:17.818 TRCE incoming request, uri: /newextent/108/data, method: GET, req_id: c6b0710d-19cb-494e-a0b5-0f0b7808191b, remote_addr: 127.0.0.1:60392, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.818 INFO request completed, latency_us: 340, response_code: 200, uri: /newextent/108/data, method: GET, req_id: c6b0710d-19cb-494e-a0b5-0f0b7808191b, remote_addr: 127.0.0.1:60392, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.823 TRCE incoming request, uri: /newextent/108/db, method: GET, req_id: 2dee5474-5181-45cf-b510-030116f445c2, remote_addr: 127.0.0.1:60392, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.823 INFO request completed, latency_us: 296, response_code: 200, uri: /newextent/108/db, method: GET, req_id: 2dee5474-5181-45cf-b510-030116f445c2, remote_addr: 127.0.0.1:60392, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.825 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/06C.copy" to "/tmp/downstairs-vrx8aK6L/00/000/06C.replace"
Sep 22 23:15:17.825 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:17.825 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/06C.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:17.826 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/06C"
Sep 22 23:15:17.826 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/06C.db"
Sep 22 23:15:17.826 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:17.826 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/06C.replace" to "/tmp/downstairs-vrx8aK6L/00/000/06C.completed"
Sep 22 23:15:17.826 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:17.826 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:17.826 DEBG [0] It's time to notify for 726
Sep 22 23:15:17.826 INFO Completion from [0] id:726 status:true
Sep 22 23:15:17.826 INFO [727/752] Repair commands completed
Sep 22 23:15:17.826 INFO Pop front: ReconcileIO { id: ReconciliationId(727), op: ExtentReopen { repair_id: ReconciliationId(727), extent_id: 108 }, state: ClientData([New, New, New]) }
Sep 22 23:15:17.826 INFO Sent repair work, now wait for resp
Sep 22 23:15:17.826 INFO [0] received reconcile message
Sep 22 23:15:17.826 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(727), op: ExtentReopen { repair_id: ReconciliationId(727), extent_id: 108 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:17.826 INFO [0] client ExtentReopen { repair_id: ReconciliationId(727), extent_id: 108 }
Sep 22 23:15:17.826 INFO [1] received reconcile message
Sep 22 23:15:17.826 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(727), op: ExtentReopen { repair_id: ReconciliationId(727), extent_id: 108 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:17.826 INFO [1] client ExtentReopen { repair_id: ReconciliationId(727), extent_id: 108 }
Sep 22 23:15:17.826 INFO [2] received reconcile message
Sep 22 23:15:17.826 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(727), op: ExtentReopen { repair_id: ReconciliationId(727), extent_id: 108 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:17.826 INFO [2] client ExtentReopen { repair_id: ReconciliationId(727), extent_id: 108 }
Sep 22 23:15:17.827 DEBG 727 Reopen extent 108
Sep 22 23:15:17.827 DEBG 727 Reopen extent 108
Sep 22 23:15:17.828 DEBG 727 Reopen extent 108
Sep 22 23:15:17.828 DEBG [2] It's time to notify for 727
Sep 22 23:15:17.828 INFO Completion from [2] id:727 status:true
Sep 22 23:15:17.828 INFO [728/752] Repair commands completed
Sep 22 23:15:17.828 INFO Pop front: ReconcileIO { id: ReconciliationId(728), op: ExtentFlush { repair_id: ReconciliationId(728), extent_id: 109, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:17.828 INFO Sent repair work, now wait for resp
Sep 22 23:15:17.828 INFO [0] received reconcile message
Sep 22 23:15:17.828 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(728), op: ExtentFlush { repair_id: ReconciliationId(728), extent_id: 109, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:17.828 INFO [0] client ExtentFlush { repair_id: ReconciliationId(728), extent_id: 109, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:17.828 INFO [1] received reconcile message
Sep 22 23:15:17.828 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(728), op: ExtentFlush { repair_id: ReconciliationId(728), extent_id: 109, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:17.828 INFO [1] client ExtentFlush { repair_id: ReconciliationId(728), extent_id: 109, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:17.829 INFO [2] received reconcile message
Sep 22 23:15:17.829 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(728), op: ExtentFlush { repair_id: ReconciliationId(728), extent_id: 109, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:17.829 INFO [2] client ExtentFlush { repair_id: ReconciliationId(728), extent_id: 109, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:17.829 DEBG 728 Flush extent 109 with f:2 g:2
Sep 22 23:15:17.829 DEBG Flush just extent 109 with f:2 and g:2
Sep 22 23:15:17.829 DEBG [1] It's time to notify for 728
Sep 22 23:15:17.829 INFO Completion from [1] id:728 status:true
Sep 22 23:15:17.829 INFO [729/752] Repair commands completed
Sep 22 23:15:17.829 INFO Pop front: ReconcileIO { id: ReconciliationId(729), op: ExtentClose { repair_id: ReconciliationId(729), extent_id: 109 }, state: ClientData([New, New, New]) }
Sep 22 23:15:17.829 INFO Sent repair work, now wait for resp
Sep 22 23:15:17.829 INFO [0] received reconcile message
Sep 22 23:15:17.829 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(729), op: ExtentClose { repair_id: ReconciliationId(729), extent_id: 109 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:17.829 INFO [0] client ExtentClose { repair_id: ReconciliationId(729), extent_id: 109 }
Sep 22 23:15:17.829 INFO [1] received reconcile message
Sep 22 23:15:17.829 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(729), op: ExtentClose { repair_id: ReconciliationId(729), extent_id: 109 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:17.829 INFO [1] client ExtentClose { repair_id: ReconciliationId(729), extent_id: 109 }
Sep 22 23:15:17.829 INFO [2] received reconcile message
Sep 22 23:15:17.829 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(729), op: ExtentClose { repair_id: ReconciliationId(729), extent_id: 109 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:17.829 INFO [2] client ExtentClose { repair_id: ReconciliationId(729), extent_id: 109 }
Sep 22 23:15:17.829 DEBG 729 Close extent 109
Sep 22 23:15:17.829 DEBG 729 Close extent 109
Sep 22 23:15:17.830 DEBG 729 Close extent 109
Sep 22 23:15:17.830 DEBG [2] It's time to notify for 729
Sep 22 23:15:17.830 INFO Completion from [2] id:729 status:true
Sep 22 23:15:17.830 INFO [730/752] Repair commands completed
Sep 22 23:15:17.830 INFO Pop front: ReconcileIO { id: ReconciliationId(730), op: ExtentRepair { repair_id: ReconciliationId(730), extent_id: 109, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:17.830 INFO Sent repair work, now wait for resp
Sep 22 23:15:17.830 INFO [0] received reconcile message
Sep 22 23:15:17.830 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(730), op: ExtentRepair { repair_id: ReconciliationId(730), extent_id: 109, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:17.830 INFO [0] client ExtentRepair { repair_id: ReconciliationId(730), extent_id: 109, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:17.830 INFO [0] Sending repair request ReconciliationId(730)
Sep 22 23:15:17.830 INFO [1] received reconcile message
Sep 22 23:15:17.830 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(730), op: ExtentRepair { repair_id: ReconciliationId(730), extent_id: 109, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:17.830 INFO [1] client ExtentRepair { repair_id: ReconciliationId(730), extent_id: 109, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:17.831 INFO [1] No action required ReconciliationId(730)
Sep 22 23:15:17.831 INFO [2] received reconcile message
Sep 22 23:15:17.831 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(730), op: ExtentRepair { repair_id: ReconciliationId(730), extent_id: 109, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:17.831 INFO [2] client ExtentRepair { repair_id: ReconciliationId(730), extent_id: 109, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:17.831 INFO [2] No action required ReconciliationId(730)
Sep 22 23:15:17.831 DEBG 730 Repair extent 109 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:17.831 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/06D.copy"
Sep 22 23:15:17.894 INFO accepted connection, remote_addr: 127.0.0.1:62743, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.894 TRCE incoming request, uri: /extent/109/files, method: GET, req_id: ae0ac088-e3ba-4d0f-8c98-820b95913887, remote_addr: 127.0.0.1:62743, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.895 INFO request completed, latency_us: 193, response_code: 200, uri: /extent/109/files, method: GET, req_id: ae0ac088-e3ba-4d0f-8c98-820b95913887, remote_addr: 127.0.0.1:62743, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.895 INFO eid:109 Found repair files: ["06D", "06D.db"]
Sep 22 23:15:17.895 TRCE incoming request, uri: /newextent/109/data, method: GET, req_id: ed958c89-ec9c-4ebe-bf0c-82f6f455cbff, remote_addr: 127.0.0.1:62743, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.895 INFO request completed, latency_us: 301, response_code: 200, uri: /newextent/109/data, method: GET, req_id: ed958c89-ec9c-4ebe-bf0c-82f6f455cbff, remote_addr: 127.0.0.1:62743, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.900 TRCE incoming request, uri: /newextent/109/db, method: GET, req_id: 0f231f08-3577-45af-84e1-f6a62d5a334b, remote_addr: 127.0.0.1:62743, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.901 INFO request completed, latency_us: 293, response_code: 200, uri: /newextent/109/db, method: GET, req_id: 0f231f08-3577-45af-84e1-f6a62d5a334b, remote_addr: 127.0.0.1:62743, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.902 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/06D.copy" to "/tmp/downstairs-vrx8aK6L/00/000/06D.replace"
Sep 22 23:15:17.902 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:17.903 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/06D.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:17.903 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/06D"
Sep 22 23:15:17.903 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/06D.db"
Sep 22 23:15:17.903 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:17.903 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/06D.replace" to "/tmp/downstairs-vrx8aK6L/00/000/06D.completed"
Sep 22 23:15:17.903 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:17.903 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:17.903 DEBG [0] It's time to notify for 730
Sep 22 23:15:17.903 INFO Completion from [0] id:730 status:true
Sep 22 23:15:17.903 INFO [731/752] Repair commands completed
Sep 22 23:15:17.903 INFO Pop front: ReconcileIO { id: ReconciliationId(731), op: ExtentReopen { repair_id: ReconciliationId(731), extent_id: 109 }, state: ClientData([New, New, New]) }
Sep 22 23:15:17.903 INFO Sent repair work, now wait for resp
Sep 22 23:15:17.903 INFO [0] received reconcile message
Sep 22 23:15:17.903 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(731), op: ExtentReopen { repair_id: ReconciliationId(731), extent_id: 109 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:17.903 INFO [0] client ExtentReopen { repair_id: ReconciliationId(731), extent_id: 109 }
Sep 22 23:15:17.904 INFO [1] received reconcile message
Sep 22 23:15:17.904 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(731), op: ExtentReopen { repair_id: ReconciliationId(731), extent_id: 109 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:17.904 INFO [1] client ExtentReopen { repair_id: ReconciliationId(731), extent_id: 109 }
Sep 22 23:15:17.904 INFO [2] received reconcile message
Sep 22 23:15:17.904 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(731), op: ExtentReopen { repair_id: ReconciliationId(731), extent_id: 109 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:17.904 INFO [2] client ExtentReopen { repair_id: ReconciliationId(731), extent_id: 109 }
Sep 22 23:15:17.904 DEBG 731 Reopen extent 109
Sep 22 23:15:17.904 DEBG 731 Reopen extent 109
Sep 22 23:15:17.905 DEBG 731 Reopen extent 109
Sep 22 23:15:17.905 DEBG [2] It's time to notify for 731
Sep 22 23:15:17.906 INFO Completion from [2] id:731 status:true
Sep 22 23:15:17.906 INFO [732/752] Repair commands completed
Sep 22 23:15:17.906 INFO Pop front: ReconcileIO { id: ReconciliationId(732), op: ExtentFlush { repair_id: ReconciliationId(732), extent_id: 104, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:17.906 INFO Sent repair work, now wait for resp
Sep 22 23:15:17.906 INFO [0] received reconcile message
Sep 22 23:15:17.906 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(732), op: ExtentFlush { repair_id: ReconciliationId(732), extent_id: 104, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:17.906 INFO [0] client ExtentFlush { repair_id: ReconciliationId(732), extent_id: 104, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:17.906 INFO [1] received reconcile message
Sep 22 23:15:17.906 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(732), op: ExtentFlush { repair_id: ReconciliationId(732), extent_id: 104, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:17.906 INFO [1] client ExtentFlush { repair_id: ReconciliationId(732), extent_id: 104, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:17.906 INFO [2] received reconcile message
Sep 22 23:15:17.906 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(732), op: ExtentFlush { repair_id: ReconciliationId(732), extent_id: 104, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:17.906 INFO [2] client ExtentFlush { repair_id: ReconciliationId(732), extent_id: 104, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:17.906 DEBG 732 Flush extent 104 with f:2 g:2
Sep 22 23:15:17.906 DEBG Flush just extent 104 with f:2 and g:2
Sep 22 23:15:17.906 DEBG [1] It's time to notify for 732
Sep 22 23:15:17.906 INFO Completion from [1] id:732 status:true
Sep 22 23:15:17.906 INFO [733/752] Repair commands completed
Sep 22 23:15:17.906 INFO Pop front: ReconcileIO { id: ReconciliationId(733), op: ExtentClose { repair_id: ReconciliationId(733), extent_id: 104 }, state: ClientData([New, New, New]) }
Sep 22 23:15:17.906 INFO Sent repair work, now wait for resp
Sep 22 23:15:17.906 INFO [0] received reconcile message
Sep 22 23:15:17.906 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(733), op: ExtentClose { repair_id: ReconciliationId(733), extent_id: 104 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:17.906 INFO [0] client ExtentClose { repair_id: ReconciliationId(733), extent_id: 104 }
Sep 22 23:15:17.906 INFO [1] received reconcile message
Sep 22 23:15:17.906 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(733), op: ExtentClose { repair_id: ReconciliationId(733), extent_id: 104 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:17.906 INFO [1] client ExtentClose { repair_id: ReconciliationId(733), extent_id: 104 }
Sep 22 23:15:17.906 INFO [2] received reconcile message
Sep 22 23:15:17.906 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(733), op: ExtentClose { repair_id: ReconciliationId(733), extent_id: 104 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:17.906 INFO [2] client ExtentClose { repair_id: ReconciliationId(733), extent_id: 104 }
Sep 22 23:15:17.907 DEBG 733 Close extent 104
Sep 22 23:15:17.907 DEBG 733 Close extent 104
Sep 22 23:15:17.907 DEBG 733 Close extent 104
Sep 22 23:15:17.907 DEBG [2] It's time to notify for 733
Sep 22 23:15:17.908 INFO Completion from [2] id:733 status:true
Sep 22 23:15:17.908 INFO [734/752] Repair commands completed
Sep 22 23:15:17.908 INFO Pop front: ReconcileIO { id: ReconciliationId(734), op: ExtentRepair { repair_id: ReconciliationId(734), extent_id: 104, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:17.908 INFO Sent repair work, now wait for resp
Sep 22 23:15:17.908 INFO [0] received reconcile message
Sep 22 23:15:17.908 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(734), op: ExtentRepair { repair_id: ReconciliationId(734), extent_id: 104, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:17.908 INFO [0] client ExtentRepair { repair_id: ReconciliationId(734), extent_id: 104, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:17.908 INFO [0] Sending repair request ReconciliationId(734)
Sep 22 23:15:17.908 INFO [1] received reconcile message
Sep 22 23:15:17.908 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(734), op: ExtentRepair { repair_id: ReconciliationId(734), extent_id: 104, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:17.908 INFO [1] client ExtentRepair { repair_id: ReconciliationId(734), extent_id: 104, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:17.908 INFO [1] No action required ReconciliationId(734)
Sep 22 23:15:17.908 INFO [2] received reconcile message
Sep 22 23:15:17.908 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(734), op: ExtentRepair { repair_id: ReconciliationId(734), extent_id: 104, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:17.908 INFO [2] client ExtentRepair { repair_id: ReconciliationId(734), extent_id: 104, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:17.908 INFO [2] No action required ReconciliationId(734)
Sep 22 23:15:17.908 DEBG 734 Repair extent 104 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:17.908 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/068.copy"
Sep 22 23:15:17.961 DEBG [1] Read AckReady 1092, : downstairs
Sep 22 23:15:17.962 DEBG up_ds_listen was notified
Sep 22 23:15:17.963 DEBG up_ds_listen process 1092
Sep 22 23:15:17.963 DEBG [A] ack job 1092:93, : downstairs
Sep 22 23:15:17.972 INFO accepted connection, remote_addr: 127.0.0.1:49958, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.972 TRCE incoming request, uri: /extent/104/files, method: GET, req_id: 79b6d2a3-d936-4eed-a693-b4eee9fedc57, remote_addr: 127.0.0.1:49958, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.972 INFO request completed, latency_us: 202, response_code: 200, uri: /extent/104/files, method: GET, req_id: 79b6d2a3-d936-4eed-a693-b4eee9fedc57, remote_addr: 127.0.0.1:49958, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.972 INFO eid:104 Found repair files: ["068", "068.db"]
Sep 22 23:15:17.973 TRCE incoming request, uri: /newextent/104/data, method: GET, req_id: 71ea988e-ecf0-4760-b633-fee5a72f2bdc, remote_addr: 127.0.0.1:49958, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.973 INFO request completed, latency_us: 325, response_code: 200, uri: /newextent/104/data, method: GET, req_id: 71ea988e-ecf0-4760-b633-fee5a72f2bdc, remote_addr: 127.0.0.1:49958, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.978 TRCE incoming request, uri: /newextent/104/db, method: GET, req_id: 81dac8df-1e7b-4bf3-b0e5-a754a5ba78ec, remote_addr: 127.0.0.1:49958, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.978 INFO request completed, latency_us: 294, response_code: 200, uri: /newextent/104/db, method: GET, req_id: 81dac8df-1e7b-4bf3-b0e5-a754a5ba78ec, remote_addr: 127.0.0.1:49958, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:17.979 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/068.copy" to "/tmp/downstairs-vrx8aK6L/00/000/068.replace"
Sep 22 23:15:17.979 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:17.980 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/068.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:17.980 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/068"
Sep 22 23:15:17.981 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/068.db"
Sep 22 23:15:17.981 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:17.981 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/068.replace" to "/tmp/downstairs-vrx8aK6L/00/000/068.completed"
Sep 22 23:15:17.981 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:17.981 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:17.981 DEBG [0] It's time to notify for 734
Sep 22 23:15:17.981 INFO Completion from [0] id:734 status:true
Sep 22 23:15:17.981 INFO [735/752] Repair commands completed
Sep 22 23:15:17.981 INFO Pop front: ReconcileIO { id: ReconciliationId(735), op: ExtentReopen { repair_id: ReconciliationId(735), extent_id: 104 }, state: ClientData([New, New, New]) }
Sep 22 23:15:17.981 INFO Sent repair work, now wait for resp
Sep 22 23:15:17.981 INFO [0] received reconcile message
Sep 22 23:15:17.981 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(735), op: ExtentReopen { repair_id: ReconciliationId(735), extent_id: 104 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:17.981 INFO [0] client ExtentReopen { repair_id: ReconciliationId(735), extent_id: 104 }
Sep 22 23:15:17.981 INFO [1] received reconcile message
Sep 22 23:15:17.981 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(735), op: ExtentReopen { repair_id: ReconciliationId(735), extent_id: 104 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:17.981 INFO [1] client ExtentReopen { repair_id: ReconciliationId(735), extent_id: 104 }
Sep 22 23:15:17.981 INFO [2] received reconcile message
Sep 22 23:15:17.981 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(735), op: ExtentReopen { repair_id: ReconciliationId(735), extent_id: 104 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:17.981 INFO [2] client ExtentReopen { repair_id: ReconciliationId(735), extent_id: 104 }
Sep 22 23:15:17.981 DEBG 735 Reopen extent 104
Sep 22 23:15:17.982 DEBG 735 Reopen extent 104
Sep 22 23:15:17.983 DEBG 735 Reopen extent 104
Sep 22 23:15:17.983 DEBG [2] It's time to notify for 735
Sep 22 23:15:17.983 INFO Completion from [2] id:735 status:true
Sep 22 23:15:17.983 INFO [736/752] Repair commands completed
Sep 22 23:15:17.983 INFO Pop front: ReconcileIO { id: ReconciliationId(736), op: ExtentFlush { repair_id: ReconciliationId(736), extent_id: 101, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:17.983 INFO Sent repair work, now wait for resp
Sep 22 23:15:17.983 INFO [0] received reconcile message
Sep 22 23:15:17.983 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(736), op: ExtentFlush { repair_id: ReconciliationId(736), extent_id: 101, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:17.983 INFO [0] client ExtentFlush { repair_id: ReconciliationId(736), extent_id: 101, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:17.983 INFO [1] received reconcile message
Sep 22 23:15:17.983 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(736), op: ExtentFlush { repair_id: ReconciliationId(736), extent_id: 101, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:17.983 INFO [1] client ExtentFlush { repair_id: ReconciliationId(736), extent_id: 101, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:17.983 INFO [2] received reconcile message
Sep 22 23:15:17.983 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(736), op: ExtentFlush { repair_id: ReconciliationId(736), extent_id: 101, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:17.984 INFO [2] client ExtentFlush { repair_id: ReconciliationId(736), extent_id: 101, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:17.984 DEBG 736 Flush extent 101 with f:2 g:2
Sep 22 23:15:17.984 DEBG Flush just extent 101 with f:2 and g:2
Sep 22 23:15:17.984 DEBG [1] It's time to notify for 736
Sep 22 23:15:17.984 INFO Completion from [1] id:736 status:true
Sep 22 23:15:17.984 INFO [737/752] Repair commands completed
Sep 22 23:15:17.984 INFO Pop front: ReconcileIO { id: ReconciliationId(737), op: ExtentClose { repair_id: ReconciliationId(737), extent_id: 101 }, state: ClientData([New, New, New]) }
Sep 22 23:15:17.984 INFO Sent repair work, now wait for resp
Sep 22 23:15:17.984 INFO [0] received reconcile message
Sep 22 23:15:17.984 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(737), op: ExtentClose { repair_id: ReconciliationId(737), extent_id: 101 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:17.984 INFO [0] client ExtentClose { repair_id: ReconciliationId(737), extent_id: 101 }
Sep 22 23:15:17.984 INFO [1] received reconcile message
Sep 22 23:15:17.984 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(737), op: ExtentClose { repair_id: ReconciliationId(737), extent_id: 101 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:17.984 INFO [1] client ExtentClose { repair_id: ReconciliationId(737), extent_id: 101 }
Sep 22 23:15:17.984 INFO [2] received reconcile message
Sep 22 23:15:17.984 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(737), op: ExtentClose { repair_id: ReconciliationId(737), extent_id: 101 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:17.984 INFO [2] client ExtentClose { repair_id: ReconciliationId(737), extent_id: 101 }
Sep 22 23:15:17.984 DEBG 737 Close extent 101
Sep 22 23:15:17.984 DEBG 737 Close extent 101
Sep 22 23:15:17.985 DEBG 737 Close extent 101
Sep 22 23:15:17.985 DEBG [2] It's time to notify for 737
Sep 22 23:15:17.985 INFO Completion from [2] id:737 status:true
Sep 22 23:15:17.985 INFO [738/752] Repair commands completed
Sep 22 23:15:17.985 INFO Pop front: ReconcileIO { id: ReconciliationId(738), op: ExtentRepair { repair_id: ReconciliationId(738), extent_id: 101, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:17.985 INFO Sent repair work, now wait for resp
Sep 22 23:15:17.985 INFO [0] received reconcile message
Sep 22 23:15:17.985 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(738), op: ExtentRepair { repair_id: ReconciliationId(738), extent_id: 101, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:17.985 INFO [0] client ExtentRepair { repair_id: ReconciliationId(738), extent_id: 101, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:17.985 INFO [0] Sending repair request ReconciliationId(738)
Sep 22 23:15:17.985 INFO [1] received reconcile message
Sep 22 23:15:17.985 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(738), op: ExtentRepair { repair_id: ReconciliationId(738), extent_id: 101, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:17.985 INFO [1] client ExtentRepair { repair_id: ReconciliationId(738), extent_id: 101, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:17.985 INFO [1] No action required ReconciliationId(738)
Sep 22 23:15:17.986 INFO [2] received reconcile message
Sep 22 23:15:17.986 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(738), op: ExtentRepair { repair_id: ReconciliationId(738), extent_id: 101, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:17.986 INFO [2] client ExtentRepair { repair_id: ReconciliationId(738), extent_id: 101, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:17.986 INFO [2] No action required ReconciliationId(738)
Sep 22 23:15:17.986 DEBG 738 Repair extent 101 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:17.986 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/065.copy"
Sep 22 23:15:18.016 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:18.019 DEBG Flush     :1093 extent_limit None deps:[JobId(1092)] res:true f:36 g:1
Sep 22 23:15:18.019 INFO [lossy] sleeping 1 second
Sep 22 23:15:18.025 DEBG Read      :1092 deps:[JobId(1091)] res:true
Sep 22 23:15:18.049 INFO accepted connection, remote_addr: 127.0.0.1:39584, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:18.050 TRCE incoming request, uri: /extent/101/files, method: GET, req_id: b42ac015-5ece-42e9-9c01-471c121351f3, remote_addr: 127.0.0.1:39584, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:18.050 INFO request completed, latency_us: 236, response_code: 200, uri: /extent/101/files, method: GET, req_id: b42ac015-5ece-42e9-9c01-471c121351f3, remote_addr: 127.0.0.1:39584, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:18.050 INFO eid:101 Found repair files: ["065", "065.db"]
Sep 22 23:15:18.050 TRCE incoming request, uri: /newextent/101/data, method: GET, req_id: fa2a560b-2358-4083-9284-b12aac1957ce, remote_addr: 127.0.0.1:39584, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:18.051 INFO request completed, latency_us: 346, response_code: 200, uri: /newextent/101/data, method: GET, req_id: fa2a560b-2358-4083-9284-b12aac1957ce, remote_addr: 127.0.0.1:39584, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:18.056 TRCE incoming request, uri: /newextent/101/db, method: GET, req_id: fd527bda-8071-4e73-bc9e-67997877c3d5, remote_addr: 127.0.0.1:39584, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:18.056 INFO request completed, latency_us: 299, response_code: 200, uri: /newextent/101/db, method: GET, req_id: fd527bda-8071-4e73-bc9e-67997877c3d5, remote_addr: 127.0.0.1:39584, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:18.057 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/065.copy" to "/tmp/downstairs-vrx8aK6L/00/000/065.replace"
Sep 22 23:15:18.057 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:18.059 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/065.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:18.059 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/065"
Sep 22 23:15:18.059 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/065.db"
Sep 22 23:15:18.059 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:18.059 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/065.replace" to "/tmp/downstairs-vrx8aK6L/00/000/065.completed"
Sep 22 23:15:18.059 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:18.059 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:18.059 DEBG [0] It's time to notify for 738
Sep 22 23:15:18.059 INFO Completion from [0] id:738 status:true
Sep 22 23:15:18.059 INFO [739/752] Repair commands completed
Sep 22 23:15:18.059 INFO Pop front: ReconcileIO { id: ReconciliationId(739), op: ExtentReopen { repair_id: ReconciliationId(739), extent_id: 101 }, state: ClientData([New, New, New]) }
Sep 22 23:15:18.060 INFO Sent repair work, now wait for resp
Sep 22 23:15:18.060 INFO [0] received reconcile message
Sep 22 23:15:18.060 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(739), op: ExtentReopen { repair_id: ReconciliationId(739), extent_id: 101 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:18.060 INFO [0] client ExtentReopen { repair_id: ReconciliationId(739), extent_id: 101 }
Sep 22 23:15:18.060 INFO [1] received reconcile message
Sep 22 23:15:18.060 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(739), op: ExtentReopen { repair_id: ReconciliationId(739), extent_id: 101 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:18.060 INFO [1] client ExtentReopen { repair_id: ReconciliationId(739), extent_id: 101 }
Sep 22 23:15:18.060 INFO [2] received reconcile message
Sep 22 23:15:18.060 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(739), op: ExtentReopen { repair_id: ReconciliationId(739), extent_id: 101 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:18.060 INFO [2] client ExtentReopen { repair_id: ReconciliationId(739), extent_id: 101 }
Sep 22 23:15:18.060 DEBG 739 Reopen extent 101
Sep 22 23:15:18.061 DEBG 739 Reopen extent 101
Sep 22 23:15:18.061 DEBG 739 Reopen extent 101
Sep 22 23:15:18.062 DEBG [2] It's time to notify for 739
Sep 22 23:15:18.062 INFO Completion from [2] id:739 status:true
Sep 22 23:15:18.062 INFO [740/752] Repair commands completed
Sep 22 23:15:18.062 INFO Pop front: ReconcileIO { id: ReconciliationId(740), op: ExtentFlush { repair_id: ReconciliationId(740), extent_id: 74, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:18.062 INFO Sent repair work, now wait for resp
Sep 22 23:15:18.062 INFO [0] received reconcile message
Sep 22 23:15:18.062 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(740), op: ExtentFlush { repair_id: ReconciliationId(740), extent_id: 74, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:18.062 INFO [0] client ExtentFlush { repair_id: ReconciliationId(740), extent_id: 74, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:18.062 INFO [1] received reconcile message
Sep 22 23:15:18.062 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(740), op: ExtentFlush { repair_id: ReconciliationId(740), extent_id: 74, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:18.062 INFO [1] client ExtentFlush { repair_id: ReconciliationId(740), extent_id: 74, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:18.062 INFO [2] received reconcile message
Sep 22 23:15:18.062 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(740), op: ExtentFlush { repair_id: ReconciliationId(740), extent_id: 74, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:18.062 INFO [2] client ExtentFlush { repair_id: ReconciliationId(740), extent_id: 74, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:18.062 DEBG 740 Flush extent 74 with f:2 g:2
Sep 22 23:15:18.062 DEBG Flush just extent 74 with f:2 and g:2
Sep 22 23:15:18.062 DEBG [1] It's time to notify for 740
Sep 22 23:15:18.062 INFO Completion from [1] id:740 status:true
Sep 22 23:15:18.062 INFO [741/752] Repair commands completed
Sep 22 23:15:18.062 INFO Pop front: ReconcileIO { id: ReconciliationId(741), op: ExtentClose { repair_id: ReconciliationId(741), extent_id: 74 }, state: ClientData([New, New, New]) }
Sep 22 23:15:18.062 INFO Sent repair work, now wait for resp
Sep 22 23:15:18.062 INFO [0] received reconcile message
Sep 22 23:15:18.062 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(741), op: ExtentClose { repair_id: ReconciliationId(741), extent_id: 74 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:18.063 INFO [0] client ExtentClose { repair_id: ReconciliationId(741), extent_id: 74 }
Sep 22 23:15:18.063 INFO [1] received reconcile message
Sep 22 23:15:18.063 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(741), op: ExtentClose { repair_id: ReconciliationId(741), extent_id: 74 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:18.063 INFO [1] client ExtentClose { repair_id: ReconciliationId(741), extent_id: 74 }
Sep 22 23:15:18.063 INFO [2] received reconcile message
Sep 22 23:15:18.063 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(741), op: ExtentClose { repair_id: ReconciliationId(741), extent_id: 74 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:18.063 INFO [2] client ExtentClose { repair_id: ReconciliationId(741), extent_id: 74 }
Sep 22 23:15:18.063 DEBG 741 Close extent 74
Sep 22 23:15:18.063 DEBG 741 Close extent 74
Sep 22 23:15:18.063 DEBG 741 Close extent 74
Sep 22 23:15:18.064 DEBG [2] It's time to notify for 741
Sep 22 23:15:18.064 INFO Completion from [2] id:741 status:true
Sep 22 23:15:18.064 INFO [742/752] Repair commands completed
Sep 22 23:15:18.064 INFO Pop front: ReconcileIO { id: ReconciliationId(742), op: ExtentRepair { repair_id: ReconciliationId(742), extent_id: 74, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:18.064 INFO Sent repair work, now wait for resp
Sep 22 23:15:18.064 INFO [0] received reconcile message
Sep 22 23:15:18.064 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(742), op: ExtentRepair { repair_id: ReconciliationId(742), extent_id: 74, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:18.064 INFO [0] client ExtentRepair { repair_id: ReconciliationId(742), extent_id: 74, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:18.064 INFO [0] Sending repair request ReconciliationId(742)
Sep 22 23:15:18.064 INFO [1] received reconcile message
Sep 22 23:15:18.064 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(742), op: ExtentRepair { repair_id: ReconciliationId(742), extent_id: 74, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:18.064 INFO [1] client ExtentRepair { repair_id: ReconciliationId(742), extent_id: 74, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:18.064 INFO [1] No action required ReconciliationId(742)
Sep 22 23:15:18.064 INFO [2] received reconcile message
Sep 22 23:15:18.064 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(742), op: ExtentRepair { repair_id: ReconciliationId(742), extent_id: 74, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:18.064 INFO [2] client ExtentRepair { repair_id: ReconciliationId(742), extent_id: 74, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:18.064 INFO [2] No action required ReconciliationId(742)
Sep 22 23:15:18.064 DEBG 742 Repair extent 74 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:18.064 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/04A.copy"
Sep 22 23:15:18.095 DEBG IO Read  1094 has deps [JobId(1093)]
Sep 22 23:15:18.128 INFO accepted connection, remote_addr: 127.0.0.1:53810, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:18.129 TRCE incoming request, uri: /extent/74/files, method: GET, req_id: 21400e2a-efd3-44d8-9bb3-1729a2814135, remote_addr: 127.0.0.1:53810, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:18.129 INFO request completed, latency_us: 203, response_code: 200, uri: /extent/74/files, method: GET, req_id: 21400e2a-efd3-44d8-9bb3-1729a2814135, remote_addr: 127.0.0.1:53810, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:18.129 INFO eid:74 Found repair files: ["04A", "04A.db"]
Sep 22 23:15:18.129 TRCE incoming request, uri: /newextent/74/data, method: GET, req_id: 56b803b4-cffb-4623-9b1c-0d1585917eb0, remote_addr: 127.0.0.1:53810, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:18.130 INFO request completed, latency_us: 255, response_code: 200, uri: /newextent/74/data, method: GET, req_id: 56b803b4-cffb-4623-9b1c-0d1585917eb0, remote_addr: 127.0.0.1:53810, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:18.135 TRCE incoming request, uri: /newextent/74/db, method: GET, req_id: cd138ed0-99c5-4e8f-b2b2-561e7538d650, remote_addr: 127.0.0.1:53810, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:18.135 INFO request completed, latency_us: 296, response_code: 200, uri: /newextent/74/db, method: GET, req_id: cd138ed0-99c5-4e8f-b2b2-561e7538d650, remote_addr: 127.0.0.1:53810, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:18.136 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/04A.copy" to "/tmp/downstairs-vrx8aK6L/00/000/04A.replace"
Sep 22 23:15:18.136 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:18.137 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/04A.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:18.138 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/04A"
Sep 22 23:15:18.138 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/04A.db"
Sep 22 23:15:18.138 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:18.138 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/04A.replace" to "/tmp/downstairs-vrx8aK6L/00/000/04A.completed"
Sep 22 23:15:18.138 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:18.138 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:18.138 DEBG [0] It's time to notify for 742
Sep 22 23:15:18.138 INFO Completion from [0] id:742 status:true
Sep 22 23:15:18.138 INFO [743/752] Repair commands completed
Sep 22 23:15:18.138 INFO Pop front: ReconcileIO { id: ReconciliationId(743), op: ExtentReopen { repair_id: ReconciliationId(743), extent_id: 74 }, state: ClientData([New, New, New]) }
Sep 22 23:15:18.138 INFO Sent repair work, now wait for resp
Sep 22 23:15:18.138 INFO [0] received reconcile message
Sep 22 23:15:18.138 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(743), op: ExtentReopen { repair_id: ReconciliationId(743), extent_id: 74 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:18.138 INFO [0] client ExtentReopen { repair_id: ReconciliationId(743), extent_id: 74 }
Sep 22 23:15:18.138 INFO [1] received reconcile message
Sep 22 23:15:18.138 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(743), op: ExtentReopen { repair_id: ReconciliationId(743), extent_id: 74 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:18.138 INFO [1] client ExtentReopen { repair_id: ReconciliationId(743), extent_id: 74 }
Sep 22 23:15:18.138 INFO [2] received reconcile message
Sep 22 23:15:18.138 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(743), op: ExtentReopen { repair_id: ReconciliationId(743), extent_id: 74 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:18.138 INFO [2] client ExtentReopen { repair_id: ReconciliationId(743), extent_id: 74 }
Sep 22 23:15:18.139 DEBG 743 Reopen extent 74
Sep 22 23:15:18.139 DEBG 743 Reopen extent 74
Sep 22 23:15:18.140 DEBG 743 Reopen extent 74
Sep 22 23:15:18.140 DEBG [2] It's time to notify for 743
Sep 22 23:15:18.140 INFO Completion from [2] id:743 status:true
Sep 22 23:15:18.140 INFO [744/752] Repair commands completed
Sep 22 23:15:18.140 INFO Pop front: ReconcileIO { id: ReconciliationId(744), op: ExtentFlush { repair_id: ReconciliationId(744), extent_id: 83, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:18.140 INFO Sent repair work, now wait for resp
Sep 22 23:15:18.140 INFO [0] received reconcile message
Sep 22 23:15:18.140 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(744), op: ExtentFlush { repair_id: ReconciliationId(744), extent_id: 83, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:18.140 INFO [0] client ExtentFlush { repair_id: ReconciliationId(744), extent_id: 83, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:18.140 INFO [1] received reconcile message
Sep 22 23:15:18.140 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(744), op: ExtentFlush { repair_id: ReconciliationId(744), extent_id: 83, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:18.141 INFO [1] client ExtentFlush { repair_id: ReconciliationId(744), extent_id: 83, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:18.141 INFO [2] received reconcile message
Sep 22 23:15:18.141 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(744), op: ExtentFlush { repair_id: ReconciliationId(744), extent_id: 83, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:18.141 INFO [2] client ExtentFlush { repair_id: ReconciliationId(744), extent_id: 83, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:18.141 DEBG 744 Flush extent 83 with f:2 g:2
Sep 22 23:15:18.141 DEBG Flush just extent 83 with f:2 and g:2
Sep 22 23:15:18.141 DEBG [1] It's time to notify for 744
Sep 22 23:15:18.141 INFO Completion from [1] id:744 status:true
Sep 22 23:15:18.141 INFO [745/752] Repair commands completed
Sep 22 23:15:18.141 INFO Pop front: ReconcileIO { id: ReconciliationId(745), op: ExtentClose { repair_id: ReconciliationId(745), extent_id: 83 }, state: ClientData([New, New, New]) }
Sep 22 23:15:18.141 INFO Sent repair work, now wait for resp
Sep 22 23:15:18.141 INFO [0] received reconcile message
Sep 22 23:15:18.141 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(745), op: ExtentClose { repair_id: ReconciliationId(745), extent_id: 83 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:18.141 INFO [0] client ExtentClose { repair_id: ReconciliationId(745), extent_id: 83 }
Sep 22 23:15:18.141 INFO [1] received reconcile message
Sep 22 23:15:18.141 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(745), op: ExtentClose { repair_id: ReconciliationId(745), extent_id: 83 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:18.141 INFO [1] client ExtentClose { repair_id: ReconciliationId(745), extent_id: 83 }
Sep 22 23:15:18.141 INFO [2] received reconcile message
Sep 22 23:15:18.141 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(745), op: ExtentClose { repair_id: ReconciliationId(745), extent_id: 83 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:18.141 INFO [2] client ExtentClose { repair_id: ReconciliationId(745), extent_id: 83 }
Sep 22 23:15:18.141 DEBG 745 Close extent 83
Sep 22 23:15:18.142 DEBG 745 Close extent 83
Sep 22 23:15:18.142 DEBG 745 Close extent 83
Sep 22 23:15:18.142 DEBG [2] It's time to notify for 745
Sep 22 23:15:18.142 INFO Completion from [2] id:745 status:true
Sep 22 23:15:18.142 INFO [746/752] Repair commands completed
Sep 22 23:15:18.142 INFO Pop front: ReconcileIO { id: ReconciliationId(746), op: ExtentRepair { repair_id: ReconciliationId(746), extent_id: 83, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:18.142 INFO Sent repair work, now wait for resp
Sep 22 23:15:18.142 INFO [0] received reconcile message
Sep 22 23:15:18.142 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(746), op: ExtentRepair { repair_id: ReconciliationId(746), extent_id: 83, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:18.142 INFO [0] client ExtentRepair { repair_id: ReconciliationId(746), extent_id: 83, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:18.142 INFO [0] Sending repair request ReconciliationId(746)
Sep 22 23:15:18.143 INFO [1] received reconcile message
Sep 22 23:15:18.143 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(746), op: ExtentRepair { repair_id: ReconciliationId(746), extent_id: 83, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:18.143 INFO [1] client ExtentRepair { repair_id: ReconciliationId(746), extent_id: 83, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:18.143 INFO [1] No action required ReconciliationId(746)
Sep 22 23:15:18.143 INFO [2] received reconcile message
Sep 22 23:15:18.143 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(746), op: ExtentRepair { repair_id: ReconciliationId(746), extent_id: 83, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:18.143 INFO [2] client ExtentRepair { repair_id: ReconciliationId(746), extent_id: 83, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:18.143 INFO [2] No action required ReconciliationId(746)
Sep 22 23:15:18.143 DEBG 746 Repair extent 83 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:18.143 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/053.copy"
Sep 22 23:15:18.204 INFO accepted connection, remote_addr: 127.0.0.1:41622, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:18.204 TRCE incoming request, uri: /extent/83/files, method: GET, req_id: a7b795f1-964c-4238-b9d0-33b73a84f4ae, remote_addr: 127.0.0.1:41622, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:18.204 INFO request completed, latency_us: 198, response_code: 200, uri: /extent/83/files, method: GET, req_id: a7b795f1-964c-4238-b9d0-33b73a84f4ae, remote_addr: 127.0.0.1:41622, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:18.204 INFO eid:83 Found repair files: ["053", "053.db"]
Sep 22 23:15:18.204 TRCE incoming request, uri: /newextent/83/data, method: GET, req_id: 251d9f17-bc9b-4f2a-87db-66c05c6787e5, remote_addr: 127.0.0.1:41622, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:18.205 INFO request completed, latency_us: 251, response_code: 200, uri: /newextent/83/data, method: GET, req_id: 251d9f17-bc9b-4f2a-87db-66c05c6787e5, remote_addr: 127.0.0.1:41622, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:18.210 TRCE incoming request, uri: /newextent/83/db, method: GET, req_id: 797bb5db-ea30-4d14-a905-45f53c265bc8, remote_addr: 127.0.0.1:41622, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:18.210 INFO request completed, latency_us: 293, response_code: 200, uri: /newextent/83/db, method: GET, req_id: 797bb5db-ea30-4d14-a905-45f53c265bc8, remote_addr: 127.0.0.1:41622, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:18.211 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/053.copy" to "/tmp/downstairs-vrx8aK6L/00/000/053.replace"
Sep 22 23:15:18.211 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:18.212 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/053.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:18.212 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/053"
Sep 22 23:15:18.212 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/053.db"
Sep 22 23:15:18.212 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:18.212 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/053.replace" to "/tmp/downstairs-vrx8aK6L/00/000/053.completed"
Sep 22 23:15:18.212 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:18.212 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:18.213 DEBG [0] It's time to notify for 746
Sep 22 23:15:18.213 INFO Completion from [0] id:746 status:true
Sep 22 23:15:18.213 INFO [747/752] Repair commands completed
Sep 22 23:15:18.213 INFO Pop front: ReconcileIO { id: ReconciliationId(747), op: ExtentReopen { repair_id: ReconciliationId(747), extent_id: 83 }, state: ClientData([New, New, New]) }
Sep 22 23:15:18.213 INFO Sent repair work, now wait for resp
Sep 22 23:15:18.213 INFO [0] received reconcile message
Sep 22 23:15:18.213 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(747), op: ExtentReopen { repair_id: ReconciliationId(747), extent_id: 83 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:18.213 INFO [0] client ExtentReopen { repair_id: ReconciliationId(747), extent_id: 83 }
Sep 22 23:15:18.213 INFO [1] received reconcile message
Sep 22 23:15:18.213 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(747), op: ExtentReopen { repair_id: ReconciliationId(747), extent_id: 83 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:18.213 INFO [1] client ExtentReopen { repair_id: ReconciliationId(747), extent_id: 83 }
Sep 22 23:15:18.213 INFO [2] received reconcile message
Sep 22 23:15:18.213 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(747), op: ExtentReopen { repair_id: ReconciliationId(747), extent_id: 83 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:18.213 INFO [2] client ExtentReopen { repair_id: ReconciliationId(747), extent_id: 83 }
Sep 22 23:15:18.213 DEBG 747 Reopen extent 83
Sep 22 23:15:18.214 DEBG 747 Reopen extent 83
Sep 22 23:15:18.214 DEBG 747 Reopen extent 83
Sep 22 23:15:18.215 DEBG [2] It's time to notify for 747
Sep 22 23:15:18.215 INFO Completion from [2] id:747 status:true
Sep 22 23:15:18.215 INFO [748/752] Repair commands completed
Sep 22 23:15:18.215 INFO Pop front: ReconcileIO { id: ReconciliationId(748), op: ExtentFlush { repair_id: ReconciliationId(748), extent_id: 119, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:15:18.215 INFO Sent repair work, now wait for resp
Sep 22 23:15:18.215 INFO [0] received reconcile message
Sep 22 23:15:18.215 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(748), op: ExtentFlush { repair_id: ReconciliationId(748), extent_id: 119, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:18.215 INFO [0] client ExtentFlush { repair_id: ReconciliationId(748), extent_id: 119, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:18.215 INFO [1] received reconcile message
Sep 22 23:15:18.215 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(748), op: ExtentFlush { repair_id: ReconciliationId(748), extent_id: 119, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:15:18.215 INFO [1] client ExtentFlush { repair_id: ReconciliationId(748), extent_id: 119, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:18.215 INFO [2] received reconcile message
Sep 22 23:15:18.215 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(748), op: ExtentFlush { repair_id: ReconciliationId(748), extent_id: 119, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:18.215 INFO [2] client ExtentFlush { repair_id: ReconciliationId(748), extent_id: 119, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:15:18.215 DEBG 748 Flush extent 119 with f:2 g:2
Sep 22 23:15:18.215 DEBG Flush just extent 119 with f:2 and g:2
Sep 22 23:15:18.215 DEBG [1] It's time to notify for 748
Sep 22 23:15:18.215 INFO Completion from [1] id:748 status:true
Sep 22 23:15:18.215 INFO [749/752] Repair commands completed
Sep 22 23:15:18.216 INFO Pop front: ReconcileIO { id: ReconciliationId(749), op: ExtentClose { repair_id: ReconciliationId(749), extent_id: 119 }, state: ClientData([New, New, New]) }
Sep 22 23:15:18.216 INFO Sent repair work, now wait for resp
Sep 22 23:15:18.216 INFO [0] received reconcile message
Sep 22 23:15:18.216 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(749), op: ExtentClose { repair_id: ReconciliationId(749), extent_id: 119 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:18.216 INFO [0] client ExtentClose { repair_id: ReconciliationId(749), extent_id: 119 }
Sep 22 23:15:18.216 INFO [1] received reconcile message
Sep 22 23:15:18.216 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(749), op: ExtentClose { repair_id: ReconciliationId(749), extent_id: 119 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:18.216 INFO [1] client ExtentClose { repair_id: ReconciliationId(749), extent_id: 119 }
Sep 22 23:15:18.216 INFO [2] received reconcile message
Sep 22 23:15:18.216 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(749), op: ExtentClose { repair_id: ReconciliationId(749), extent_id: 119 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:18.216 INFO [2] client ExtentClose { repair_id: ReconciliationId(749), extent_id: 119 }
Sep 22 23:15:18.216 DEBG 749 Close extent 119
Sep 22 23:15:18.216 DEBG 749 Close extent 119
Sep 22 23:15:18.216 DEBG 749 Close extent 119
Sep 22 23:15:18.217 DEBG [2] It's time to notify for 749
Sep 22 23:15:18.217 INFO Completion from [2] id:749 status:true
Sep 22 23:15:18.217 INFO [750/752] Repair commands completed
Sep 22 23:15:18.217 INFO Pop front: ReconcileIO { id: ReconciliationId(750), op: ExtentRepair { repair_id: ReconciliationId(750), extent_id: 119, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:15:18.217 INFO Sent repair work, now wait for resp
Sep 22 23:15:18.217 INFO [0] received reconcile message
Sep 22 23:15:18.217 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(750), op: ExtentRepair { repair_id: ReconciliationId(750), extent_id: 119, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:18.217 INFO [0] client ExtentRepair { repair_id: ReconciliationId(750), extent_id: 119, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:18.217 INFO [0] Sending repair request ReconciliationId(750)
Sep 22 23:15:18.217 INFO [1] received reconcile message
Sep 22 23:15:18.217 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(750), op: ExtentRepair { repair_id: ReconciliationId(750), extent_id: 119, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:18.217 INFO [1] client ExtentRepair { repair_id: ReconciliationId(750), extent_id: 119, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:18.217 INFO [1] No action required ReconciliationId(750)
Sep 22 23:15:18.217 INFO [2] received reconcile message
Sep 22 23:15:18.217 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(750), op: ExtentRepair { repair_id: ReconciliationId(750), extent_id: 119, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:15:18.217 INFO [2] client ExtentRepair { repair_id: ReconciliationId(750), extent_id: 119, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:46213, dest_clients: [ClientId(0)] }
Sep 22 23:15:18.217 INFO [2] No action required ReconciliationId(750)
Sep 22 23:15:18.217 DEBG 750 Repair extent 119 source:[1] 127.0.0.1:46213 dest:[ClientId(0)]
Sep 22 23:15:18.217 INFO Created copy dir "/tmp/downstairs-vrx8aK6L/00/000/077.copy"
Sep 22 23:15:18.280 INFO accepted connection, remote_addr: 127.0.0.1:44274, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:18.280 TRCE incoming request, uri: /extent/119/files, method: GET, req_id: 62beac8e-278f-4ac3-b123-b86e144a2a78, remote_addr: 127.0.0.1:44274, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:18.281 INFO request completed, latency_us: 190, response_code: 200, uri: /extent/119/files, method: GET, req_id: 62beac8e-278f-4ac3-b123-b86e144a2a78, remote_addr: 127.0.0.1:44274, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:18.281 INFO eid:119 Found repair files: ["077", "077.db"]
Sep 22 23:15:18.281 TRCE incoming request, uri: /newextent/119/data, method: GET, req_id: 52a95044-6a92-4278-b72e-80ab157b9d18, remote_addr: 127.0.0.1:44274, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:18.281 INFO request completed, latency_us: 299, response_code: 200, uri: /newextent/119/data, method: GET, req_id: 52a95044-6a92-4278-b72e-80ab157b9d18, remote_addr: 127.0.0.1:44274, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:18.286 TRCE incoming request, uri: /newextent/119/db, method: GET, req_id: 4bae20a0-7f94-442c-be90-91fe1ddbc59b, remote_addr: 127.0.0.1:44274, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:18.287 INFO request completed, latency_us: 291, response_code: 200, uri: /newextent/119/db, method: GET, req_id: 4bae20a0-7f94-442c-be90-91fe1ddbc59b, remote_addr: 127.0.0.1:44274, local_addr: 127.0.0.1:46213, task: repair
Sep 22 23:15:18.288 INFO Repair files downloaded, move directory "/tmp/downstairs-vrx8aK6L/00/000/077.copy" to "/tmp/downstairs-vrx8aK6L/00/000/077.replace"
Sep 22 23:15:18.288 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:18.289 INFO Copy files from "/tmp/downstairs-vrx8aK6L/00/000/077.replace" in "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:18.289 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/077"
Sep 22 23:15:18.289 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000/077.db"
Sep 22 23:15:18.289 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:18.289 INFO Move directory  "/tmp/downstairs-vrx8aK6L/00/000/077.replace" to "/tmp/downstairs-vrx8aK6L/00/000/077.completed"
Sep 22 23:15:18.289 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:18.289 DEBG fsync completed for: "/tmp/downstairs-vrx8aK6L/00/000"
Sep 22 23:15:18.289 DEBG [0] It's time to notify for 750
Sep 22 23:15:18.290 INFO Completion from [0] id:750 status:true
Sep 22 23:15:18.290 INFO [751/752] Repair commands completed
Sep 22 23:15:18.290 INFO Pop front: ReconcileIO { id: ReconciliationId(751), op: ExtentReopen { repair_id: ReconciliationId(751), extent_id: 119 }, state: ClientData([New, New, New]) }
Sep 22 23:15:18.290 INFO Sent repair work, now wait for resp
Sep 22 23:15:18.290 INFO [0] received reconcile message
Sep 22 23:15:18.290 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(751), op: ExtentReopen { repair_id: ReconciliationId(751), extent_id: 119 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:15:18.290 INFO [0] client ExtentReopen { repair_id: ReconciliationId(751), extent_id: 119 }
Sep 22 23:15:18.290 INFO [1] received reconcile message
Sep 22 23:15:18.290 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(751), op: ExtentReopen { repair_id: ReconciliationId(751), extent_id: 119 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:15:18.290 INFO [1] client ExtentReopen { repair_id: ReconciliationId(751), extent_id: 119 }
Sep 22 23:15:18.290 INFO [2] received reconcile message
Sep 22 23:15:18.290 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(751), op: ExtentReopen { repair_id: ReconciliationId(751), extent_id: 119 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:15:18.290 INFO [2] client ExtentReopen { repair_id: ReconciliationId(751), extent_id: 119 }
Sep 22 23:15:18.290 DEBG 751 Reopen extent 119
Sep 22 23:15:18.291 DEBG 751 Reopen extent 119
Sep 22 23:15:18.291 DEBG 751 Reopen extent 119
Sep 22 23:15:18.292 DEBG [2] It's time to notify for 751
Sep 22 23:15:18.292 INFO Completion from [2] id:751 status:true
Sep 22 23:15:18.292 INFO [752/752] Repair commands completed
Sep 22 23:15:18.292 INFO 188 extents repaired in 14.627 ave:0.0778
Sep 22 23:15:18.292 INFO All required repair work is completed
Sep 22 23:15:18.292 INFO Set Downstairs and Upstairs active after repairs
Sep 22 23:15:18.292 INFO 6e5c0f70-fd56-4280-9d20-71288e488216 is now active with session: 2cf90053-4dbd-4cc4-8468-9d35f085e47a
Sep 22 23:15:18.292 INFO Notify all downstairs, region set compare is done.
Sep 22 23:15:18.292 INFO Set check for repair
Sep 22 23:15:18.292 INFO [1] 127.0.0.1:64149 task reports connection:true
Sep 22 23:15:18.292 INFO 6e5c0f70-fd56-4280-9d20-71288e488216 Active Active Active
Sep 22 23:15:18.292 INFO Set check for repair
Sep 22 23:15:18.292 INFO [2] 127.0.0.1:58182 task reports connection:true
Sep 22 23:15:18.292 INFO 6e5c0f70-fd56-4280-9d20-71288e488216 Active Active Active
Sep 22 23:15:18.292 INFO Set check for repair
Sep 22 23:15:18.292 INFO [0] received reconcile message
Sep 22 23:15:18.292 INFO [0] All repairs completed, exit
Sep 22 23:15:18.292 INFO [0] Starts cmd_loop
Sep 22 23:15:18.292 INFO [1] received reconcile message
Sep 22 23:15:18.292 INFO [1] All repairs completed, exit
Sep 22 23:15:18.292 INFO [1] Starts cmd_loop
Sep 22 23:15:18.292 INFO [2] received reconcile message
Sep 22 23:15:18.292 INFO [2] All repairs completed, exit
Sep 22 23:15:18.292 INFO [2] Starts cmd_loop
The guest has finished waiting for activation
Sep 22 23:15:18.311 DEBG IO Read  1000 has deps []
Sep 22 23:15:18.433 DEBG up_ds_listen was notified
Sep 22 23:15:18.433 DEBG up_ds_listen process 1093
Sep 22 23:15:18.433 DEBG [A] ack job 1093:94, : downstairs
Sep 22 23:15:18.433 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:18.440 DEBG IO Flush 1095 has deps [JobId(1094), JobId(1093)]
Sep 22 23:15:18.443 DEBG Flush     :1093 extent_limit None deps:[JobId(1092)] res:true f:36 g:1
Sep 22 23:15:18.449 DEBG Read      :1094 deps:[JobId(1093)] res:true
Sep 22 23:15:18.850 DEBG [rc] retire 1093 clears [JobId(1092), JobId(1093)], : downstairs
Sep 22 23:15:18.857 DEBG Read      :1094 deps:[JobId(1093)] res:true
Sep 22 23:15:18.881 DEBG Flush     :1095 extent_limit None deps:[JobId(1094), JobId(1093)] res:true f:37 g:1
Sep 22 23:15:18.881 INFO [lossy] sleeping 1 second
Sep 22 23:15:19.263 DEBG [2] Read AckReady 1094, : downstairs
Sep 22 23:15:19.264 DEBG up_ds_listen was notified
Sep 22 23:15:19.264 DEBG up_ds_listen process 1094
Sep 22 23:15:19.264 DEBG [A] ack job 1094:95, : downstairs
Sep 22 23:15:19.318 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:19.320 WARN returning error on flush!
Sep 22 23:15:19.320 DEBG Flush     :1095 extent_limit None deps:[JobId(1094), JobId(1093)] res:false f:37 g:1
Sep 22 23:15:19.320 INFO [lossy] skipping 1095
Sep 22 23:15:19.320 DEBG Flush     :1095 extent_limit None deps:[JobId(1094), JobId(1093)] res:true f:37 g:1
Sep 22 23:15:19.320 INFO [lossy] sleeping 1 second
Sep 22 23:15:19.340 DEBG Read      :1094 deps:[JobId(1093)] res:true
Sep 22 23:15:19.507 DEBG IO Read  1096 has deps [JobId(1095)]
Sep 22 23:15:19.877 ERRO [1] job id 1095 saw error GenericError("test error")
Sep 22 23:15:19.882 DEBG up_ds_listen was notified
Sep 22 23:15:19.882 DEBG up_ds_listen process 1095
Sep 22 23:15:19.882 DEBG [A] ack job 1095:96, : downstairs
Sep 22 23:15:19.882 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:19.889 DEBG IO Flush 1097 has deps [JobId(1096), JobId(1095)]
Sep 22 23:15:19.889 INFO [lossy] sleeping 1 second
Sep 22 23:15:19.892 INFO [lossy] skipping 1095
Sep 22 23:15:19.892 DEBG Flush     :1095 extent_limit None deps:[JobId(1094), JobId(1093)] res:true f:37 g:1
Sep 22 23:15:19.910 DEBG Read      :1096 deps:[JobId(1095)] res:true
Sep 22 23:15:20.450 DEBG [rc] retire 1095 clears [JobId(1094), JobId(1095)], : downstairs
Sep 22 23:15:20.451 WARN returning error on read!
Sep 22 23:15:20.451 DEBG Read      :1096 deps:[JobId(1095)] res:false
Sep 22 23:15:20.457 DEBG Read      :1096 deps:[JobId(1095)] res:true
Sep 22 23:15:20.479 ERRO [1] job id 1096 saw error GenericError("test error")
Sep 22 23:15:20.481 INFO [lossy] skipping 1097
Sep 22 23:15:20.481 INFO [lossy] skipping 1097
Sep 22 23:15:20.481 INFO [lossy] skipping 1097
Sep 22 23:15:20.481 INFO [lossy] skipping 1097
Sep 22 23:15:20.481 DEBG Flush     :1097 extent_limit None deps:[JobId(1096), JobId(1095)] res:true f:38 g:1
Sep 22 23:15:20.481 INFO [lossy] sleeping 1 second
Sep 22 23:15:21.054 DEBG Read      :1000 deps:[] res:true
Sep 22 23:15:21.055 DEBG [0] Read AckReady 1096, : downstairs
Sep 22 23:15:21.056 DEBG up_ds_listen was notified
Sep 22 23:15:21.056 DEBG up_ds_listen process 1096
Sep 22 23:15:21.056 DEBG [A] ack job 1096:97, : downstairs
Sep 22 23:15:21.109 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:21.111 DEBG Flush     :1097 extent_limit None deps:[JobId(1096), JobId(1095)] res:true f:38 g:1
Sep 22 23:15:21.111 INFO [lossy] sleeping 1 second
Sep 22 23:15:21.118 DEBG Read      :1096 deps:[JobId(1095)] res:true
Sep 22 23:15:21.188 DEBG IO Read  1098 has deps [JobId(1097)]
Sep 22 23:15:21.525 DEBG up_ds_listen was notified
Sep 22 23:15:21.525 DEBG up_ds_listen process 1097
Sep 22 23:15:21.525 DEBG [A] ack job 1097:98, : downstairs
Sep 22 23:15:21.525 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:21.532 INFO [lossy] sleeping 1 second
Sep 22 23:15:21.534 WARN returning error on flush!
Sep 22 23:15:21.534 DEBG Flush     :1097 extent_limit None deps:[JobId(1096), JobId(1095)] res:false f:38 g:1
Sep 22 23:15:21.534 WARN returning error on flush!
Sep 22 23:15:21.534 DEBG Flush     :1097 extent_limit None deps:[JobId(1096), JobId(1095)] res:false f:38 g:1
Sep 22 23:15:21.534 INFO [lossy] skipping 1097
Sep 22 23:15:21.534 INFO [lossy] skipping 1097
Sep 22 23:15:21.534 INFO [lossy] skipping 1097
Sep 22 23:15:21.534 WARN returning error on flush!
Sep 22 23:15:21.534 DEBG Flush     :1097 extent_limit None deps:[JobId(1096), JobId(1095)] res:false f:38 g:1
Sep 22 23:15:21.534 WARN returning error on flush!
Sep 22 23:15:21.534 DEBG Flush     :1097 extent_limit None deps:[JobId(1096), JobId(1095)] res:false f:38 g:1
Sep 22 23:15:21.534 WARN returning error on flush!
Sep 22 23:15:21.534 DEBG Flush     :1097 extent_limit None deps:[JobId(1096), JobId(1095)] res:false f:38 g:1
Sep 22 23:15:21.534 DEBG Flush     :1097 extent_limit None deps:[JobId(1096), JobId(1095)] res:true f:38 g:1
Sep 22 23:15:21.535 WARN returning error on read!
Sep 22 23:15:21.535 DEBG Read      :1098 deps:[JobId(1097)] res:false
Sep 22 23:15:21.541 DEBG Read      :1098 deps:[JobId(1097)] res:true
Sep 22 23:15:21.943 ERRO [2] job id 1097 saw error GenericError("test error")
Sep 22 23:15:21.943 ERRO [2] job id 1097 saw error GenericError("test error")
Sep 22 23:15:21.943 ERRO [2] job id 1097 saw error GenericError("test error")
Sep 22 23:15:21.943 ERRO [2] job id 1097 saw error GenericError("test error")
Sep 22 23:15:21.943 ERRO [2] job id 1097 saw error GenericError("test error")
Sep 22 23:15:21.943 DEBG [rc] retire 1097 clears [JobId(1096), JobId(1097)], : downstairs
Sep 22 23:15:21.943 ERRO [2] job id 1098 saw error GenericError("test error")
Sep 22 23:15:21.943 DEBG IO Flush 1099 has deps [JobId(1098)]
Sep 22 23:15:21.946 WARN returning error on flush!
Sep 22 23:15:21.946 DEBG Flush     :1099 extent_limit None deps:[JobId(1098)] res:false f:39 g:1
Sep 22 23:15:21.946 DEBG Flush     :1099 extent_limit None deps:[JobId(1098)] res:true f:39 g:1
Sep 22 23:15:21.946 INFO [lossy] sleeping 1 second
Sep 22 23:15:22.322 DEBG [2] Read AckReady 1098, : downstairs
Sep 22 23:15:22.323 ERRO [2] job id 1099 saw error GenericError("test error")
Sep 22 23:15:22.323 DEBG up_ds_listen was notified
Sep 22 23:15:22.324 DEBG up_ds_listen process 1098
Sep 22 23:15:22.324 DEBG [A] ack job 1098:99, : downstairs
Sep 22 23:15:22.376 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:22.423 DEBG Read      :1098 deps:[JobId(1097)] res:true
Sep 22 23:15:22.445 DEBG IO Read  1100 has deps [JobId(1099)]
Sep 22 23:15:22.445 DEBG IO Flush 1101 has deps [JobId(1100), JobId(1099)]
Sep 22 23:15:22.459 DEBG Flush     :1099 extent_limit None deps:[JobId(1098)] res:true f:39 g:1
Sep 22 23:15:22.459 INFO [lossy] skipping 1100
Sep 22 23:15:22.459 INFO [lossy] skipping 1100
Sep 22 23:15:22.480 DEBG Read      :1100 deps:[JobId(1099)] res:true
Sep 22 23:15:22.639 DEBG Read      :1000 deps:[] res:true
Sep 22 23:15:23.017 DEBG up_ds_listen was notified
Sep 22 23:15:23.017 DEBG up_ds_listen process 1099
Sep 22 23:15:23.018 DEBG [A] ack job 1099:100, : downstairs
Sep 22 23:15:23.018 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:23.024 DEBG Read      :1098 deps:[JobId(1097)] res:true
Sep 22 23:15:23.045 INFO [lossy] skipping 1100
Sep 22 23:15:23.045 INFO [lossy] skipping 1101
Sep 22 23:15:23.051 DEBG Read      :1100 deps:[JobId(1099)] res:true
Sep 22 23:15:23.075 INFO [lossy] skipping 1101
Sep 22 23:15:23.075 WARN returning error on flush!
Sep 22 23:15:23.075 DEBG Flush     :1101 extent_limit None deps:[JobId(1100), JobId(1099)] res:false f:40 g:1
Sep 22 23:15:23.075 DEBG Flush     :1101 extent_limit None deps:[JobId(1100), JobId(1099)] res:true f:40 g:1
Sep 22 23:15:23.075 INFO [lossy] sleeping 1 second
Sep 22 23:15:23.480 DEBG [1] Read AckReady 1100, : downstairs
Sep 22 23:15:23.481 ERRO [1] job id 1101 saw error GenericError("test error")
Sep 22 23:15:23.481 DEBG up_ds_listen was notified
Sep 22 23:15:23.481 DEBG up_ds_listen process 1100
Sep 22 23:15:23.481 DEBG [A] ack job 1100:101, : downstairs
Sep 22 23:15:23.534 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:23.536 WARN returning error on flush!
Sep 22 23:15:23.536 DEBG Flush     :1101 extent_limit None deps:[JobId(1100), JobId(1099)] res:false f:40 g:1
Sep 22 23:15:23.536 INFO [lossy] skipping 1101
Sep 22 23:15:23.536 INFO [lossy] skipping 1101
Sep 22 23:15:23.536 INFO [lossy] skipping 1101
Sep 22 23:15:23.536 WARN returning error on flush!
Sep 22 23:15:23.536 DEBG Flush     :1101 extent_limit None deps:[JobId(1100), JobId(1099)] res:false f:40 g:1
Sep 22 23:15:23.536 DEBG Flush     :1101 extent_limit None deps:[JobId(1100), JobId(1099)] res:true f:40 g:1
Sep 22 23:15:23.536 INFO [lossy] sleeping 1 second
Sep 22 23:15:23.538 DEBG Flush     :1099 extent_limit None deps:[JobId(1098)] res:true f:39 g:1
Sep 22 23:15:23.538 INFO [lossy] skipping 1100
Sep 22 23:15:23.544 DEBG Read      :1100 deps:[JobId(1099)] res:true
Sep 22 23:15:23.685 DEBG IO Read  1102 has deps [JobId(1101)]
Sep 22 23:15:23.685 DEBG IO Flush 1103 has deps [JobId(1102), JobId(1101)]
Sep 22 23:15:24.086 DEBG [rc] retire 1099 clears [JobId(1098), JobId(1099)], : downstairs
Sep 22 23:15:24.088 DEBG Read      :1000 deps:[] res:true
Sep 22 23:15:24.416 ERRO [2] job id 1101 saw error GenericError("test error")
Sep 22 23:15:24.416 ERRO [2] job id 1101 saw error GenericError("test error")
Sep 22 23:15:24.422 DEBG up_ds_listen was notified
Sep 22 23:15:24.422 DEBG up_ds_listen process 1101
Sep 22 23:15:24.422 DEBG [A] ack job 1101:102, : downstairs
Sep 22 23:15:24.422 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:24.429 INFO [lossy] sleeping 1 second
Sep 22 23:15:24.431 DEBG Flush     :1101 extent_limit None deps:[JobId(1100), JobId(1099)] res:true f:40 g:1
Sep 22 23:15:24.437 DEBG Read      :1102 deps:[JobId(1101)] res:true
Sep 22 23:15:24.838 DEBG [rc] retire 1101 clears [JobId(1100), JobId(1101)], : downstairs
Sep 22 23:15:24.838 INFO [lossy] skipping 1102
Sep 22 23:15:24.838 INFO [lossy] skipping 1102
Sep 22 23:15:24.838 WARN returning error on read!
Sep 22 23:15:24.838 DEBG Read      :1102 deps:[JobId(1101)] res:false
Sep 22 23:15:24.844 DEBG Read      :1102 deps:[JobId(1101)] res:true
Sep 22 23:15:24.866 ERRO [2] job id 1102 saw error GenericError("test error")
Sep 22 23:15:24.868 WARN returning error on flush!
Sep 22 23:15:24.868 DEBG Flush     :1103 extent_limit None deps:[JobId(1102), JobId(1101)] res:false f:41 g:1
Sep 22 23:15:24.868 WARN returning error on flush!
Sep 22 23:15:24.868 DEBG Flush     :1103 extent_limit None deps:[JobId(1102), JobId(1101)] res:false f:41 g:1
Sep 22 23:15:24.868 DEBG Flush     :1103 extent_limit None deps:[JobId(1102), JobId(1101)] res:true f:41 g:1
Sep 22 23:15:24.868 INFO [lossy] sleeping 1 second
Sep 22 23:15:24.869 WARN returning error on flush!
Sep 22 23:15:24.869 DEBG Flush     :1103 extent_limit None deps:[JobId(1102), JobId(1101)] res:false f:41 g:1
Sep 22 23:15:24.869 DEBG Flush     :1103 extent_limit None deps:[JobId(1102), JobId(1101)] res:true f:41 g:1
Sep 22 23:15:24.869 INFO [lossy] sleeping 1 second
Sep 22 23:15:25.294 DEBG [0] Read AckReady 1102, : downstairs
Sep 22 23:15:25.294 ERRO [0] job id 1103 saw error GenericError("test error")
Sep 22 23:15:25.343 DEBG IO Flush 1001 has deps [JobId(1000)]
Sep 22 23:15:25.343 INFO Checking if live repair is needed
Sep 22 23:15:25.343 INFO No Live Repair required at this time
Sep 22 23:15:25.517 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:2 g:2
Sep 22 23:15:25.579 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:2 g:2
Sep 22 23:15:25.623 DEBG [2] Read already AckReady 1102, : downstairs
Sep 22 23:15:25.626 ERRO [2] job id 1103 saw error GenericError("test error")
Sep 22 23:15:25.626 ERRO [2] job id 1103 saw error GenericError("test error")
Sep 22 23:15:25.626 DEBG up_ds_listen was notified
Sep 22 23:15:25.626 DEBG up_ds_listen process 1102
Sep 22 23:15:25.626 DEBG [A] ack job 1102:103, : downstairs
Sep 22 23:15:25.669 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:2 g:2
Sep 22 23:15:25.683 DEBG up_ds_listen process 1103
Sep 22 23:15:25.683 DEBG [A] ack job 1103:104, : downstairs
Sep 22 23:15:25.683 DEBG up_ds_listen checked 2 jobs, back to waiting
Sep 22 23:15:25.683 DEBG up_ds_listen was notified
Sep 22 23:15:25.683 DEBG up_ds_listen checked 0 jobs, back to waiting
Sep 22 23:15:25.684 INFO [lossy] skipping 1102
Sep 22 23:15:25.690 DEBG Read      :1102 deps:[JobId(1101)] res:true
Sep 22 23:15:25.711 DEBG IO Read  1104 has deps [JobId(1103)]
Sep 22 23:15:25.725 DEBG Flush     :1103 extent_limit None deps:[JobId(1102), JobId(1101)] res:true f:41 g:1
Sep 22 23:15:25.731 DEBG Read      :1104 deps:[JobId(1103)] res:true
Sep 22 23:15:26.129 DEBG [rc] retire 1103 clears [JobId(1102), JobId(1103)], : downstairs
Sep 22 23:15:26.129 INFO [lossy] skipping 1104
Sep 22 23:15:26.129 WARN returning error on read!
Sep 22 23:15:26.129 DEBG Read      :1104 deps:[JobId(1103)] res:false
Sep 22 23:15:26.129 WARN returning error on read!
Sep 22 23:15:26.129 DEBG Read      :1104 deps:[JobId(1103)] res:false
Sep 22 23:15:26.135 DEBG Read      :1104 deps:[JobId(1103)] res:true
Sep 22 23:15:26.162 DEBG Read      :1104 deps:[JobId(1103)] res:true
Sep 22 23:15:26.184 ERRO [2] job id 1104 saw error GenericError("test error")
Sep 22 23:15:26.184 ERRO [2] job id 1104 saw error GenericError("test error")
Sep 22 23:15:26.185 DEBG IO Flush 1105 has deps [JobId(1104)]
Sep 22 23:15:26.187 INFO [lossy] skipping 1105
Sep 22 23:15:26.187 DEBG Flush     :1105 extent_limit None deps:[JobId(1104)] res:true f:42 g:1
Sep 22 23:15:26.187 INFO [lossy] sleeping 1 second
Sep 22 23:15:26.563 DEBG [1] Read AckReady 1104, : downstairs
Sep 22 23:15:26.564 DEBG up_ds_listen was notified
Sep 22 23:15:26.564 DEBG up_ds_listen process 1104
Sep 22 23:15:26.564 DEBG [A] ack job 1104:105, : downstairs
Sep 22 23:15:26.616 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:26.618 INFO [lossy] sleeping 1 second
Sep 22 23:15:26.619 DEBG Flush     :1105 extent_limit None deps:[JobId(1104)] res:true f:42 g:1
Sep 22 23:15:26.620 INFO [lossy] sleeping 1 second
Sep 22 23:15:26.714 DEBG IO Read  1106 has deps [JobId(1105)]
Sep 22 23:15:27.380 DEBG up_ds_listen was notified
Sep 22 23:15:27.380 DEBG up_ds_listen process 1105
Sep 22 23:15:27.380 DEBG [A] ack job 1105:106, : downstairs
Sep 22 23:15:27.380 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:27.387 DEBG IO Flush 1107 has deps [JobId(1106), JobId(1105)]
Sep 22 23:15:27.387 INFO [lossy] sleeping 1 second
Sep 22 23:15:27.620 DEBG Flush     :1105 extent_limit None deps:[JobId(1104)] res:true f:42 g:1
Sep 22 23:15:27.626 DEBG Read      :1106 deps:[JobId(1105)] res:true
Sep 22 23:15:27.647 INFO [lossy] skipping 1106
Sep 22 23:15:27.653 DEBG Read      :1106 deps:[JobId(1105)] res:true
test test::integration_test_problematic_downstairs has been running for over 60 seconds
Sep 22 23:15:27.675 DEBG [rc] retire 1105 clears [JobId(1104), JobId(1105)], : downstairs
Sep 22 23:15:27.677 DEBG Flush     :1107 extent_limit None deps:[JobId(1106), JobId(1105)] res:true f:43 g:1
Sep 22 23:15:27.677 INFO [lossy] sleeping 1 second
Sep 22 23:15:27.678 DEBG Flush     :1107 extent_limit None deps:[JobId(1106), JobId(1105)] res:true f:43 g:1
Sep 22 23:15:27.678 INFO [lossy] sleeping 1 second
test test::integration_test_volume_replace_downstairs_then_takeover has been running for over 60 seconds
Sep 22 23:15:28.101 DEBG [0] Read AckReady 1106, : downstairs
Sep 22 23:15:28.429 DEBG [2] Read already AckReady 1106, : downstairs
Sep 22 23:15:28.431 DEBG up_ds_listen was notified
Sep 22 23:15:28.431 DEBG up_ds_listen process 1106
Sep 22 23:15:28.431 DEBG [A] ack job 1106:107, : downstairs
Sep 22 23:15:28.484 DEBG up_ds_listen process 1107
Sep 22 23:15:28.484 DEBG [A] ack job 1107:108, : downstairs
Sep 22 23:15:28.484 DEBG up_ds_listen checked 2 jobs, back to waiting
Sep 22 23:15:28.484 DEBG up_ds_listen was notified
Sep 22 23:15:28.484 DEBG up_ds_listen checked 0 jobs, back to waiting
Sep 22 23:15:28.485 INFO [lossy] skipping 1106
Sep 22 23:15:28.485 INFO [lossy] skipping 1107
Sep 22 23:15:28.485 WARN returning error on read!
Sep 22 23:15:28.485 DEBG Read      :1106 deps:[JobId(1105)] res:false
Sep 22 23:15:28.485 INFO [lossy] skipping 1106
Sep 22 23:15:28.491 DEBG Read      :1106 deps:[JobId(1105)] res:true
Sep 22 23:15:28.513 DEBG IO Read  1108 has deps [JobId(1107)]
Sep 22 23:15:28.513 ERRO [1] job id 1106 saw error GenericError("test error")
Sep 22 23:15:28.527 DEBG Flush     :1107 extent_limit None deps:[JobId(1106), JobId(1105)] res:true f:43 g:1
Sep 22 23:15:28.532 DEBG Read      :1108 deps:[JobId(1107)] res:true
Sep 22 23:15:28.931 DEBG [rc] retire 1107 clears [JobId(1106), JobId(1107)], : downstairs
Sep 22 23:15:28.937 DEBG Read      :1108 deps:[JobId(1107)] res:true
Sep 22 23:15:28.965 DEBG Read      :1108 deps:[JobId(1107)] res:true
Sep 22 23:15:28.986 DEBG IO Flush 1109 has deps [JobId(1108)]
Sep 22 23:15:28.989 INFO [lossy] skipping 1109
Sep 22 23:15:28.989 DEBG Flush     :1109 extent_limit None deps:[JobId(1108)] res:true f:44 g:1
Sep 22 23:15:28.989 INFO [lossy] sleeping 1 second
Sep 22 23:15:29.366 DEBG [1] Read AckReady 1108, : downstairs
Sep 22 23:15:29.367 DEBG up_ds_listen was notified
Sep 22 23:15:29.367 DEBG up_ds_listen process 1108
Sep 22 23:15:29.367 DEBG [A] ack job 1108:109, : downstairs
Sep 22 23:15:29.419 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:29.422 INFO [lossy] skipping 1109
Sep 22 23:15:29.422 INFO [lossy] skipping 1109
Sep 22 23:15:29.422 DEBG Flush     :1109 extent_limit None deps:[JobId(1108)] res:true f:44 g:1
Sep 22 23:15:29.422 INFO [lossy] sleeping 1 second
Sep 22 23:15:29.423 DEBG Flush     :1109 extent_limit None deps:[JobId(1108)] res:true f:44 g:1
Sep 22 23:15:29.423 INFO [lossy] sleeping 1 second
Sep 22 23:15:29.518 DEBG IO Read  1110 has deps [JobId(1109)]
Sep 22 23:15:30.187 DEBG up_ds_listen was notified
Sep 22 23:15:30.187 DEBG up_ds_listen process 1109
Sep 22 23:15:30.188 DEBG [A] ack job 1109:110, : downstairs
Sep 22 23:15:30.188 DEBG [rc] retire 1109 clears [JobId(1108), JobId(1109)], : downstairs
Sep 22 23:15:30.188 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:30.195 DEBG IO Flush 1111 has deps [JobId(1110)]
Sep 22 23:15:30.195 INFO [lossy] sleeping 1 second
Sep 22 23:15:30.429 DEBG Read      :1110 deps:[JobId(1109)] res:true
Sep 22 23:15:30.457 DEBG Read      :1110 deps:[JobId(1109)] res:true
Sep 22 23:15:30.481 INFO [lossy] skipping 1111
Sep 22 23:15:30.481 INFO [lossy] skipping 1111
Sep 22 23:15:30.481 WARN returning error on flush!
Sep 22 23:15:30.481 DEBG Flush     :1111 extent_limit None deps:[JobId(1110)] res:false f:45 g:1
Sep 22 23:15:30.481 WARN returning error on flush!
Sep 22 23:15:30.481 DEBG Flush     :1111 extent_limit None deps:[JobId(1110)] res:false f:45 g:1
Sep 22 23:15:30.481 INFO [lossy] skipping 1111
Sep 22 23:15:30.481 DEBG Flush     :1111 extent_limit None deps:[JobId(1110)] res:true f:45 g:1
Sep 22 23:15:30.481 INFO [lossy] sleeping 1 second
Sep 22 23:15:30.529 INFO [lossy] skipping 1111
Sep 22 23:15:30.530 DEBG Flush     :1111 extent_limit None deps:[JobId(1110)] res:true f:45 g:1
Sep 22 23:15:30.859 DEBG [2] Read AckReady 1110, : downstairs
Sep 22 23:15:30.860 ERRO [2] job id 1111 saw error GenericError("test error")
Sep 22 23:15:30.860 ERRO [2] job id 1111 saw error GenericError("test error")
Sep 22 23:15:30.860 DEBG up_ds_listen was notified
Sep 22 23:15:30.860 DEBG up_ds_listen process 1110
Sep 22 23:15:30.860 DEBG [A] ack job 1110:111, : downstairs
Sep 22 23:15:30.912 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:31.291 DEBG up_ds_listen was notified
Sep 22 23:15:31.291 DEBG up_ds_listen process 1111
Sep 22 23:15:31.291 DEBG [A] ack job 1111:112, : downstairs
Sep 22 23:15:31.291 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:31.298 DEBG Read      :1110 deps:[JobId(1109)] res:true
Sep 22 23:15:31.319 DEBG IO Read  1112 has deps [JobId(1111)]
Sep 22 23:15:31.332 INFO [lossy] skipping 1112
Sep 22 23:15:31.332 INFO [lossy] skipping 1112
Sep 22 23:15:31.338 DEBG Read      :1112 deps:[JobId(1111)] res:true
Sep 22 23:15:31.361 WARN returning error on flush!
Sep 22 23:15:31.361 DEBG Flush     :1111 extent_limit None deps:[JobId(1110)] res:false f:45 g:1
Sep 22 23:15:31.361 INFO [lossy] skipping 1111
Sep 22 23:15:31.361 DEBG Flush     :1111 extent_limit None deps:[JobId(1110)] res:true f:45 g:1
Sep 22 23:15:31.361 INFO [lossy] sleeping 1 second
Sep 22 23:15:31.738 ERRO [1] job id 1111 saw error GenericError("test error")
Sep 22 23:15:31.739 DEBG [rc] retire 1111 clears [JobId(1110), JobId(1111)], : downstairs
Sep 22 23:15:31.786 DEBG IO Flush 1113 has deps [JobId(1112)]
Sep 22 23:15:31.786 WARN returning error on read!
Sep 22 23:15:31.786 DEBG Read      :1112 deps:[JobId(1111)] res:false
Sep 22 23:15:31.792 DEBG Read      :1112 deps:[JobId(1111)] res:true
Sep 22 23:15:32.142 DEBG [0] Read AckReady 1112, : downstairs
Sep 22 23:15:32.143 DEBG up_ds_listen was notified
Sep 22 23:15:32.143 DEBG up_ds_listen process 1112
Sep 22 23:15:32.143 DEBG [A] ack job 1112:113, : downstairs
Sep 22 23:15:32.196 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:32.197 ERRO [2] job id 1112 saw error GenericError("test error")
Sep 22 23:15:32.197 INFO [lossy] skipping 1113
Sep 22 23:15:32.197 INFO [lossy] skipping 1113
Sep 22 23:15:32.197 DEBG Flush     :1113 extent_limit None deps:[JobId(1112)] res:true f:46 g:1
Sep 22 23:15:32.197 DEBG IO Read  1114 has deps [JobId(1113)]
Sep 22 23:15:32.216 DEBG Read      :1114 deps:[JobId(1113)] res:true
Sep 22 23:15:32.239 DEBG Flush     :1113 extent_limit None deps:[JobId(1112)] res:true f:46 g:1
Sep 22 23:15:32.245 DEBG Read      :1114 deps:[JobId(1113)] res:true
Sep 22 23:15:32.645 DEBG up_ds_listen was notified
Sep 22 23:15:32.646 DEBG up_ds_listen process 1113
Sep 22 23:15:32.646 DEBG [A] ack job 1113:114, : downstairs
Sep 22 23:15:32.646 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:32.646 DEBG IO Flush 1115 has deps [JobId(1114), JobId(1113)]
Sep 22 23:15:32.646 INFO [lossy] skipping 1112
Sep 22 23:15:32.657 DEBG Read      :1112 deps:[JobId(1111)] res:true
Sep 22 23:15:32.680 DEBG Flush     :1115 extent_limit None deps:[JobId(1114), JobId(1113)] res:true f:47 g:1
Sep 22 23:15:33.171 DEBG [0] Read AckReady 1114, : downstairs
Sep 22 23:15:33.172 DEBG up_ds_listen was notified
Sep 22 23:15:33.172 DEBG up_ds_listen process 1114
Sep 22 23:15:33.172 DEBG [A] ack job 1114:115, : downstairs
Sep 22 23:15:33.225 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:33.226 DEBG IO Read  1116 has deps [JobId(1115)]
Sep 22 23:15:33.246 DEBG Read      :1116 deps:[JobId(1115)] res:true
Sep 22 23:15:33.269 INFO [lossy] skipping 1115
Sep 22 23:15:33.270 DEBG Flush     :1115 extent_limit None deps:[JobId(1114), JobId(1113)] res:true f:47 g:1
Sep 22 23:15:33.270 INFO [lossy] sleeping 1 second
Sep 22 23:15:33.651 DEBG up_ds_listen was notified
Sep 22 23:15:33.651 DEBG up_ds_listen process 1115
Sep 22 23:15:33.651 DEBG [A] ack job 1115:116, : downstairs
Sep 22 23:15:33.651 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:33.652 WARN returning error on flush!
Sep 22 23:15:33.653 DEBG Flush     :1113 extent_limit None deps:[JobId(1112)] res:false f:46 g:1
Sep 22 23:15:33.653 DEBG Flush     :1113 extent_limit None deps:[JobId(1112)] res:true f:46 g:1
Sep 22 23:15:33.653 WARN returning error on read!
Sep 22 23:15:33.653 DEBG Read      :1114 deps:[JobId(1113)] res:false
Sep 22 23:15:33.653 WARN 1115 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:15:33.653 INFO [lossy] skipping 1116
Sep 22 23:15:33.653 WARN returning error on read!
Sep 22 23:15:33.653 DEBG Read      :1114 deps:[JobId(1113)] res:false
Sep 22 23:15:33.653 WARN returning error on read!
Sep 22 23:15:33.653 DEBG Read      :1114 deps:[JobId(1113)] res:false
Sep 22 23:15:33.659 DEBG Read      :1114 deps:[JobId(1113)] res:true
Sep 22 23:15:34.057 ERRO [1] job id 1113 saw error GenericError("test error")
Sep 22 23:15:34.057 DEBG [rc] retire 1113 clears [JobId(1112), JobId(1113)], : downstairs
Sep 22 23:15:34.057 ERRO [1] job id 1114 saw error GenericError("test error")
Sep 22 23:15:34.057 ERRO [1] job id 1114 saw error GenericError("test error")
Sep 22 23:15:34.057 ERRO [1] job id 1114 saw error GenericError("test error")
Sep 22 23:15:34.058 DEBG IO Flush 1117 has deps [JobId(1116), JobId(1115)]
Sep 22 23:15:34.060 INFO [lossy] sleeping 1 second
Sep 22 23:15:34.435 DEBG [0] Read AckReady 1116, : downstairs
Sep 22 23:15:34.436 DEBG up_ds_listen was notified
Sep 22 23:15:34.436 DEBG up_ds_listen process 1116
Sep 22 23:15:34.436 DEBG [A] ack job 1116:117, : downstairs
Sep 22 23:15:34.489 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:34.495 DEBG Read      :1116 deps:[JobId(1115)] res:true
Sep 22 23:15:34.517 DEBG IO Read  1118 has deps [JobId(1117)]
Sep 22 23:15:34.531 DEBG Flush     :1115 extent_limit None deps:[JobId(1114), JobId(1113)] res:true f:47 g:1
Sep 22 23:15:34.531 INFO [lossy] skipping 1116
Sep 22 23:15:34.531 WARN returning error on read!
Sep 22 23:15:34.531 DEBG Read      :1116 deps:[JobId(1115)] res:false
Sep 22 23:15:34.537 DEBG Read      :1116 deps:[JobId(1115)] res:true
Sep 22 23:15:34.606 DEBG IO Flush 1119 has deps [JobId(1118), JobId(1117)]
Sep 22 23:15:34.937 DEBG [rc] retire 1115 clears [JobId(1114), JobId(1115)], : downstairs
Sep 22 23:15:34.937 ERRO [1] job id 1116 saw error GenericError("test error")
Sep 22 23:15:34.939 INFO [lossy] skipping 1117
Sep 22 23:15:34.939 DEBG Flush     :1117 extent_limit None deps:[JobId(1116), JobId(1115)] res:true f:48 g:1
Sep 22 23:15:34.940 WARN returning error on read!
Sep 22 23:15:34.940 DEBG Read      :1118 deps:[JobId(1117)] res:false
Sep 22 23:15:34.945 DEBG Read      :1118 deps:[JobId(1117)] res:true
Sep 22 23:15:35.344 ERRO [2] job id 1118 saw error GenericError("test error")
Sep 22 23:15:35.344 INFO [lossy] skipping 1117
Sep 22 23:15:35.344 INFO [lossy] skipping 1119
Sep 22 23:15:35.344 DEBG Flush     :1117 extent_limit None deps:[JobId(1116), JobId(1115)] res:true f:48 g:1
Sep 22 23:15:35.350 DEBG Read      :1118 deps:[JobId(1117)] res:true
Sep 22 23:15:35.372 DEBG up_ds_listen was notified
Sep 22 23:15:35.372 DEBG up_ds_listen process 1117
Sep 22 23:15:35.372 DEBG [A] ack job 1117:118, : downstairs
Sep 22 23:15:35.372 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:35.374 DEBG Flush     :1117 extent_limit None deps:[JobId(1116), JobId(1115)] res:true f:48 g:1
Sep 22 23:15:35.380 DEBG Read      :1118 deps:[JobId(1117)] res:true
Sep 22 23:15:35.784 DEBG [rc] retire 1117 clears [JobId(1116), JobId(1117)], : downstairs
Sep 22 23:15:35.786 INFO [lossy] skipping 1119
Sep 22 23:15:35.786 DEBG Flush     :1119 extent_limit None deps:[JobId(1118), JobId(1117)] res:true f:49 g:1
Sep 22 23:15:36.162 DEBG [2] Read AckReady 1118, : downstairs
Sep 22 23:15:36.162 DEBG up_ds_listen was notified
Sep 22 23:15:36.163 DEBG up_ds_listen process 1118
Sep 22 23:15:36.163 DEBG [A] ack job 1118:119, : downstairs
Sep 22 23:15:36.215 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:36.218 INFO [lossy] skipping 1119
Sep 22 23:15:36.218 DEBG Flush     :1119 extent_limit None deps:[JobId(1118), JobId(1117)] res:true f:49 g:1
Sep 22 23:15:36.218 INFO [lossy] sleeping 1 second
Sep 22 23:15:36.265 DEBG IO Read  1120 has deps [JobId(1119)]
Sep 22 23:15:36.600 DEBG up_ds_listen was notified
Sep 22 23:15:36.600 DEBG up_ds_listen process 1119
Sep 22 23:15:36.600 DEBG [A] ack job 1119:120, : downstairs
Sep 22 23:15:36.600 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:36.607 INFO [lossy] sleeping 1 second
Sep 22 23:15:36.609 INFO [lossy] skipping 1119
Sep 22 23:15:36.609 DEBG Flush     :1119 extent_limit None deps:[JobId(1118), JobId(1117)] res:true f:49 g:1
Sep 22 23:15:36.609 INFO [lossy] skipping 1120
Sep 22 23:15:36.615 DEBG Read      :1120 deps:[JobId(1119)] res:true
Sep 22 23:15:37.013 DEBG [rc] retire 1119 clears [JobId(1118), JobId(1119)], : downstairs
Sep 22 23:15:37.014 DEBG IO Flush 1121 has deps [JobId(1120)]
Sep 22 23:15:37.016 DEBG Flush     :1121 extent_limit None deps:[JobId(1120)] res:true f:50 g:1
Sep 22 23:15:37.016 INFO [lossy] sleeping 1 second
Sep 22 23:15:37.392 DEBG [1] Read AckReady 1120, : downstairs
Sep 22 23:15:37.392 DEBG up_ds_listen was notified
Sep 22 23:15:37.392 DEBG up_ds_listen process 1120
Sep 22 23:15:37.392 DEBG [A] ack job 1120:121, : downstairs
Sep 22 23:15:37.445 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:37.446 INFO [lossy] skipping 1120
Sep 22 23:15:37.446 WARN returning error on read!
Sep 22 23:15:37.446 DEBG Read      :1120 deps:[JobId(1119)] res:false
Sep 22 23:15:37.452 DEBG Read      :1120 deps:[JobId(1119)] res:true
Sep 22 23:15:37.473 DEBG IO Read  1122 has deps [JobId(1121)]
Sep 22 23:15:37.473 ERRO [0] job id 1120 saw error GenericError("test error")
Sep 22 23:15:37.487 DEBG Flush     :1121 extent_limit None deps:[JobId(1120)] res:true f:50 g:1
Sep 22 23:15:37.493 DEBG Read      :1122 deps:[JobId(1121)] res:true
Sep 22 23:15:37.891 DEBG up_ds_listen was notified
Sep 22 23:15:37.891 DEBG up_ds_listen process 1121
Sep 22 23:15:37.891 DEBG [A] ack job 1121:122, : downstairs
Sep 22 23:15:37.891 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:37.891 DEBG IO Flush 1123 has deps [JobId(1122), JobId(1121)]
Sep 22 23:15:37.892 WARN returning error on read!
Sep 22 23:15:37.892 DEBG Read      :1120 deps:[JobId(1119)] res:false
Sep 22 23:15:37.892 INFO [lossy] skipping 1120
Sep 22 23:15:37.892 INFO [lossy] skipping 1120
Sep 22 23:15:37.897 DEBG Read      :1120 deps:[JobId(1119)] res:true
Sep 22 23:15:37.919 ERRO [2] job id 1120 saw error GenericError("test error")
Sep 22 23:15:37.921 DEBG Flush     :1123 extent_limit None deps:[JobId(1122), JobId(1121)] res:true f:51 g:1
Sep 22 23:15:37.921 INFO [lossy] sleeping 1 second
Sep 22 23:15:38.296 DEBG [0] Read AckReady 1122, : downstairs
Sep 22 23:15:38.297 DEBG up_ds_listen was notified
Sep 22 23:15:38.297 DEBG up_ds_listen process 1122
Sep 22 23:15:38.297 DEBG [A] ack job 1122:123, : downstairs
Sep 22 23:15:38.350 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:38.352 DEBG Flush     :1121 extent_limit None deps:[JobId(1120)] res:true f:50 g:1
Sep 22 23:15:38.352 INFO [lossy] skipping 1122
Sep 22 23:15:38.352 INFO [lossy] skipping 1122
Sep 22 23:15:38.358 DEBG Read      :1122 deps:[JobId(1121)] res:true
Sep 22 23:15:38.385 DEBG Read      :1122 deps:[JobId(1121)] res:true
Sep 22 23:15:38.454 DEBG IO Read  1124 has deps [JobId(1123)]
Sep 22 23:15:38.784 DEBG [rc] retire 1121 clears [JobId(1120), JobId(1121)], : downstairs
Sep 22 23:15:38.800 INFO [lossy] skipping 1123
Sep 22 23:15:38.800 WARN returning error on flush!
Sep 22 23:15:38.800 DEBG Flush     :1123 extent_limit None deps:[JobId(1122), JobId(1121)] res:false f:51 g:1
Sep 22 23:15:38.800 INFO [lossy] skipping 1123
Sep 22 23:15:38.800 WARN returning error on flush!
Sep 22 23:15:38.800 DEBG Flush     :1123 extent_limit None deps:[JobId(1122), JobId(1121)] res:false f:51 g:1
Sep 22 23:15:38.800 DEBG Flush     :1123 extent_limit None deps:[JobId(1122), JobId(1121)] res:true f:51 g:1
Sep 22 23:15:38.800 WARN returning error on read!
Sep 22 23:15:38.800 DEBG Read      :1124 deps:[JobId(1123)] res:false
Sep 22 23:15:38.806 DEBG Read      :1124 deps:[JobId(1123)] res:true
Sep 22 23:15:38.827 INFO [lossy] skipping 1123
Sep 22 23:15:38.827 WARN returning error on flush!
Sep 22 23:15:38.828 DEBG Flush     :1123 extent_limit None deps:[JobId(1122), JobId(1121)] res:false f:51 g:1
Sep 22 23:15:38.828 DEBG Flush     :1123 extent_limit None deps:[JobId(1122), JobId(1121)] res:true f:51 g:1
Sep 22 23:15:38.833 DEBG Read      :1124 deps:[JobId(1123)] res:true
Sep 22 23:15:39.281 ERRO [2] job id 1123 saw error GenericError("test error")
Sep 22 23:15:39.610 ERRO [1] job id 1123 saw error GenericError("test error")
Sep 22 23:15:39.611 ERRO [1] job id 1123 saw error GenericError("test error")
Sep 22 23:15:39.611 ERRO [1] job id 1124 saw error GenericError("test error")
Sep 22 23:15:39.611 DEBG up_ds_listen was notified
Sep 22 23:15:39.611 DEBG up_ds_listen process 1123
Sep 22 23:15:39.611 DEBG [A] ack job 1123:124, : downstairs
Sep 22 23:15:39.611 DEBG [rc] retire 1123 clears [JobId(1122), JobId(1123)], : downstairs
Sep 22 23:15:39.611 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:39.611 WARN returning error on read!
Sep 22 23:15:39.611 DEBG Read      :1124 deps:[JobId(1123)] res:false
Sep 22 23:15:39.617 DEBG Read      :1124 deps:[JobId(1123)] res:true
Sep 22 23:15:39.638 DEBG IO Flush 1125 has deps [JobId(1124)]
Sep 22 23:15:39.639 ERRO [0] job id 1124 saw error GenericError("test error")
Sep 22 23:15:39.641 DEBG Flush     :1125 extent_limit None deps:[JobId(1124)] res:true f:52 g:1
Sep 22 23:15:39.642 INFO [lossy] sleeping 1 second
Sep 22 23:15:40.017 DEBG [1] Read AckReady 1124, : downstairs
Sep 22 23:15:40.018 DEBG up_ds_listen was notified
Sep 22 23:15:40.018 DEBG up_ds_listen process 1124
Sep 22 23:15:40.018 DEBG [A] ack job 1124:125, : downstairs
Sep 22 23:15:40.071 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:40.073 DEBG Flush     :1125 extent_limit None deps:[JobId(1124)] res:true f:52 g:1
Sep 22 23:15:40.075 DEBG Flush     :1125 extent_limit None deps:[JobId(1124)] res:true f:52 g:1
Sep 22 23:15:40.075 INFO [lossy] sleeping 1 second
Sep 22 23:15:40.170 DEBG IO Read  1126 has deps [JobId(1125)]
Sep 22 23:15:40.835 DEBG up_ds_listen was notified
Sep 22 23:15:40.835 DEBG up_ds_listen process 1125
Sep 22 23:15:40.835 DEBG [A] ack job 1125:126, : downstairs
Sep 22 23:15:40.835 DEBG [rc] retire 1125 clears [JobId(1124), JobId(1125)], : downstairs
Sep 22 23:15:40.835 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:40.842 DEBG IO Flush 1127 has deps [JobId(1126)]
Sep 22 23:15:40.848 DEBG Read      :1126 deps:[JobId(1125)] res:true
Sep 22 23:15:40.875 DEBG Read      :1126 deps:[JobId(1125)] res:true
Sep 22 23:15:40.899 DEBG Flush     :1127 extent_limit None deps:[JobId(1126)] res:true f:53 g:1
Sep 22 23:15:40.900 INFO [lossy] skipping 1127
Sep 22 23:15:40.900 DEBG Flush     :1127 extent_limit None deps:[JobId(1126)] res:true f:53 g:1
Sep 22 23:15:41.323 DEBG [2] Read AckReady 1126, : downstairs
Sep 22 23:15:41.652 DEBG [1] Read already AckReady 1126, : downstairs
Sep 22 23:15:41.654 DEBG up_ds_listen was notified
Sep 22 23:15:41.654 DEBG up_ds_listen process 1126
Sep 22 23:15:41.654 DEBG [A] ack job 1126:127, : downstairs
Sep 22 23:15:41.707 DEBG up_ds_listen process 1127
Sep 22 23:15:41.707 DEBG [A] ack job 1127:128, : downstairs
Sep 22 23:15:41.707 DEBG up_ds_listen checked 2 jobs, back to waiting
Sep 22 23:15:41.707 DEBG up_ds_listen was notified
Sep 22 23:15:41.707 DEBG up_ds_listen checked 0 jobs, back to waiting
Sep 22 23:15:41.708 WARN returning error on read!
Sep 22 23:15:41.708 DEBG Read      :1126 deps:[JobId(1125)] res:false
Sep 22 23:15:41.708 INFO [lossy] skipping 1127
Sep 22 23:15:41.714 DEBG Read      :1126 deps:[JobId(1125)] res:true
Sep 22 23:15:41.736 DEBG IO Read  1128 has deps [JobId(1127)]
Sep 22 23:15:41.736 ERRO [0] job id 1126 saw error GenericError("test error")
Sep 22 23:15:41.748 INFO [lossy] sleeping 1 second
Sep 22 23:15:41.754 DEBG Read      :1128 deps:[JobId(1127)] res:true
Sep 22 23:15:41.777 DEBG Flush     :1127 extent_limit None deps:[JobId(1126)] res:true f:53 g:1
Sep 22 23:15:41.783 DEBG Read      :1128 deps:[JobId(1127)] res:true
Sep 22 23:15:42.182 DEBG [rc] retire 1127 clears [JobId(1126), JobId(1127)], : downstairs
Sep 22 23:15:42.559 DEBG [2] Read AckReady 1128, : downstairs
Sep 22 23:15:42.560 DEBG up_ds_listen was notified
Sep 22 23:15:42.560 DEBG up_ds_listen process 1128
Sep 22 23:15:42.560 DEBG [A] ack job 1128:129, : downstairs
Sep 22 23:15:42.612 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:42.613 DEBG IO Flush 1129 has deps [JobId(1128)]
Sep 22 23:15:42.614 DEBG IO Read  1130 has deps [JobId(1129)]
Sep 22 23:15:42.619 DEBG Flush     :1129 extent_limit None deps:[JobId(1128)] res:true f:54 g:1
Sep 22 23:15:42.626 INFO [lossy] sleeping 1 second
Sep 22 23:15:42.628 WARN returning error on flush!
Sep 22 23:15:42.628 DEBG Flush     :1129 extent_limit None deps:[JobId(1128)] res:false f:54 g:1
Sep 22 23:15:42.628 INFO [lossy] skipping 1130
Sep 22 23:15:42.628 WARN returning error on flush!
Sep 22 23:15:42.628 DEBG Flush     :1129 extent_limit None deps:[JobId(1128)] res:false f:54 g:1
Sep 22 23:15:42.628 INFO [lossy] skipping 1130
Sep 22 23:15:42.628 WARN returning error on flush!
Sep 22 23:15:42.628 DEBG Flush     :1129 extent_limit None deps:[JobId(1128)] res:false f:54 g:1
Sep 22 23:15:42.628 DEBG Flush     :1129 extent_limit None deps:[JobId(1128)] res:true f:54 g:1
Sep 22 23:15:42.628 INFO [lossy] skipping 1130
Sep 22 23:15:42.629 WARN returning error on read!
Sep 22 23:15:42.629 DEBG Read      :1130 deps:[JobId(1129)] res:false
Sep 22 23:15:42.629 INFO [lossy] skipping 1130
Sep 22 23:15:42.634 DEBG Read      :1130 deps:[JobId(1129)] res:true
Sep 22 23:15:43.033 ERRO [0] job id 1129 saw error GenericError("test error")
Sep 22 23:15:43.033 ERRO [0] job id 1129 saw error GenericError("test error")
Sep 22 23:15:43.033 ERRO [0] job id 1129 saw error GenericError("test error")
Sep 22 23:15:43.033 ERRO [0] job id 1130 saw error GenericError("test error")
Sep 22 23:15:43.033 DEBG up_ds_listen was notified
Sep 22 23:15:43.033 DEBG up_ds_listen process 1129
Sep 22 23:15:43.033 DEBG [A] ack job 1129:130, : downstairs
Sep 22 23:15:43.033 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:43.034 INFO [lossy] skipping 1128
Sep 22 23:15:43.034 INFO [lossy] skipping 1130
Sep 22 23:15:43.034 INFO [lossy] skipping 1128
Sep 22 23:15:43.034 INFO [lossy] skipping 1128
Sep 22 23:15:43.034 INFO [lossy] skipping 1128
Sep 22 23:15:43.040 DEBG Read      :1128 deps:[JobId(1127)] res:true
Sep 22 23:15:43.063 WARN returning error on flush!
Sep 22 23:15:43.063 DEBG Flush     :1129 extent_limit None deps:[JobId(1128)] res:false f:54 g:1
Sep 22 23:15:43.063 INFO [lossy] skipping 1130
Sep 22 23:15:43.063 WARN returning error on flush!
Sep 22 23:15:43.063 DEBG Flush     :1129 extent_limit None deps:[JobId(1128)] res:false f:54 g:1
Sep 22 23:15:43.063 WARN returning error on flush!
Sep 22 23:15:43.063 DEBG Flush     :1129 extent_limit None deps:[JobId(1128)] res:false f:54 g:1
Sep 22 23:15:43.063 DEBG Flush     :1129 extent_limit None deps:[JobId(1128)] res:true f:54 g:1
Sep 22 23:15:43.063 INFO [lossy] skipping 1130
Sep 22 23:15:43.069 DEBG Read      :1130 deps:[JobId(1129)] res:true
Sep 22 23:15:43.091 INFO [lossy] sleeping 1 second
Sep 22 23:15:43.514 DEBG [0] Read AckReady 1130, : downstairs
Sep 22 23:15:43.844 ERRO [1] job id 1129 saw error GenericError("test error")
Sep 22 23:15:43.844 ERRO [1] job id 1129 saw error GenericError("test error")
Sep 22 23:15:43.844 ERRO [1] job id 1129 saw error GenericError("test error")
Sep 22 23:15:43.845 DEBG [rc] retire 1129 clears [JobId(1128), JobId(1129)], : downstairs
Sep 22 23:15:43.845 DEBG up_ds_listen was notified
Sep 22 23:15:43.845 DEBG up_ds_listen process 1130
Sep 22 23:15:43.845 DEBG [A] ack job 1130:131, : downstairs
Sep 22 23:15:43.898 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:43.899 DEBG IO Flush 1131 has deps [JobId(1130)]
Sep 22 23:15:43.905 DEBG Read      :1130 deps:[JobId(1129)] res:true
Sep 22 23:15:43.927 DEBG IO Read  1132 has deps [JobId(1131)]
Sep 22 23:15:43.941 INFO [lossy] sleeping 1 second
Sep 22 23:15:44.320 INFO [lossy] sleeping 1 second
Sep 22 23:15:44.321 DEBG Flush     :1131 extent_limit None deps:[JobId(1130)] res:true f:55 g:1
Sep 22 23:15:44.327 DEBG Read      :1132 deps:[JobId(1131)] res:true
Sep 22 23:15:44.725 DEBG IO Flush 1133 has deps [JobId(1132), JobId(1131)]
Sep 22 23:15:44.728 INFO [lossy] sleeping 1 second
Sep 22 23:15:45.103 DEBG [0] Read AckReady 1132, : downstairs
Sep 22 23:15:45.104 DEBG up_ds_listen was notified
Sep 22 23:15:45.104 DEBG up_ds_listen process 1132
Sep 22 23:15:45.104 DEBG [A] ack job 1132:133, : downstairs
Sep 22 23:15:45.157 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:45.158 INFO [lossy] skipping 1131
Sep 22 23:15:45.158 INFO [lossy] skipping 1132
Sep 22 23:15:45.158 WARN returning error on flush!
Sep 22 23:15:45.158 DEBG Flush     :1131 extent_limit None deps:[JobId(1130)] res:false f:55 g:1
Sep 22 23:15:45.158 WARN returning error on flush!
Sep 22 23:15:45.158 DEBG Flush     :1131 extent_limit None deps:[JobId(1130)] res:false f:55 g:1
Sep 22 23:15:45.158 INFO [lossy] skipping 1131
Sep 22 23:15:45.158 INFO [lossy] skipping 1131
Sep 22 23:15:45.158 WARN returning error on flush!
Sep 22 23:15:45.158 DEBG Flush     :1131 extent_limit None deps:[JobId(1130)] res:false f:55 g:1
Sep 22 23:15:45.158 DEBG Flush     :1131 extent_limit None deps:[JobId(1130)] res:true f:55 g:1
Sep 22 23:15:45.158 INFO [lossy] skipping 1132
Sep 22 23:15:45.158 WARN 1133 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:15:45.164 DEBG Read      :1132 deps:[JobId(1131)] res:true
Sep 22 23:15:45.186 DEBG IO Read  1134 has deps [JobId(1133)]
Sep 22 23:15:45.186 ERRO [1] job id 1131 saw error GenericError("test error")
Sep 22 23:15:45.186 ERRO [1] job id 1131 saw error GenericError("test error")
Sep 22 23:15:45.186 ERRO [1] job id 1131 saw error GenericError("test error")
Sep 22 23:15:45.191 DEBG up_ds_listen was notified
Sep 22 23:15:45.191 DEBG up_ds_listen process 1131
Sep 22 23:15:45.191 DEBG [A] ack job 1131:132, : downstairs
Sep 22 23:15:45.191 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:45.199 DEBG Flush     :1133 extent_limit None deps:[JobId(1132), JobId(1131)] res:true f:56 g:1
Sep 22 23:15:45.199 INFO [lossy] skipping 1134
Sep 22 23:15:45.199 WARN returning error on read!
Sep 22 23:15:45.199 DEBG Read      :1134 deps:[JobId(1133)] res:false
Sep 22 23:15:45.205 DEBG Read      :1134 deps:[JobId(1133)] res:true
Sep 22 23:15:45.274 DEBG IO Flush 1135 has deps [JobId(1134), JobId(1133)]
Sep 22 23:15:45.604 ERRO [1] job id 1134 saw error GenericError("test error")
Sep 22 23:15:45.605 DEBG Flush     :1131 extent_limit None deps:[JobId(1130)] res:true f:55 g:1
Sep 22 23:15:45.610 DEBG Read      :1132 deps:[JobId(1131)] res:true
Sep 22 23:15:45.632 DEBG [rc] retire 1131 clears [JobId(1130), JobId(1131)], : downstairs
Sep 22 23:15:45.633 INFO [lossy] skipping 1135
Sep 22 23:15:45.633 INFO [lossy] skipping 1135
Sep 22 23:15:45.633 INFO [lossy] skipping 1135
Sep 22 23:15:45.633 DEBG Flush     :1135 extent_limit None deps:[JobId(1134), JobId(1133)] res:true f:57 g:1
Sep 22 23:15:46.009 DEBG [1] Read AckReady 1134, : downstairs
Sep 22 23:15:46.010 DEBG up_ds_listen was notified
Sep 22 23:15:46.010 DEBG up_ds_listen process 1134
Sep 22 23:15:46.010 DEBG [A] ack job 1134:135, : downstairs
Sep 22 23:15:46.063 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:46.065 WARN returning error on flush!
Sep 22 23:15:46.065 DEBG Flush     :1133 extent_limit None deps:[JobId(1132), JobId(1131)] res:false f:56 g:1
Sep 22 23:15:46.065 DEBG Flush     :1133 extent_limit None deps:[JobId(1132), JobId(1131)] res:true f:56 g:1
Sep 22 23:15:46.071 DEBG Read      :1134 deps:[JobId(1133)] res:true
Sep 22 23:15:46.093 DEBG Flush     :1133 extent_limit None deps:[JobId(1132), JobId(1131)] res:true f:56 g:1
Sep 22 23:15:46.093 WARN returning error on read!
Sep 22 23:15:46.093 DEBG Read      :1134 deps:[JobId(1133)] res:false
Sep 22 23:15:46.093 WARN returning error on read!
Sep 22 23:15:46.093 DEBG Read      :1134 deps:[JobId(1133)] res:false
Sep 22 23:15:46.093 INFO [lossy] skipping 1134
Sep 22 23:15:46.093 INFO [lossy] skipping 1134
Sep 22 23:15:46.093 WARN returning error on read!
Sep 22 23:15:46.093 DEBG Read      :1134 deps:[JobId(1133)] res:false
Sep 22 23:15:46.093 INFO [lossy] skipping 1134
Sep 22 23:15:46.093 INFO [lossy] skipping 1134
Sep 22 23:15:46.093 WARN returning error on read!
Sep 22 23:15:46.093 DEBG Read      :1134 deps:[JobId(1133)] res:false
Sep 22 23:15:46.100 DEBG Read      :1134 deps:[JobId(1133)] res:true
Sep 22 23:15:46.170 DEBG IO Read  1136 has deps [JobId(1135)]
Sep 22 23:15:46.170 ERRO [0] job id 1134 saw error GenericError("test error")
Sep 22 23:15:46.170 ERRO [0] job id 1134 saw error GenericError("test error")
Sep 22 23:15:46.170 ERRO [0] job id 1134 saw error GenericError("test error")
Sep 22 23:15:46.170 ERRO [0] job id 1134 saw error GenericError("test error")
Sep 22 23:15:46.500 ERRO [2] job id 1133 saw error GenericError("test error")
Sep 22 23:15:46.505 DEBG up_ds_listen was notified
Sep 22 23:15:46.505 DEBG up_ds_listen process 1133
Sep 22 23:15:46.505 DEBG [A] ack job 1133:134, : downstairs
Sep 22 23:15:46.505 DEBG [rc] retire 1133 clears [JobId(1132), JobId(1133)], : downstairs
Sep 22 23:15:46.505 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:46.512 INFO [lossy] skipping 1136
Sep 22 23:15:46.518 DEBG Read      :1136 deps:[JobId(1135)] res:true
Sep 22 23:15:46.542 WARN returning error on flush!
Sep 22 23:15:46.542 DEBG Flush     :1135 extent_limit None deps:[JobId(1134), JobId(1133)] res:false f:57 g:1
Sep 22 23:15:46.542 DEBG Flush     :1135 extent_limit None deps:[JobId(1134), JobId(1133)] res:true f:57 g:1
Sep 22 23:15:46.542 INFO [lossy] sleeping 1 second
Sep 22 23:15:46.544 INFO [lossy] sleeping 1 second
Sep 22 23:15:47.300 ERRO [2] job id 1135 saw error GenericError("test error")
Sep 22 23:15:47.300 DEBG up_ds_listen was notified
Sep 22 23:15:47.300 DEBG up_ds_listen process 1135
Sep 22 23:15:47.300 DEBG [A] ack job 1135:136, : downstairs
Sep 22 23:15:47.300 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:47.302 DEBG IO Flush 1137 has deps [JobId(1136), JobId(1135)]
Sep 22 23:15:47.302 INFO [lossy] sleeping 1 second
Sep 22 23:15:47.678 DEBG [1] Read AckReady 1136, : downstairs
Sep 22 23:15:47.679 DEBG up_ds_listen was notified
Sep 22 23:15:47.679 DEBG up_ds_listen process 1136
Sep 22 23:15:47.679 DEBG [A] ack job 1136:137, : downstairs
Sep 22 23:15:47.731 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:47.738 DEBG Read      :1136 deps:[JobId(1135)] res:true
Sep 22 23:15:47.759 DEBG Flush     :1135 extent_limit None deps:[JobId(1134), JobId(1133)] res:true f:57 g:1
Sep 22 23:15:47.765 DEBG Read      :1136 deps:[JobId(1135)] res:true
Sep 22 23:15:47.786 DEBG IO Read  1138 has deps [JobId(1137)]
Sep 22 23:15:47.787 DEBG [rc] retire 1135 clears [JobId(1134), JobId(1135)], : downstairs
Sep 22 23:15:47.801 DEBG Flush     :1137 extent_limit None deps:[JobId(1136), JobId(1135)] res:true f:58 g:1
Sep 22 23:15:47.806 DEBG Read      :1138 deps:[JobId(1137)] res:true
Sep 22 23:15:47.828 DEBG Flush     :1137 extent_limit None deps:[JobId(1136), JobId(1135)] res:true f:58 g:1
Sep 22 23:15:47.828 WARN returning error on read!
Sep 22 23:15:47.828 DEBG Read      :1138 deps:[JobId(1137)] res:false
Sep 22 23:15:47.834 DEBG Read      :1138 deps:[JobId(1137)] res:true
Sep 22 23:15:47.952 DEBG IO Flush 1139 has deps [JobId(1138), JobId(1137)]
Sep 22 23:15:48.282 ERRO [0] job id 1138 saw error GenericError("test error")
Sep 22 23:15:48.518 DEBG [0] Read AckReady 1000, : downstairs
Sep 22 23:15:48.613 DEBG up_ds_listen was notified
Sep 22 23:15:48.613 DEBG up_ds_listen process 1137
Sep 22 23:15:48.614 DEBG [A] ack job 1137:138, : downstairs
Sep 22 23:15:48.614 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:48.614 INFO [lossy] skipping 1137
Sep 22 23:15:48.614 INFO [lossy] skipping 1138
Sep 22 23:15:48.614 DEBG Flush     :1137 extent_limit None deps:[JobId(1136), JobId(1135)] res:true f:58 g:1
Sep 22 23:15:48.620 DEBG Read      :1138 deps:[JobId(1137)] res:true
Sep 22 23:15:48.642 DEBG [rc] retire 1137 clears [JobId(1136), JobId(1137)], : downstairs
Sep 22 23:15:48.645 DEBG Flush     :1139 extent_limit None deps:[JobId(1138), JobId(1137)] res:true f:59 g:1
Sep 22 23:15:48.646 INFO [lossy] skipping 1139
Sep 22 23:15:48.646 INFO [lossy] skipping 1139
Sep 22 23:15:48.646 INFO [lossy] skipping 1139
Sep 22 23:15:48.646 WARN returning error on flush!
Sep 22 23:15:48.646 DEBG Flush     :1139 extent_limit None deps:[JobId(1138), JobId(1137)] res:false f:59 g:1
Sep 22 23:15:48.646 WARN returning error on flush!
Sep 22 23:15:48.646 DEBG Flush     :1139 extent_limit None deps:[JobId(1138), JobId(1137)] res:false f:59 g:1
Sep 22 23:15:48.646 DEBG Flush     :1139 extent_limit None deps:[JobId(1138), JobId(1137)] res:true f:59 g:1
Sep 22 23:15:48.646 INFO [lossy] sleeping 1 second
Sep 22 23:15:49.070 DEBG [0] Read AckReady 1138, : downstairs
Sep 22 23:15:49.071 ERRO [0] job id 1139 saw error GenericError("test error")
Sep 22 23:15:49.071 ERRO [0] job id 1139 saw error GenericError("test error")
Sep 22 23:15:49.399 DEBG [2] Read already AckReady 1138, : downstairs
Sep 22 23:15:49.401 DEBG up_ds_listen was notified
Sep 22 23:15:49.401 DEBG up_ds_listen process 1138
Sep 22 23:15:49.401 DEBG [A] ack job 1138:139, : downstairs
Sep 22 23:15:49.454 DEBG up_ds_listen process 1139
Sep 22 23:15:49.454 DEBG [A] ack job 1139:140, : downstairs
Sep 22 23:15:49.454 DEBG up_ds_listen checked 2 jobs, back to waiting
Sep 22 23:15:49.454 DEBG up_ds_listen was notified
Sep 22 23:15:49.454 DEBG up_ds_listen checked 0 jobs, back to waiting
Sep 22 23:15:49.456 INFO [lossy] sleeping 1 second
Sep 22 23:15:49.504 DEBG IO Read  1140 has deps [JobId(1139)]
Sep 22 23:15:49.851 DEBG Read      :1140 deps:[JobId(1139)] res:true
Sep 22 23:15:49.878 DEBG Read      :1140 deps:[JobId(1139)] res:true
Sep 22 23:15:50.326 DEBG [0] Read AckReady 1140, : downstairs
Sep 22 23:15:50.655 DEBG [2] Read already AckReady 1140, : downstairs
Sep 22 23:15:50.657 DEBG up_ds_listen was notified
Sep 22 23:15:50.657 DEBG up_ds_listen process 1140
Sep 22 23:15:50.657 DEBG [A] ack job 1140:141, : downstairs
Sep 22 23:15:50.709 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:50.711 DEBG IO Flush 1141 has deps [JobId(1140), JobId(1139)]
Sep 22 23:15:50.711 WARN returning error on flush!
Sep 22 23:15:50.711 DEBG Flush     :1139 extent_limit None deps:[JobId(1138), JobId(1137)] res:false f:59 g:1
Sep 22 23:15:50.711 INFO [lossy] skipping 1140
Sep 22 23:15:50.711 DEBG Flush     :1139 extent_limit None deps:[JobId(1138), JobId(1137)] res:true f:59 g:1
Sep 22 23:15:50.711 INFO [lossy] skipping 1140
Sep 22 23:15:50.711 WARN returning error on read!
Sep 22 23:15:50.711 DEBG Read      :1140 deps:[JobId(1139)] res:false
Sep 22 23:15:50.711 INFO [lossy] skipping 1140
Sep 22 23:15:50.711 WARN returning error on read!
Sep 22 23:15:50.711 DEBG Read      :1140 deps:[JobId(1139)] res:false
Sep 22 23:15:50.717 DEBG Read      :1140 deps:[JobId(1139)] res:true
Sep 22 23:15:50.739 DEBG IO Read  1142 has deps [JobId(1141)]
Sep 22 23:15:50.739 ERRO [1] job id 1139 saw error GenericError("test error")
Sep 22 23:15:50.739 DEBG [rc] retire 1139 clears [JobId(1138), JobId(1139)], : downstairs
Sep 22 23:15:50.739 ERRO [1] job id 1140 saw error GenericError("test error")
Sep 22 23:15:50.739 ERRO [1] job id 1140 saw error GenericError("test error")
Sep 22 23:15:50.745 INFO [lossy] sleeping 1 second
Sep 22 23:15:50.745 DEBG Flush     :1141 extent_limit None deps:[JobId(1140), JobId(1139)] res:true f:60 g:1
Sep 22 23:15:50.751 INFO [lossy] sleeping 1 second
Sep 22 23:15:50.753 INFO [lossy] skipping 1141
Sep 22 23:15:50.753 DEBG Flush     :1141 extent_limit None deps:[JobId(1140), JobId(1139)] res:true f:60 g:1
Sep 22 23:15:50.753 INFO [lossy] skipping 1142
Sep 22 23:15:50.753 WARN returning error on read!
Sep 22 23:15:50.753 DEBG Read      :1142 deps:[JobId(1141)] res:false
Sep 22 23:15:50.753 WARN returning error on read!
Sep 22 23:15:50.753 DEBG Read      :1142 deps:[JobId(1141)] res:false
Sep 22 23:15:50.759 DEBG Read      :1142 deps:[JobId(1141)] res:true
Sep 22 23:15:51.157 ERRO [1] job id 1142 saw error GenericError("test error")
Sep 22 23:15:51.158 ERRO [1] job id 1142 saw error GenericError("test error")
Sep 22 23:15:51.158 DEBG up_ds_listen was notified
Sep 22 23:15:51.158 DEBG up_ds_listen process 1141
Sep 22 23:15:51.158 DEBG [A] ack job 1141:142, : downstairs
Sep 22 23:15:51.158 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:51.535 DEBG [1] Read AckReady 1142, : downstairs
Sep 22 23:15:51.536 DEBG up_ds_listen was notified
Sep 22 23:15:51.536 DEBG up_ds_listen process 1142
Sep 22 23:15:51.536 DEBG [A] ack job 1142:143, : downstairs
Sep 22 23:15:51.588 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:51.589 DEBG IO Flush 1143 has deps [JobId(1142), JobId(1141)]
Sep 22 23:15:51.590 DEBG IO Read  1144 has deps [JobId(1143)]
Sep 22 23:15:51.595 DEBG Flush     :1143 extent_limit None deps:[JobId(1142), JobId(1141)] res:true f:61 g:1
Sep 22 23:15:51.608 DEBG Read      :1144 deps:[JobId(1143)] res:true
Sep 22 23:15:52.005 DEBG [1] Read AckReady 1144, : downstairs
Sep 22 23:15:52.006 DEBG up_ds_listen was notified
Sep 22 23:15:52.006 DEBG up_ds_listen process 1144
Sep 22 23:15:52.006 DEBG [A] ack job 1144:145, : downstairs
Sep 22 23:15:52.058 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:52.059 DEBG Flush     :1141 extent_limit None deps:[JobId(1140), JobId(1139)] res:true f:60 g:1
Sep 22 23:15:52.065 DEBG Read      :1142 deps:[JobId(1141)] res:true
Sep 22 23:15:52.092 DEBG Read      :1142 deps:[JobId(1141)] res:true
Sep 22 23:15:52.114 DEBG IO Flush 1145 has deps [JobId(1144), JobId(1143)]
Sep 22 23:15:52.114 DEBG IO Read  1146 has deps [JobId(1145)]
Sep 22 23:15:52.114 DEBG [rc] retire 1141 clears [JobId(1140), JobId(1141)], : downstairs
Sep 22 23:15:52.127 DEBG Flush     :1145 extent_limit None deps:[JobId(1144), JobId(1143)] res:true f:62 g:1
Sep 22 23:15:52.127 INFO [lossy] skipping 1146
Sep 22 23:15:52.127 INFO [lossy] skipping 1146
Sep 22 23:15:52.133 DEBG Read      :1146 deps:[JobId(1145)] res:true
Sep 22 23:15:52.157 DEBG Flush     :1143 extent_limit None deps:[JobId(1142), JobId(1141)] res:true f:61 g:1
Sep 22 23:15:52.163 DEBG Read      :1144 deps:[JobId(1143)] res:true
Sep 22 23:15:52.184 INFO [lossy] skipping 1143
Sep 22 23:15:52.184 INFO [lossy] skipping 1144
Sep 22 23:15:52.184 WARN returning error on flush!
Sep 22 23:15:52.184 DEBG Flush     :1143 extent_limit None deps:[JobId(1142), JobId(1141)] res:false f:61 g:1
Sep 22 23:15:52.184 DEBG Flush     :1143 extent_limit None deps:[JobId(1142), JobId(1141)] res:true f:61 g:1
Sep 22 23:15:52.190 DEBG Read      :1144 deps:[JobId(1143)] res:true
Sep 22 23:15:52.642 ERRO [0] job id 1143 saw error GenericError("test error")
Sep 22 23:15:52.972 DEBG up_ds_listen was notified
Sep 22 23:15:52.972 DEBG up_ds_listen process 1143
Sep 22 23:15:52.972 DEBG [A] ack job 1143:144, : downstairs
Sep 22 23:15:52.972 DEBG [rc] retire 1143 clears [JobId(1142), JobId(1143)], : downstairs
Sep 22 23:15:52.972 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:52.974 DEBG IO Flush 1147 has deps [JobId(1146), JobId(1145)]
Sep 22 23:15:53.350 DEBG [1] Read AckReady 1146, : downstairs
Sep 22 23:15:53.351 DEBG up_ds_listen was notified
Sep 22 23:15:53.351 DEBG up_ds_listen process 1146
Sep 22 23:15:53.351 DEBG [A] ack job 1146:147, : downstairs
Sep 22 23:15:53.403 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:53.403 INFO [lossy] sleeping 1 second
Sep 22 23:15:53.405 DEBG IO Read  1148 has deps [JobId(1147)]
Sep 22 23:15:53.419 INFO [lossy] sleeping 1 second
Sep 22 23:15:53.420 INFO [lossy] skipping 1145
Sep 22 23:15:53.420 INFO [lossy] skipping 1146
Sep 22 23:15:53.420 DEBG Flush     :1145 extent_limit None deps:[JobId(1144), JobId(1143)] res:true f:62 g:1
Sep 22 23:15:53.420 INFO [lossy] skipping 1146
Sep 22 23:15:53.420 INFO [lossy] skipping 1146
Sep 22 23:15:53.426 DEBG Read      :1146 deps:[JobId(1145)] res:true
Sep 22 23:15:54.203 DEBG up_ds_listen was notified
Sep 22 23:15:54.203 DEBG up_ds_listen process 1145
Sep 22 23:15:54.203 DEBG [A] ack job 1145:146, : downstairs
Sep 22 23:15:54.203 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:54.203 DEBG IO Flush 1149 has deps [JobId(1148), JobId(1147)]
Sep 22 23:15:54.206 INFO [lossy] skipping 1147
Sep 22 23:15:54.206 INFO [lossy] skipping 1147
Sep 22 23:15:54.206 DEBG Flush     :1147 extent_limit None deps:[JobId(1146), JobId(1145)] res:true f:63 g:1
Sep 22 23:15:54.211 DEBG Read      :1148 deps:[JobId(1147)] res:true
Sep 22 23:15:54.610 INFO [lossy] skipping 1147
Sep 22 23:15:54.610 DEBG Flush     :1147 extent_limit None deps:[JobId(1146), JobId(1145)] res:true f:63 g:1
Sep 22 23:15:54.610 INFO [lossy] sleeping 1 second
Sep 22 23:15:54.611 WARN returning error on flush!
Sep 22 23:15:54.611 DEBG Flush     :1145 extent_limit None deps:[JobId(1144), JobId(1143)] res:false f:62 g:1
Sep 22 23:15:54.611 INFO [lossy] skipping 1146
Sep 22 23:15:54.611 WARN returning error on flush!
Sep 22 23:15:54.611 DEBG Flush     :1145 extent_limit None deps:[JobId(1144), JobId(1143)] res:false f:62 g:1
Sep 22 23:15:54.611 DEBG Flush     :1145 extent_limit None deps:[JobId(1144), JobId(1143)] res:true f:62 g:1
Sep 22 23:15:54.616 DEBG Read      :1146 deps:[JobId(1145)] res:true
Sep 22 23:15:54.638 ERRO [2] job id 1145 saw error GenericError("test error")
Sep 22 23:15:54.638 ERRO [2] job id 1145 saw error GenericError("test error")
Sep 22 23:15:54.638 DEBG [rc] retire 1145 clears [JobId(1144), JobId(1145)], : downstairs
Sep 22 23:15:54.638 DEBG up_ds_listen was notified
Sep 22 23:15:54.638 DEBG up_ds_listen process 1147
Sep 22 23:15:54.638 DEBG [A] ack job 1147:148, : downstairs
Sep 22 23:15:54.638 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:54.640 DEBG Flush     :1149 extent_limit None deps:[JobId(1148), JobId(1147)] res:true f:64 g:1
Sep 22 23:15:54.640 INFO [lossy] sleeping 1 second
Sep 22 23:15:55.016 DEBG [0] Read AckReady 1148, : downstairs
Sep 22 23:15:55.016 DEBG up_ds_listen was notified
Sep 22 23:15:55.016 DEBG up_ds_listen process 1148
Sep 22 23:15:55.017 DEBG [A] ack job 1148:149, : downstairs
Sep 22 23:15:55.069 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:55.071 INFO [lossy] skipping 1147
Sep 22 23:15:55.071 DEBG Flush     :1147 extent_limit None deps:[JobId(1146), JobId(1145)] res:true f:63 g:1
Sep 22 23:15:55.071 INFO [lossy] sleeping 1 second
Sep 22 23:15:55.119 DEBG IO Read  1150 has deps [JobId(1149)]
Sep 22 23:15:55.449 DEBG [rc] retire 1147 clears [JobId(1146), JobId(1147)], : downstairs
Sep 22 23:15:55.572 DEBG IO Flush 1151 has deps [JobId(1150), JobId(1149)]
Sep 22 23:15:55.612 INFO [lossy] skipping 1148
Sep 22 23:15:55.612 WARN 1149 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:15:55.618 DEBG Read      :1148 deps:[JobId(1147)] res:true
Sep 22 23:15:55.640 WARN returning error on flush!
Sep 22 23:15:55.641 DEBG Flush     :1149 extent_limit None deps:[JobId(1148), JobId(1147)] res:false f:64 g:1
Sep 22 23:15:55.641 INFO [lossy] skipping 1151
Sep 22 23:15:55.641 DEBG Flush     :1149 extent_limit None deps:[JobId(1148), JobId(1147)] res:true f:64 g:1
Sep 22 23:15:55.641 INFO [lossy] skipping 1151
Sep 22 23:15:55.641 WARN 1151 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:15:55.641 INFO [lossy] skipping 1150
Sep 22 23:15:55.646 DEBG Read      :1150 deps:[JobId(1149)] res:true
Sep 22 23:15:55.715 WARN returning error on read!
Sep 22 23:15:55.715 DEBG Read      :1150 deps:[JobId(1149)] res:false
Sep 22 23:15:55.722 DEBG Read      :1150 deps:[JobId(1149)] res:true
Sep 22 23:15:56.073 ERRO [1] job id 1149 saw error GenericError("test error")
Sep 22 23:15:56.073 DEBG up_ds_listen was notified
Sep 22 23:15:56.073 DEBG up_ds_listen process 1149
Sep 22 23:15:56.073 DEBG [A] ack job 1149:150, : downstairs
Sep 22 23:15:56.073 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:56.074 INFO [lossy] skipping 1148
Sep 22 23:15:56.074 INFO [lossy] skipping 1149
Sep 22 23:15:56.074 INFO [lossy] skipping 1151
Sep 22 23:15:56.074 INFO [lossy] skipping 1148
Sep 22 23:15:56.074 WARN 1149 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:15:56.080 DEBG Read      :1148 deps:[JobId(1147)] res:true
Sep 22 23:15:56.101 ERRO [0] job id 1150 saw error GenericError("test error")
Sep 22 23:15:56.104 INFO [lossy] sleeping 1 second
Sep 22 23:15:56.152 INFO [lossy] skipping 1151
Sep 22 23:15:56.152 INFO [lossy] skipping 1151
Sep 22 23:15:56.152 INFO [lossy] skipping 1151
Sep 22 23:15:56.152 DEBG Flush     :1151 extent_limit None deps:[JobId(1150), JobId(1149)] res:true f:65 g:1
Sep 22 23:15:56.152 INFO [lossy] sleeping 1 second
Sep 22 23:15:56.480 DEBG [1] Read AckReady 1150, : downstairs
Sep 22 23:15:56.480 DEBG up_ds_listen was notified
Sep 22 23:15:56.480 DEBG up_ds_listen process 1150
Sep 22 23:15:56.481 DEBG [A] ack job 1150:151, : downstairs
Sep 22 23:15:56.533 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:56.582 INFO [lossy] skipping 1149
Sep 22 23:15:56.582 DEBG Flush     :1149 extent_limit None deps:[JobId(1148), JobId(1147)] res:true f:64 g:1
Sep 22 23:15:56.588 DEBG Read      :1150 deps:[JobId(1149)] res:true
Sep 22 23:15:56.987 DEBG IO Read  1152 has deps [JobId(1151)]
Sep 22 23:15:57.317 DEBG [rc] retire 1149 clears [JobId(1148), JobId(1149)], : downstairs
Sep 22 23:15:57.329 DEBG Flush     :1151 extent_limit None deps:[JobId(1150), JobId(1149)] res:true f:65 g:1
Sep 22 23:15:57.329 INFO [lossy] skipping 1152
Sep 22 23:15:57.329 WARN returning error on read!
Sep 22 23:15:57.329 DEBG Read      :1152 deps:[JobId(1151)] res:false
Sep 22 23:15:57.329 INFO [lossy] skipping 1152
Sep 22 23:15:57.329 INFO [lossy] skipping 1152
Sep 22 23:15:57.335 DEBG Read      :1152 deps:[JobId(1151)] res:true
Sep 22 23:15:57.356 WARN returning error on read!
Sep 22 23:15:57.356 DEBG Read      :1152 deps:[JobId(1151)] res:false
Sep 22 23:15:57.362 DEBG Read      :1152 deps:[JobId(1151)] res:true
Sep 22 23:15:57.384 ERRO [0] job id 1152 saw error GenericError("test error")
Sep 22 23:15:57.384 ERRO [1] job id 1152 saw error GenericError("test error")
Sep 22 23:15:57.384 DEBG up_ds_listen was notified
Sep 22 23:15:57.384 DEBG up_ds_listen process 1151
Sep 22 23:15:57.384 DEBG [A] ack job 1151:152, : downstairs
Sep 22 23:15:57.384 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:57.386 DEBG Flush     :1151 extent_limit None deps:[JobId(1150), JobId(1149)] res:true f:65 g:1
Sep 22 23:15:57.392 DEBG Read      :1152 deps:[JobId(1151)] res:true
Sep 22 23:15:57.792 DEBG [rc] retire 1151 clears [JobId(1150), JobId(1151)], : downstairs
Sep 22 23:15:57.795 DEBG IO Flush 1153 has deps [JobId(1152)]
Sep 22 23:15:58.218 DEBG [0] Read AckReady 1152, : downstairs
Sep 22 23:15:58.548 DEBG [1] Read already AckReady 1152, : downstairs
Sep 22 23:15:58.549 DEBG up_ds_listen was notified
Sep 22 23:15:58.549 DEBG up_ds_listen process 1152
Sep 22 23:15:58.549 DEBG [A] ack job 1152:153, : downstairs
Sep 22 23:15:58.602 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:58.602 INFO [lossy] skipping 1153
Sep 22 23:15:58.602 DEBG Flush     :1153 extent_limit None deps:[JobId(1152)] res:true f:66 g:1
Sep 22 23:15:58.602 INFO [lossy] skipping 1153
Sep 22 23:15:58.602 DEBG Flush     :1153 extent_limit None deps:[JobId(1152)] res:true f:66 g:1
Sep 22 23:15:58.604 INFO [lossy] sleeping 1 second
Sep 22 23:15:58.604 DEBG up_ds_listen was notified
Sep 22 23:15:58.604 DEBG up_ds_listen process 1153
Sep 22 23:15:58.604 DEBG [A] ack job 1153:154, : downstairs
Sep 22 23:15:58.604 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:58.604 DEBG IO Read  1154 has deps [JobId(1153)]
Sep 22 23:15:58.616 INFO [lossy] skipping 1154
Sep 22 23:15:58.616 WARN returning error on read!
Sep 22 23:15:58.616 DEBG Read      :1154 deps:[JobId(1153)] res:false
Sep 22 23:15:58.622 DEBG Read      :1154 deps:[JobId(1153)] res:true
Sep 22 23:15:58.643 ERRO [0] job id 1154 saw error GenericError("test error")
Sep 22 23:15:58.645 INFO [lossy] sleeping 1 second
Sep 22 23:15:59.398 DEBG [0] Read AckReady 1154, : downstairs
Sep 22 23:15:59.398 DEBG up_ds_listen was notified
Sep 22 23:15:59.399 DEBG up_ds_listen process 1154
Sep 22 23:15:59.399 DEBG [A] ack job 1154:155, : downstairs
Sep 22 23:15:59.451 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:15:59.452 DEBG IO Flush 1155 has deps [JobId(1154), JobId(1153)]
Sep 22 23:15:59.453 DEBG IO Read  1156 has deps [JobId(1155)]
Sep 22 23:15:59.458 DEBG Flush     :1155 extent_limit None deps:[JobId(1154), JobId(1153)] res:true f:67 g:1
Sep 22 23:15:59.465 INFO [lossy] sleeping 1 second
Sep 22 23:15:59.610 DEBG Read      :1154 deps:[JobId(1153)] res:true
Sep 22 23:15:59.633 INFO [lossy] skipping 1155
Sep 22 23:15:59.633 INFO [lossy] skipping 1156
Sep 22 23:15:59.633 DEBG Flush     :1155 extent_limit None deps:[JobId(1154), JobId(1153)] res:true f:67 g:1
Sep 22 23:15:59.638 DEBG Read      :1156 deps:[JobId(1155)] res:true
Sep 22 23:15:59.707 WARN returning error on flush!
Sep 22 23:15:59.707 DEBG Flush     :1153 extent_limit None deps:[JobId(1152)] res:false f:66 g:1
Sep 22 23:15:59.707 INFO [lossy] skipping 1156
Sep 22 23:15:59.707 DEBG Flush     :1153 extent_limit None deps:[JobId(1152)] res:true f:66 g:1
Sep 22 23:15:59.707 WARN returning error on read!
Sep 22 23:15:59.707 DEBG Read      :1154 deps:[JobId(1153)] res:false
Sep 22 23:15:59.707 WARN 1155 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:15:59.707 WARN returning error on read!
Sep 22 23:15:59.707 DEBG Read      :1154 deps:[JobId(1153)] res:false
Sep 22 23:15:59.714 DEBG Read      :1154 deps:[JobId(1153)] res:true
Sep 22 23:16:00.066 DEBG up_ds_listen was notified
Sep 22 23:16:00.066 DEBG up_ds_listen process 1155
Sep 22 23:16:00.066 DEBG [A] ack job 1155:156, : downstairs
Sep 22 23:16:00.066 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:00.066 DEBG IO Flush 1157 has deps [JobId(1156), JobId(1155)]
Sep 22 23:16:00.066 ERRO [2] job id 1153 saw error GenericError("test error")
Sep 22 23:16:00.066 DEBG [rc] retire 1153 clears [JobId(1152), JobId(1153)], : downstairs
Sep 22 23:16:00.066 ERRO [2] job id 1154 saw error GenericError("test error")
Sep 22 23:16:00.066 ERRO [2] job id 1154 saw error GenericError("test error")
Sep 22 23:16:00.069 INFO [lossy] skipping 1157
Sep 22 23:16:00.069 INFO [lossy] skipping 1157
Sep 22 23:16:00.070 DEBG Flush     :1157 extent_limit None deps:[JobId(1156), JobId(1155)] res:true f:68 g:1
Sep 22 23:16:00.118 DEBG Flush     :1155 extent_limit None deps:[JobId(1154), JobId(1153)] res:true f:67 g:1
Sep 22 23:16:00.118 WARN returning error on read!
Sep 22 23:16:00.118 DEBG Read      :1156 deps:[JobId(1155)] res:false
Sep 22 23:16:00.118 WARN returning error on read!
Sep 22 23:16:00.118 DEBG Read      :1156 deps:[JobId(1155)] res:false
Sep 22 23:16:00.124 DEBG Read      :1156 deps:[JobId(1155)] res:true
Sep 22 23:16:00.473 DEBG [1] Read AckReady 1156, : downstairs
Sep 22 23:16:00.473 DEBG up_ds_listen was notified
Sep 22 23:16:00.474 DEBG up_ds_listen process 1156
Sep 22 23:16:00.474 DEBG [A] ack job 1156:157, : downstairs
Sep 22 23:16:00.526 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:00.574 INFO [lossy] skipping 1156
Sep 22 23:16:00.580 DEBG Read      :1156 deps:[JobId(1155)] res:true
Sep 22 23:16:00.932 DEBG [rc] retire 1155 clears [JobId(1154), JobId(1155)], : downstairs
Sep 22 23:16:00.932 ERRO [2] job id 1156 saw error GenericError("test error")
Sep 22 23:16:00.932 ERRO [2] job id 1156 saw error GenericError("test error")
Sep 22 23:16:00.933 DEBG IO Read  1158 has deps [JobId(1157)]
Sep 22 23:16:00.933 DEBG IO Flush 1159 has deps [JobId(1158), JobId(1157)]
Sep 22 23:16:00.945 INFO [lossy] skipping 1158
Sep 22 23:16:00.945 INFO [lossy] skipping 1158
Sep 22 23:16:00.951 DEBG Read      :1158 deps:[JobId(1157)] res:true
Sep 22 23:16:00.975 INFO [lossy] sleeping 1 second
Sep 22 23:16:01.023 DEBG Flush     :1157 extent_limit None deps:[JobId(1156), JobId(1155)] res:true f:68 g:1
Sep 22 23:16:01.023 WARN returning error on read!
Sep 22 23:16:01.023 DEBG Read      :1158 deps:[JobId(1157)] res:false
Sep 22 23:16:01.023 INFO [lossy] skipping 1158
Sep 22 23:16:01.023 INFO [lossy] skipping 1158
Sep 22 23:16:01.023 INFO [lossy] skipping 1158
Sep 22 23:16:01.029 DEBG Read      :1158 deps:[JobId(1157)] res:true
Sep 22 23:16:01.756 ERRO [0] job id 1158 saw error GenericError("test error")
Sep 22 23:16:01.756 DEBG up_ds_listen was notified
Sep 22 23:16:01.756 DEBG up_ds_listen process 1157
Sep 22 23:16:01.756 DEBG [A] ack job 1157:158, : downstairs
Sep 22 23:16:01.756 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:01.758 DEBG Flush     :1159 extent_limit None deps:[JobId(1158), JobId(1157)] res:true f:69 g:1
Sep 22 23:16:02.135 DEBG [1] Read AckReady 1158, : downstairs
Sep 22 23:16:02.136 DEBG up_ds_listen was notified
Sep 22 23:16:02.136 DEBG up_ds_listen process 1158
Sep 22 23:16:02.136 DEBG [A] ack job 1158:159, : downstairs
Sep 22 23:16:02.189 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:02.190 DEBG Flush     :1157 extent_limit None deps:[JobId(1156), JobId(1155)] res:true f:68 g:1
Sep 22 23:16:02.190 INFO [lossy] skipping 1158
Sep 22 23:16:02.190 WARN returning error on read!
Sep 22 23:16:02.190 DEBG Read      :1158 deps:[JobId(1157)] res:false
Sep 22 23:16:02.196 DEBG Read      :1158 deps:[JobId(1157)] res:true
Sep 22 23:16:02.218 DEBG IO Read  1160 has deps [JobId(1159)]
Sep 22 23:16:02.219 DEBG [rc] retire 1157 clears [JobId(1156), JobId(1157)], : downstairs
Sep 22 23:16:02.219 ERRO [2] job id 1158 saw error GenericError("test error")
Sep 22 23:16:02.231 INFO [lossy] skipping 1160
Sep 22 23:16:02.237 DEBG Read      :1160 deps:[JobId(1159)] res:true
Sep 22 23:16:02.259 DEBG IO Flush 1161 has deps [JobId(1160), JobId(1159)]
Sep 22 23:16:02.261 DEBG Flush     :1159 extent_limit None deps:[JobId(1158), JobId(1157)] res:true f:69 g:1
Sep 22 23:16:02.267 DEBG Read      :1160 deps:[JobId(1159)] res:true
Sep 22 23:16:02.668 DEBG up_ds_listen was notified
Sep 22 23:16:02.668 DEBG up_ds_listen process 1159
Sep 22 23:16:02.668 DEBG [A] ack job 1159:160, : downstairs
Sep 22 23:16:02.668 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:02.670 INFO [lossy] sleeping 1 second
Sep 22 23:16:03.048 DEBG Flush     :1161 extent_limit None deps:[JobId(1160), JobId(1159)] res:true f:70 g:1
Sep 22 23:16:03.424 DEBG [1] Read AckReady 1160, : downstairs
Sep 22 23:16:03.425 DEBG up_ds_listen was notified
Sep 22 23:16:03.425 DEBG up_ds_listen process 1160
Sep 22 23:16:03.425 DEBG [A] ack job 1160:161, : downstairs
Sep 22 23:16:03.477 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:03.479 DEBG IO Read  1162 has deps [JobId(1161)]
Sep 22 23:16:03.497 DEBG Read      :1162 deps:[JobId(1161)] res:true
Sep 22 23:16:03.519 INFO [lossy] skipping 1161
Sep 22 23:16:03.519 DEBG Flush     :1161 extent_limit None deps:[JobId(1160), JobId(1159)] res:true f:70 g:1
Sep 22 23:16:03.519 WARN returning error on read!
Sep 22 23:16:03.519 DEBG Read      :1162 deps:[JobId(1161)] res:false
Sep 22 23:16:03.520 WARN returning error on read!
Sep 22 23:16:03.520 DEBG Read      :1162 deps:[JobId(1161)] res:false
Sep 22 23:16:03.520 INFO [lossy] skipping 1162
Sep 22 23:16:03.520 WARN returning error on read!
Sep 22 23:16:03.520 DEBG Read      :1162 deps:[JobId(1161)] res:false
Sep 22 23:16:03.525 DEBG Read      :1162 deps:[JobId(1161)] res:true
Sep 22 23:16:03.924 ERRO [0] job id 1162 saw error GenericError("test error")
Sep 22 23:16:03.924 ERRO [0] job id 1162 saw error GenericError("test error")
Sep 22 23:16:03.924 ERRO [0] job id 1162 saw error GenericError("test error")
Sep 22 23:16:03.924 DEBG up_ds_listen was notified
Sep 22 23:16:03.924 DEBG up_ds_listen process 1161
Sep 22 23:16:03.924 DEBG [A] ack job 1161:162, : downstairs
Sep 22 23:16:03.924 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:03.924 DEBG IO Flush 1163 has deps [JobId(1162), JobId(1161)]
Sep 22 23:16:03.924 WARN returning error on flush!
Sep 22 23:16:03.924 DEBG Flush     :1159 extent_limit None deps:[JobId(1158), JobId(1157)] res:false f:69 g:1
Sep 22 23:16:03.924 INFO [lossy] skipping 1160
Sep 22 23:16:03.924 INFO [lossy] skipping 1159
Sep 22 23:16:03.924 DEBG Flush     :1159 extent_limit None deps:[JobId(1158), JobId(1157)] res:true f:69 g:1
Sep 22 23:16:03.930 DEBG Read      :1160 deps:[JobId(1159)] res:true
Sep 22 23:16:03.952 ERRO [2] job id 1159 saw error GenericError("test error")
Sep 22 23:16:03.952 DEBG [rc] retire 1159 clears [JobId(1158), JobId(1159)], : downstairs
Sep 22 23:16:03.953 INFO [lossy] skipping 1163
Sep 22 23:16:03.953 WARN returning error on flush!
Sep 22 23:16:03.953 DEBG Flush     :1163 extent_limit None deps:[JobId(1162), JobId(1161)] res:false f:71 g:1
Sep 22 23:16:03.953 DEBG Flush     :1163 extent_limit None deps:[JobId(1162), JobId(1161)] res:true f:71 g:1
Sep 22 23:16:04.005 DEBG [1] Read already AckReady 1000, : downstairs
Sep 22 23:16:04.330 DEBG [1] Read AckReady 1162, : downstairs
Sep 22 23:16:04.330 ERRO [1] job id 1163 saw error GenericError("test error")
Sep 22 23:16:04.330 DEBG up_ds_listen was notified
Sep 22 23:16:04.330 DEBG up_ds_listen process 1162
Sep 22 23:16:04.331 DEBG [A] ack job 1162:163, : downstairs
Sep 22 23:16:04.383 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:04.385 DEBG IO Read  1164 has deps [JobId(1163)]
Sep 22 23:16:04.403 DEBG Read      :1164 deps:[JobId(1163)] res:true
Sep 22 23:16:04.425 DEBG Flush     :1163 extent_limit None deps:[JobId(1162), JobId(1161)] res:true f:71 g:1
Sep 22 23:16:04.426 INFO [lossy] skipping 1164
Sep 22 23:16:04.426 WARN returning error on read!
Sep 22 23:16:04.426 DEBG Read      :1164 deps:[JobId(1163)] res:false
Sep 22 23:16:04.426 WARN returning error on read!
Sep 22 23:16:04.426 DEBG Read      :1164 deps:[JobId(1163)] res:false
Sep 22 23:16:04.431 DEBG Read      :1164 deps:[JobId(1163)] res:true
Sep 22 23:16:04.453 DEBG IO Flush 1165 has deps [JobId(1164), JobId(1163)]
Sep 22 23:16:04.831 ERRO [0] job id 1164 saw error GenericError("test error")
Sep 22 23:16:04.831 ERRO [0] job id 1164 saw error GenericError("test error")
Sep 22 23:16:04.831 DEBG up_ds_listen was notified
Sep 22 23:16:04.831 DEBG up_ds_listen process 1163
Sep 22 23:16:04.831 DEBG [A] ack job 1163:164, : downstairs
Sep 22 23:16:04.831 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:04.833 DEBG Flush     :1161 extent_limit None deps:[JobId(1160), JobId(1159)] res:true f:70 g:1
Sep 22 23:16:04.839 DEBG Read      :1162 deps:[JobId(1161)] res:true
Sep 22 23:16:05.238 DEBG [rc] retire 1161 clears [JobId(1160), JobId(1161)], : downstairs
Sep 22 23:16:05.240 DEBG Flush     :1165 extent_limit None deps:[JobId(1164), JobId(1163)] res:true f:72 g:1
Sep 22 23:16:05.616 DEBG [1] Read AckReady 1164, : downstairs
Sep 22 23:16:05.617 DEBG up_ds_listen was notified
Sep 22 23:16:05.617 DEBG up_ds_listen process 1164
Sep 22 23:16:05.617 DEBG [A] ack job 1164:165, : downstairs
Sep 22 23:16:05.669 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:07.770 DEBG IO Write 1166 has deps [JobId(1165), JobId(1163)]
Sep 22 23:16:07.770 DEBG up_ds_listen was notified
Sep 22 23:16:07.770 DEBG up_ds_listen process 1166
Sep 22 23:16:07.770 DEBG [A] ack job 1166:167, : downstairs
Sep 22 23:16:07.770 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:07.834 DEBG Flush     :1165 extent_limit None deps:[JobId(1164), JobId(1163)] res:true f:72 g:1
Sep 22 23:16:07.834 INFO [lossy] sleeping 1 second
Sep 22 23:16:08.163 DEBG IO Write 1167 has deps [JobId(1165), JobId(1163)]
Sep 22 23:16:08.163 DEBG up_ds_listen was notified
Sep 22 23:16:08.163 DEBG up_ds_listen process 1167
Sep 22 23:16:08.163 DEBG [A] ack job 1167:168, : downstairs
Sep 22 23:16:08.163 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:08.164 DEBG IO Flush 1168 has deps [JobId(1167), JobId(1166), JobId(1165)]
Sep 22 23:16:08.493 DEBG IO Write 1169 has deps [JobId(1168), JobId(1165), JobId(1163)]
Sep 22 23:16:08.493 DEBG up_ds_listen was notified
Sep 22 23:16:08.493 DEBG up_ds_listen process 1169
Sep 22 23:16:08.493 DEBG [A] ack job 1169:170, : downstairs
Sep 22 23:16:08.493 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:08.822 DEBG IO Write 1170 has deps [JobId(1168), JobId(1165), JobId(1163)]
Sep 22 23:16:08.822 DEBG up_ds_listen was notified
Sep 22 23:16:08.822 DEBG up_ds_listen process 1170
Sep 22 23:16:08.823 DEBG [A] ack job 1170:171, : downstairs
Sep 22 23:16:08.823 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:08.823 DEBG IO Flush 1171 has deps [JobId(1170), JobId(1169), JobId(1168)]
Sep 22 23:16:09.152 DEBG IO Write 1172 has deps [JobId(1171), JobId(1168), JobId(1165), JobId(1163)]
Sep 22 23:16:09.152 DEBG up_ds_listen was notified
Sep 22 23:16:09.152 DEBG up_ds_listen process 1172
Sep 22 23:16:09.153 DEBG [A] ack job 1172:173, : downstairs
Sep 22 23:16:09.153 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:09.153 INFO [lossy] sleeping 1 second
Sep 22 23:16:09.482 DEBG IO Write 1173 has deps [JobId(1171), JobId(1168), JobId(1165), JobId(1163)]
Sep 22 23:16:09.482 DEBG up_ds_listen was notified
Sep 22 23:16:09.482 DEBG up_ds_listen process 1173
Sep 22 23:16:09.482 DEBG [A] ack job 1173:174, : downstairs
Sep 22 23:16:09.483 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:09.483 DEBG IO Flush 1174 has deps [JobId(1173), JobId(1172), JobId(1171)]
Sep 22 23:16:09.812 DEBG IO Write 1175 has deps [JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)]
Sep 22 23:16:09.812 DEBG up_ds_listen was notified
Sep 22 23:16:09.812 DEBG up_ds_listen process 1175
Sep 22 23:16:09.812 DEBG [A] ack job 1175:176, : downstairs
Sep 22 23:16:09.812 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:10.142 DEBG IO Write 1176 has deps [JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)]
Sep 22 23:16:10.142 DEBG up_ds_listen was notified
Sep 22 23:16:10.142 DEBG up_ds_listen process 1176
Sep 22 23:16:10.142 DEBG [A] ack job 1176:177, : downstairs
Sep 22 23:16:10.142 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:10.142 DEBG IO Flush 1177 has deps [JobId(1176), JobId(1175), JobId(1174)]
Sep 22 23:16:10.471 DEBG IO Write 1178 has deps [JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)]
Sep 22 23:16:10.472 DEBG up_ds_listen was notified
Sep 22 23:16:10.472 DEBG up_ds_listen process 1178
Sep 22 23:16:10.472 DEBG [A] ack job 1178:179, : downstairs
Sep 22 23:16:10.472 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:10.911 DEBG IO Write 1179 has deps [JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)]
Sep 22 23:16:11.241 DEBG up_ds_listen was notified
Sep 22 23:16:11.241 DEBG up_ds_listen process 1165
Sep 22 23:16:11.241 DEBG [A] ack job 1165:166, : downstairs
Sep 22 23:16:11.242 DEBG up_ds_listen process 1179
Sep 22 23:16:11.242 DEBG [A] ack job 1179:180, : downstairs
Sep 22 23:16:11.242 DEBG up_ds_listen checked 2 jobs, back to waiting
Sep 22 23:16:11.242 DEBG up_ds_listen was notified
Sep 22 23:16:11.242 DEBG up_ds_listen checked 0 jobs, back to waiting
Sep 22 23:16:11.289 WARN returning error on flush!
Sep 22 23:16:11.289 DEBG Flush     :1163 extent_limit None deps:[JobId(1162), JobId(1161)] res:false f:71 g:1
Sep 22 23:16:11.289 WARN returning error on flush!
Sep 22 23:16:11.289 DEBG Flush     :1163 extent_limit None deps:[JobId(1162), JobId(1161)] res:false f:71 g:1
Sep 22 23:16:11.289 DEBG Flush     :1163 extent_limit None deps:[JobId(1162), JobId(1161)] res:true f:71 g:1
Sep 22 23:16:11.290 WARN returning error on read!
Sep 22 23:16:11.290 DEBG Read      :1164 deps:[JobId(1163)] res:false
Sep 22 23:16:11.290 WARN 1165 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:11.295 DEBG Read      :1164 deps:[JobId(1163)] res:true
Sep 22 23:16:11.409 DEBG IO Flush 1180 has deps [JobId(1179), JobId(1178), JobId(1177)]
Sep 22 23:16:11.409 INFO [lossy] sleeping 1 second
Sep 22 23:16:11.411 WARN returning error on write!
Sep 22 23:16:11.411 DEBG Write     :1166 deps:[JobId(1165), JobId(1163)] res:false
Sep 22 23:16:11.440 DEBG Write     :1166 deps:[JobId(1165), JobId(1163)] res:true
Sep 22 23:16:11.770 DEBG IO Write 1181 has deps [JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)]
Sep 22 23:16:11.770 DEBG up_ds_listen was notified
Sep 22 23:16:11.770 DEBG up_ds_listen process 1181
Sep 22 23:16:11.770 DEBG [A] ack job 1181:182, : downstairs
Sep 22 23:16:11.770 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:12.099 DEBG IO Write 1182 has deps [JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)]
Sep 22 23:16:12.100 DEBG up_ds_listen was notified
Sep 22 23:16:12.100 DEBG up_ds_listen process 1182
Sep 22 23:16:12.100 DEBG [A] ack job 1182:183, : downstairs
Sep 22 23:16:12.100 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:12.100 DEBG IO Flush 1183 has deps [JobId(1182), JobId(1181), JobId(1180)]
Sep 22 23:16:12.430 DEBG IO Write 1184 has deps [JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)]
Sep 22 23:16:12.430 DEBG up_ds_listen was notified
Sep 22 23:16:12.430 DEBG up_ds_listen process 1184
Sep 22 23:16:12.430 DEBG [A] ack job 1184:185, : downstairs
Sep 22 23:16:12.430 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:12.432 WARN returning error on write!
Sep 22 23:16:12.432 DEBG Write     :1166 deps:[JobId(1165), JobId(1163)] res:false
Sep 22 23:16:12.433 WARN returning error on write!
Sep 22 23:16:12.433 DEBG Write     :1166 deps:[JobId(1165), JobId(1163)] res:false
Sep 22 23:16:12.433 INFO [lossy] skipping 1166
Sep 22 23:16:12.433 INFO [lossy] skipping 1166
Sep 22 23:16:12.463 DEBG Write     :1166 deps:[JobId(1165), JobId(1163)] res:true
Sep 22 23:16:12.793 DEBG IO Write 1185 has deps [JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)]
Sep 22 23:16:12.793 DEBG up_ds_listen was notified
Sep 22 23:16:12.793 DEBG up_ds_listen process 1185
Sep 22 23:16:12.793 DEBG [A] ack job 1185:186, : downstairs
Sep 22 23:16:12.793 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:12.793 DEBG IO Flush 1186 has deps [JobId(1185), JobId(1184), JobId(1183)]
Sep 22 23:16:13.123 DEBG IO Write 1187 has deps [JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)]
Sep 22 23:16:13.123 DEBG up_ds_listen was notified
Sep 22 23:16:13.123 DEBG up_ds_listen process 1187
Sep 22 23:16:13.123 DEBG [A] ack job 1187:188, : downstairs
Sep 22 23:16:13.123 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:13.453 DEBG IO Write 1188 has deps [JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)]
Sep 22 23:16:13.453 DEBG up_ds_listen was notified
Sep 22 23:16:13.453 DEBG up_ds_listen process 1188
Sep 22 23:16:13.453 DEBG [A] ack job 1188:189, : downstairs
Sep 22 23:16:13.453 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:13.454 DEBG IO Flush 1189 has deps [JobId(1188), JobId(1187), JobId(1186)]
Sep 22 23:16:13.783 DEBG IO Write 1190 has deps [JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)]
Sep 22 23:16:13.783 DEBG up_ds_listen was notified
Sep 22 23:16:13.783 DEBG up_ds_listen process 1190
Sep 22 23:16:13.783 DEBG [A] ack job 1190:191, : downstairs
Sep 22 23:16:13.783 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:14.113 DEBG IO Write 1191 has deps [JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)]
Sep 22 23:16:14.113 DEBG up_ds_listen was notified
Sep 22 23:16:14.113 DEBG up_ds_listen process 1191
Sep 22 23:16:14.113 DEBG [A] ack job 1191:192, : downstairs
Sep 22 23:16:14.113 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:14.114 DEBG IO Flush 1192 has deps [JobId(1191), JobId(1190), JobId(1189)]
Sep 22 23:16:14.505 DEBG IO Write 1193 has deps [JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)]
Sep 22 23:16:14.505 DEBG up_ds_listen was notified
Sep 22 23:16:14.505 DEBG up_ds_listen process 1193
Sep 22 23:16:14.505 DEBG [A] ack job 1193:194, : downstairs
Sep 22 23:16:14.505 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:14.645 INFO [lossy] sleeping 1 second
Sep 22 23:16:14.646 WARN returning error on write!
Sep 22 23:16:14.646 DEBG Write     :1167 deps:[JobId(1165), JobId(1163)] res:false
Sep 22 23:16:14.677 DEBG Write     :1167 deps:[JobId(1165), JobId(1163)] res:true
Sep 22 23:16:14.678 INFO [lossy] sleeping 1 second
Sep 22 23:16:15.007 DEBG IO Write 1194 has deps [JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)]
Sep 22 23:16:15.007 DEBG IO Flush 1195 has deps [JobId(1194), JobId(1193), JobId(1192)]
Sep 22 23:16:15.007 DEBG up_ds_listen was notified
Sep 22 23:16:15.007 DEBG up_ds_listen process 1194
Sep 22 23:16:15.007 DEBG [A] ack job 1194:195, : downstairs
Sep 22 23:16:15.007 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:15.337 DEBG IO Write 1196 has deps [JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)]
Sep 22 23:16:15.337 DEBG up_ds_listen was notified
Sep 22 23:16:15.337 DEBG up_ds_listen process 1196
Sep 22 23:16:15.337 DEBG [A] ack job 1196:197, : downstairs
Sep 22 23:16:15.337 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:15.667 DEBG IO Write 1197 has deps [JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)]
Sep 22 23:16:15.667 DEBG up_ds_listen was notified
Sep 22 23:16:15.667 DEBG up_ds_listen process 1197
Sep 22 23:16:15.667 DEBG [A] ack job 1197:198, : downstairs
Sep 22 23:16:15.667 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:15.668 DEBG IO Flush 1198 has deps [JobId(1197), JobId(1196), JobId(1195)]
Sep 22 23:16:15.668 INFO [lossy] skipping 1167
Sep 22 23:16:15.668 INFO [lossy] skipping 1168
Sep 22 23:16:15.698 DEBG Write     :1167 deps:[JobId(1165), JobId(1163)] res:true
Sep 22 23:16:15.707 DEBG Flush     :1168 extent_limit None deps:[JobId(1167), JobId(1166), JobId(1165)] res:true f:73 g:1
Sep 22 23:16:15.707 INFO [lossy] sleeping 1 second
Sep 22 23:16:16.035 DEBG IO Write 1199 has deps [JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)]
Sep 22 23:16:16.044 DEBG up_ds_listen was notified
Sep 22 23:16:16.044 DEBG up_ds_listen process 1199
Sep 22 23:16:16.044 DEBG [A] ack job 1199:200, : downstairs
Sep 22 23:16:16.044 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:16.044 DEBG Flush     :1168 extent_limit None deps:[JobId(1167), JobId(1166), JobId(1165)] res:true f:73 g:1
Sep 22 23:16:16.373 DEBG IO Write 1200 has deps [JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)]
Sep 22 23:16:16.373 DEBG up_ds_listen was notified
Sep 22 23:16:16.373 DEBG up_ds_listen process 1200
Sep 22 23:16:16.374 DEBG [A] ack job 1200:201, : downstairs
Sep 22 23:16:16.374 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:16.374 DEBG IO Flush 1201 has deps [JobId(1200), JobId(1199), JobId(1198)]
Sep 22 23:16:16.703 DEBG IO Write 1202 has deps [JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)]
Sep 22 23:16:16.703 DEBG up_ds_listen was notified
Sep 22 23:16:16.703 DEBG up_ds_listen process 1202
Sep 22 23:16:16.703 DEBG [A] ack job 1202:203, : downstairs
Sep 22 23:16:16.703 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:17.033 DEBG IO Write 1203 has deps [JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)]
Sep 22 23:16:17.033 DEBG up_ds_listen was notified
Sep 22 23:16:17.033 DEBG up_ds_listen process 1203
Sep 22 23:16:17.033 DEBG [A] ack job 1203:204, : downstairs
Sep 22 23:16:17.033 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:17.034 INFO [lossy] sleeping 1 second
Sep 22 23:16:17.034 DEBG IO Flush 1204 has deps [JobId(1203), JobId(1202), JobId(1201)]
Sep 22 23:16:17.363 DEBG IO Write 1205 has deps [JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)]
Sep 22 23:16:17.363 DEBG up_ds_listen was notified
Sep 22 23:16:17.363 DEBG up_ds_listen process 1205
Sep 22 23:16:17.363 DEBG [A] ack job 1205:206, : downstairs
Sep 22 23:16:17.363 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:17.755 DEBG IO Write 1206 has deps [JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)]
Sep 22 23:16:17.755 DEBG up_ds_listen was notified
Sep 22 23:16:17.755 DEBG up_ds_listen process 1206
Sep 22 23:16:17.755 DEBG [A] ack job 1206:207, : downstairs
Sep 22 23:16:17.755 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:17.896 DEBG IO Flush 1207 has deps [JobId(1206), JobId(1205), JobId(1204)]
Sep 22 23:16:17.896 INFO [lossy] skipping 1169
Sep 22 23:16:17.926 DEBG Write     :1169 deps:[JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:18.256 DEBG IO Write 1208 has deps [JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)]
Sep 22 23:16:18.257 DEBG up_ds_listen was notified
Sep 22 23:16:18.257 DEBG up_ds_listen process 1208
Sep 22 23:16:18.257 DEBG [A] ack job 1208:209, : downstairs
Sep 22 23:16:18.257 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:18.257 INFO [lossy] skipping 1169
Sep 22 23:16:18.257 INFO [lossy] skipping 1169
Sep 22 23:16:18.287 DEBG Write     :1169 deps:[JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:18.288 INFO [lossy] sleeping 1 second
Sep 22 23:16:18.617 DEBG IO Write 1209 has deps [JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)]
Sep 22 23:16:18.617 DEBG up_ds_listen was notified
Sep 22 23:16:18.617 DEBG up_ds_listen process 1209
Sep 22 23:16:18.617 DEBG [A] ack job 1209:210, : downstairs
Sep 22 23:16:18.617 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:18.618 DEBG IO Flush 1210 has deps [JobId(1209), JobId(1208), JobId(1207)]
Sep 22 23:16:18.947 DEBG IO Write 1211 has deps [JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)]
Sep 22 23:16:18.947 DEBG up_ds_listen was notified
Sep 22 23:16:18.947 DEBG up_ds_listen process 1211
Sep 22 23:16:18.947 DEBG [A] ack job 1211:212, : downstairs
Sep 22 23:16:18.947 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:19.277 DEBG IO Write 1212 has deps [JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)]
Sep 22 23:16:19.277 DEBG up_ds_listen was notified
Sep 22 23:16:19.277 DEBG up_ds_listen process 1212
Sep 22 23:16:19.277 DEBG [A] ack job 1212:213, : downstairs
Sep 22 23:16:19.277 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:19.278 DEBG IO Flush 1213 has deps [JobId(1212), JobId(1211), JobId(1210)]
Sep 22 23:16:19.553 DEBG [2] Read already AckReady 1000, : downstairs
Sep 22 23:16:19.607 DEBG IO Write 1214 has deps [JobId(1213), JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)]
Sep 22 23:16:19.608 DEBG up_ds_listen was notified
Sep 22 23:16:19.608 DEBG up_ds_listen process 1214
Sep 22 23:16:19.608 DEBG [A] ack job 1214:215, : downstairs
Sep 22 23:16:19.608 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:19.663 DEBG up_ds_listen was notified
Sep 22 23:16:19.663 DEBG up_ds_listen process 1000
Sep 22 23:16:19.666 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:16:19.938 DEBG IO Write 1215 has deps [JobId(1213), JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)]
Sep 22 23:16:19.938 DEBG up_ds_listen was notified
Sep 22 23:16:19.938 DEBG up_ds_listen process 1215
Sep 22 23:16:19.938 DEBG [A] ack job 1215:216, : downstairs
Sep 22 23:16:19.938 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:19.939 DEBG IO Flush 1216 has deps [JobId(1215), JobId(1214), JobId(1213)]
Sep 22 23:16:19.939 INFO [lossy] sleeping 1 second
Sep 22 23:16:20.268 DEBG IO Write 1217 has deps [JobId(1216), JobId(1213), JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)]
Sep 22 23:16:20.268 DEBG up_ds_listen was notified
Sep 22 23:16:20.268 DEBG up_ds_listen process 1217
Sep 22 23:16:20.268 DEBG [A] ack job 1217:218, : downstairs
Sep 22 23:16:20.268 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:20.598 DEBG IO Write 1218 has deps [JobId(1216), JobId(1213), JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)]
Sep 22 23:16:20.598 DEBG up_ds_listen was notified
Sep 22 23:16:20.598 DEBG up_ds_listen process 1218
Sep 22 23:16:20.599 DEBG [A] ack job 1218:219, : downstairs
Sep 22 23:16:20.599 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:20.599 DEBG IO Flush 1219 has deps [JobId(1218), JobId(1217), JobId(1216)]
Sep 22 23:16:20.937 DEBG IO Write 1220 has deps [JobId(1219), JobId(1216), JobId(1213), JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)]
Sep 22 23:16:20.937 DEBG up_ds_listen was notified
Sep 22 23:16:20.937 DEBG up_ds_listen process 1220
Sep 22 23:16:20.937 DEBG [A] ack job 1220:221, : downstairs
Sep 22 23:16:20.937 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:21.330 DEBG IO Write 1221 has deps [JobId(1219), JobId(1216), JobId(1213), JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)]
Sep 22 23:16:21.330 DEBG up_ds_listen was notified
Sep 22 23:16:21.330 DEBG up_ds_listen process 1221
Sep 22 23:16:21.330 DEBG [A] ack job 1221:222, : downstairs
Sep 22 23:16:21.330 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:21.469 DEBG IO Flush 1222 has deps [JobId(1221), JobId(1220), JobId(1219)]
Sep 22 23:16:21.471 WARN returning error on write!
Sep 22 23:16:21.471 DEBG Write     :1170 deps:[JobId(1168), JobId(1165), JobId(1163)] res:false
Sep 22 23:16:21.501 DEBG Write     :1170 deps:[JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:21.502 INFO [lossy] sleeping 1 second
Sep 22 23:16:21.510 DEBG Flush     :1171 extent_limit None deps:[JobId(1170), JobId(1169), JobId(1168)] res:true f:74 g:1
Sep 22 23:16:21.839 DEBG IO Write 1223 has deps [JobId(1222), JobId(1219), JobId(1216), JobId(1213), JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)]
Sep 22 23:16:21.839 DEBG up_ds_listen was notified
Sep 22 23:16:21.839 DEBG up_ds_listen process 1223
Sep 22 23:16:21.839 DEBG [A] ack job 1223:224, : downstairs
Sep 22 23:16:21.839 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:22.168 DEBG IO Write 1224 has deps [JobId(1222), JobId(1219), JobId(1216), JobId(1213), JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)]
Sep 22 23:16:22.169 DEBG up_ds_listen was notified
Sep 22 23:16:22.169 DEBG up_ds_listen process 1224
Sep 22 23:16:22.169 DEBG [A] ack job 1224:225, : downstairs
Sep 22 23:16:22.169 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:22.169 DEBG IO Flush 1225 has deps [JobId(1224), JobId(1223), JobId(1222)]
Sep 22 23:16:22.178 DEBG up_ds_listen process 1001
Sep 22 23:16:22.178 DEBG [A] ack job 1001:2, : downstairs
Sep 22 23:16:22.178 DEBG [rc] retire 1001 clears [JobId(1000), JobId(1001)], : downstairs
Sep 22 23:16:22.178 DEBG up_ds_listen checked 2 jobs, back to waiting
Sep 22 23:16:22.178 DEBG up_ds_listen was notified
Sep 22 23:16:22.178 DEBG up_ds_listen checked 0 jobs, back to waiting
test test::integration_test_volume_replace_downstairs_then_takeover ... ok
Sep 22 23:16:22.511 DEBG IO Write 1226 has deps [JobId(1225), JobId(1222), JobId(1219), JobId(1216), JobId(1213), JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)]
Sep 22 23:16:22.511 DEBG up_ds_listen was notified
Sep 22 23:16:22.512 DEBG up_ds_listen process 1226
Sep 22 23:16:22.512 DEBG [A] ack job 1226:227, : downstairs
Sep 22 23:16:22.512 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:22.544 DEBG Write     :1170 deps:[JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:22.553 DEBG Flush     :1171 extent_limit None deps:[JobId(1170), JobId(1169), JobId(1168)] res:true f:74 g:1
Sep 22 23:16:22.553 INFO [lossy] sleeping 1 second
Sep 22 23:16:22.882 DEBG IO Write 1227 has deps [JobId(1225), JobId(1222), JobId(1219), JobId(1216), JobId(1213), JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)]
Sep 22 23:16:22.882 DEBG up_ds_listen was notified
Sep 22 23:16:22.882 DEBG up_ds_listen process 1227
Sep 22 23:16:22.882 DEBG [A] ack job 1227:228, : downstairs
Sep 22 23:16:22.882 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:22.883 DEBG IO Flush 1228 has deps [JobId(1227), JobId(1226), JobId(1225)]
Sep 22 23:16:23.212 DEBG IO Write 1229 has deps [JobId(1228), JobId(1225), JobId(1222), JobId(1219), JobId(1216), JobId(1213), JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)]
Sep 22 23:16:23.212 DEBG up_ds_listen was notified
Sep 22 23:16:23.212 DEBG up_ds_listen process 1229
Sep 22 23:16:23.212 DEBG [A] ack job 1229:230, : downstairs
Sep 22 23:16:23.213 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:23.543 DEBG IO Write 1230 has deps [JobId(1228), JobId(1225), JobId(1222), JobId(1219), JobId(1216), JobId(1213), JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)]
Sep 22 23:16:23.543 DEBG up_ds_listen was notified
Sep 22 23:16:23.543 DEBG up_ds_listen process 1230
Sep 22 23:16:23.543 DEBG [A] ack job 1230:231, : downstairs
Sep 22 23:16:23.543 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:23.543 DEBG IO Flush 1231 has deps [JobId(1230), JobId(1229), JobId(1228)]
Sep 22 23:16:23.873 DEBG IO Write 1232 has deps [JobId(1231), JobId(1228), JobId(1225), JobId(1222), JobId(1219), JobId(1216), JobId(1213), JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)]
Sep 22 23:16:23.873 DEBG up_ds_listen was notified
Sep 22 23:16:23.873 DEBG up_ds_listen process 1232
Sep 22 23:16:23.873 DEBG [A] ack job 1232:233, : downstairs
Sep 22 23:16:23.873 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:24.203 DEBG IO Write 1233 has deps [JobId(1231), JobId(1228), JobId(1225), JobId(1222), JobId(1219), JobId(1216), JobId(1213), JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163), JobId(1162)]
Sep 22 23:16:24.203 DEBG up_ds_listen was notified
Sep 22 23:16:24.203 DEBG up_ds_listen process 1233
Sep 22 23:16:24.203 DEBG [A] ack job 1233:234, : downstairs
Sep 22 23:16:24.203 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:24.204 DEBG IO Flush 1234 has deps [JobId(1233), JobId(1232), JobId(1231)]
Sep 22 23:16:24.595 DEBG IO Write 1235 has deps [JobId(1234), JobId(1231), JobId(1228), JobId(1225), JobId(1222), JobId(1219), JobId(1216), JobId(1213), JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1164), JobId(1163)]
Sep 22 23:16:24.595 DEBG up_ds_listen was notified
Sep 22 23:16:24.595 DEBG up_ds_listen process 1235
Sep 22 23:16:24.595 DEBG [A] ack job 1235:236, : downstairs
Sep 22 23:16:24.595 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:24.735 INFO [lossy] sleeping 1 second
Sep 22 23:16:24.766 DEBG Write     :1172 deps:[JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:24.767 DEBG IO Flush 1236 has deps [JobId(1235), JobId(1234)]
Sep 22 23:16:24.767 DEBG IO Read  1237 has deps [JobId(1236)]
Sep 22 23:16:24.970 INFO [lossy] sleeping 1 second
Sep 22 23:16:25.377 DEBG IO Flush 1238 has deps [JobId(1237), JobId(1236)]
Sep 22 23:16:25.490 ERRO [0] job id 1166 saw error GenericError("test error")
Sep 22 23:16:25.819 ERRO [2] job id 1163 saw error GenericError("test error")
Sep 22 23:16:25.819 ERRO [2] job id 1163 saw error GenericError("test error")
Sep 22 23:16:25.819 DEBG [rc] retire 1163 clears [JobId(1162), JobId(1163)], : downstairs
Sep 22 23:16:25.819 ERRO [2] job id 1164 saw error GenericError("test error")
Sep 22 23:16:25.819 ERRO [1] job id 1166 saw error GenericError("test error")
Sep 22 23:16:25.819 ERRO [1] job id 1166 saw error GenericError("test error")
Sep 22 23:16:25.819 ERRO [1] job id 1167 saw error GenericError("test error")
Sep 22 23:16:25.819 ERRO [1] job id 1170 saw error GenericError("test error")
Sep 22 23:16:25.820 DEBG up_ds_listen was notified
Sep 22 23:16:25.820 DEBG up_ds_listen process 1168
Sep 22 23:16:25.820 DEBG [A] ack job 1168:169, : downstairs
Sep 22 23:16:25.820 DEBG up_ds_listen process 1171
Sep 22 23:16:25.820 DEBG [A] ack job 1171:172, : downstairs
Sep 22 23:16:25.820 DEBG up_ds_listen checked 2 jobs, back to waiting
Sep 22 23:16:25.820 DEBG up_ds_listen was notified
Sep 22 23:16:25.820 DEBG up_ds_listen checked 0 jobs, back to waiting
Sep 22 23:16:25.958 INFO [lossy] skipping 1172
Sep 22 23:16:25.989 DEBG Write     :1173 deps:[JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:26.020 DEBG Write     :1172 deps:[JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:26.022 INFO [lossy] skipping 1174
Sep 22 23:16:26.022 INFO [lossy] skipping 1176
Sep 22 23:16:26.022 WARN returning error on flush!
Sep 22 23:16:26.022 DEBG Flush     :1174 extent_limit None deps:[JobId(1173), JobId(1172), JobId(1171)] res:false f:75 g:1
Sep 22 23:16:26.022 WARN returning error on flush!
Sep 22 23:16:26.022 DEBG Flush     :1174 extent_limit None deps:[JobId(1173), JobId(1172), JobId(1171)] res:false f:75 g:1
Sep 22 23:16:26.030 DEBG Flush     :1174 extent_limit None deps:[JobId(1173), JobId(1172), JobId(1171)] res:true f:75 g:1
Sep 22 23:16:26.060 DEBG Write     :1175 deps:[JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:26.092 DEBG Write     :1176 deps:[JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:26.093 WARN returning error on flush!
Sep 22 23:16:26.093 DEBG Flush     :1177 extent_limit None deps:[JobId(1176), JobId(1175), JobId(1174)] res:false f:76 g:1
Sep 22 23:16:26.101 DEBG Flush     :1177 extent_limit None deps:[JobId(1176), JobId(1175), JobId(1174)] res:true f:76 g:1
Sep 22 23:16:26.101 INFO [lossy] sleeping 1 second
Sep 22 23:16:26.102 WARN returning error on write!
Sep 22 23:16:26.102 DEBG Write     :1173 deps:[JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:false
Sep 22 23:16:26.103 INFO [lossy] skipping 1174
Sep 22 23:16:26.103 INFO [lossy] skipping 1173
Sep 22 23:16:26.103 WARN returning error on write!
Sep 22 23:16:26.103 DEBG Write     :1173 deps:[JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:false
Sep 22 23:16:26.104 INFO [lossy] skipping 1173
Sep 22 23:16:26.134 DEBG Write     :1173 deps:[JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:26.143 DEBG Flush     :1174 extent_limit None deps:[JobId(1173), JobId(1172), JobId(1171)] res:true f:75 g:1
Sep 22 23:16:26.173 DEBG Write     :1175 deps:[JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:26.176 WARN returning error on write!
Sep 22 23:16:26.176 DEBG Write     :1176 deps:[JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:false
Sep 22 23:16:26.176 WARN 1177 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:26.176 INFO [lossy] skipping 1178
Sep 22 23:16:26.177 WARN returning error on write!
Sep 22 23:16:26.177 DEBG Write     :1176 deps:[JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:false
Sep 22 23:16:26.177 WARN 1178 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:26.177 INFO [lossy] skipping 1176
Sep 22 23:16:26.207 DEBG Write     :1176 deps:[JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:26.208 WARN returning error on flush!
Sep 22 23:16:26.208 DEBG Flush     :1177 extent_limit None deps:[JobId(1176), JobId(1175), JobId(1174)] res:false f:76 g:1
Sep 22 23:16:26.216 DEBG Flush     :1177 extent_limit None deps:[JobId(1176), JobId(1175), JobId(1174)] res:true f:76 g:1
Sep 22 23:16:26.216 INFO [lossy] sleeping 1 second
Sep 22 23:16:27.235 WARN returning error on write!
Sep 22 23:16:27.235 DEBG Write     :1178 deps:[JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:false
Sep 22 23:16:27.266 DEBG Write     :1179 deps:[JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:27.267 INFO [lossy] skipping 1180
Sep 22 23:16:27.267 INFO [lossy] skipping 1185
Sep 22 23:16:27.297 DEBG Write     :1178 deps:[JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:27.298 INFO [lossy] skipping 1180
Sep 22 23:16:27.299 INFO [lossy] skipping 1178
Sep 22 23:16:27.299 INFO [lossy] skipping 1179
Sep 22 23:16:27.299 INFO [lossy] skipping 1180
Sep 22 23:16:27.299 INFO [lossy] skipping 1182
Sep 22 23:16:27.299 INFO [lossy] skipping 1183
Sep 22 23:16:27.299 INFO [lossy] skipping 1184
Sep 22 23:16:27.299 INFO [lossy] skipping 1186
Sep 22 23:16:27.329 DEBG Write     :1178 deps:[JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:27.360 DEBG Write     :1179 deps:[JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:27.377 DEBG Flush     :1180 extent_limit None deps:[JobId(1179), JobId(1178), JobId(1177)] res:true f:77 g:1
Sep 22 23:16:27.407 DEBG Write     :1181 deps:[JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:27.439 DEBG Write     :1182 deps:[JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:27.440 WARN returning error on flush!
Sep 22 23:16:27.440 DEBG Flush     :1183 extent_limit None deps:[JobId(1182), JobId(1181), JobId(1180)] res:false f:78 g:1
Sep 22 23:16:27.440 WARN 1184 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:27.440 WARN 1185 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:27.440 INFO [lossy] skipping 1186
Sep 22 23:16:27.440 DEBG Flush     :1180 extent_limit None deps:[JobId(1179), JobId(1178), JobId(1177)] res:true f:77 g:1
Sep 22 23:16:27.470 DEBG Write     :1182 deps:[JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:27.472 INFO [lossy] skipping 1186
Sep 22 23:16:27.502 DEBG Write     :1181 deps:[JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:27.503 INFO [lossy] skipping 1183
Sep 22 23:16:27.503 INFO [lossy] skipping 1184
Sep 22 23:16:27.503 INFO [lossy] skipping 1185
Sep 22 23:16:27.519 DEBG Flush     :1183 extent_limit None deps:[JobId(1182), JobId(1181), JobId(1180)] res:true f:78 g:1
Sep 22 23:16:27.519 WARN 1186 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:16:27.549 DEBG Write     :1184 deps:[JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:27.580 DEBG Write     :1185 deps:[JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:27.582 DEBG Flush     :1183 extent_limit None deps:[JobId(1182), JobId(1181), JobId(1180)] res:true f:78 g:1
Sep 22 23:16:27.612 DEBG Write     :1184 deps:[JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:27.643 DEBG Write     :1185 deps:[JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:27.645 INFO [lossy] skipping 1186
Sep 22 23:16:27.660 DEBG Flush     :1186 extent_limit None deps:[JobId(1185), JobId(1184), JobId(1183)] res:true f:79 g:1
Sep 22 23:16:27.660 INFO [lossy] sleeping 1 second
Sep 22 23:16:27.660 DEBG Flush     :1186 extent_limit None deps:[JobId(1185), JobId(1184), JobId(1183)] res:true f:79 g:1
Sep 22 23:16:27.660 INFO [lossy] sleeping 1 second
Sep 22 23:16:28.678 WARN returning error on write!
Sep 22 23:16:28.678 DEBG Write     :1187 deps:[JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:false
Sep 22 23:16:28.680 WARN returning error on write!
Sep 22 23:16:28.680 DEBG Write     :1188 deps:[JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:false
Sep 22 23:16:28.680 INFO [lossy] skipping 1190
Sep 22 23:16:28.680 INFO [lossy] skipping 1193
Sep 22 23:16:28.710 DEBG Write     :1187 deps:[JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:28.712 WARN returning error on write!
Sep 22 23:16:28.712 DEBG Write     :1188 deps:[JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:false
Sep 22 23:16:28.712 INFO [lossy] skipping 1190
Sep 22 23:16:28.712 INFO [lossy] skipping 1188
Sep 22 23:16:28.712 INFO [lossy] skipping 1190
Sep 22 23:16:28.742 DEBG Write     :1188 deps:[JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:28.743 INFO [lossy] sleeping 1 second
Sep 22 23:16:28.743 INFO [lossy] skipping 1187
Sep 22 23:16:28.743 INFO [lossy] skipping 1188
Sep 22 23:16:28.743 INFO [lossy] skipping 1189
Sep 22 23:16:28.743 INFO [lossy] skipping 1191
Sep 22 23:16:28.773 DEBG Write     :1187 deps:[JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:28.775 WARN returning error on write!
Sep 22 23:16:28.775 DEBG Write     :1188 deps:[JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:false
Sep 22 23:16:28.806 DEBG Write     :1188 deps:[JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:28.815 DEBG Flush     :1189 extent_limit None deps:[JobId(1188), JobId(1187), JobId(1186)] res:true f:80 g:1
Sep 22 23:16:28.816 WARN returning error on write!
Sep 22 23:16:28.816 DEBG Write     :1190 deps:[JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:false
Sep 22 23:16:28.846 DEBG Write     :1191 deps:[JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:28.847 INFO [lossy] skipping 1192
Sep 22 23:16:28.847 WARN 1193 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:28.848 WARN returning error on write!
Sep 22 23:16:28.848 DEBG Write     :1190 deps:[JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:false
Sep 22 23:16:28.848 WARN 1192 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:28.878 DEBG Write     :1190 deps:[JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:28.887 DEBG Flush     :1192 extent_limit None deps:[JobId(1191), JobId(1190), JobId(1189)] res:true f:81 g:1
Sep 22 23:16:28.917 DEBG Write     :1193 deps:[JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:28.918 INFO [lossy] sleeping 1 second
Sep 22 23:16:29.941 DEBG Flush     :1189 extent_limit None deps:[JobId(1188), JobId(1187), JobId(1186)] res:true f:80 g:1
Sep 22 23:16:29.942 WARN returning error on write!
Sep 22 23:16:29.942 DEBG Write     :1190 deps:[JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:false
Sep 22 23:16:29.942 INFO [lossy] skipping 1191
Sep 22 23:16:29.942 WARN 1192 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:16:29.942 INFO [lossy] skipping 1193
Sep 22 23:16:29.972 DEBG Write     :1190 deps:[JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:29.973 INFO [lossy] skipping 1191
Sep 22 23:16:29.973 WARN 1193 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:30.003 DEBG Write     :1191 deps:[JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:30.012 DEBG Flush     :1192 extent_limit None deps:[JobId(1191), JobId(1190), JobId(1189)] res:true f:81 g:1
Sep 22 23:16:30.042 DEBG Write     :1193 deps:[JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:30.074 DEBG Write     :1194 deps:[JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:30.075 WARN returning error on flush!
Sep 22 23:16:30.075 DEBG Flush     :1195 extent_limit None deps:[JobId(1194), JobId(1193), JobId(1192)] res:false f:82 g:1
Sep 22 23:16:30.075 INFO [lossy] skipping 1196
Sep 22 23:16:30.075 WARN 1197 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:30.075 INFO [lossy] skipping 1199
Sep 22 23:16:30.075 INFO [lossy] skipping 1195
Sep 22 23:16:30.075 INFO [lossy] skipping 1196
Sep 22 23:16:30.075 WARN 1199 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:16:30.083 DEBG Flush     :1195 extent_limit None deps:[JobId(1194), JobId(1193), JobId(1192)] res:true f:82 g:1
Sep 22 23:16:30.113 DEBG Write     :1196 deps:[JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:30.144 DEBG Write     :1197 deps:[JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:30.153 DEBG Flush     :1198 extent_limit None deps:[JobId(1197), JobId(1196), JobId(1195)] res:true f:83 g:1
Sep 22 23:16:30.183 DEBG Write     :1199 deps:[JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:30.185 INFO [lossy] sleeping 1 second
Sep 22 23:16:30.215 DEBG Write     :1194 deps:[JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:30.224 DEBG Flush     :1195 extent_limit None deps:[JobId(1194), JobId(1193), JobId(1192)] res:true f:82 g:1
Sep 22 23:16:30.254 DEBG Write     :1196 deps:[JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:30.286 DEBG Write     :1197 deps:[JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:30.295 DEBG Flush     :1198 extent_limit None deps:[JobId(1197), JobId(1196), JobId(1195)] res:true f:83 g:1
Sep 22 23:16:30.296 WARN returning error on write!
Sep 22 23:16:30.296 DEBG Write     :1199 deps:[JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:false
Sep 22 23:16:30.326 DEBG Write     :1200 deps:[JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:30.327 INFO [lossy] skipping 1199
Sep 22 23:16:30.327 INFO [lossy] skipping 1199
Sep 22 23:16:30.357 DEBG Write     :1199 deps:[JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:30.366 DEBG Flush     :1201 extent_limit None deps:[JobId(1200), JobId(1199), JobId(1198)] res:true f:84 g:1
Sep 22 23:16:30.366 INFO [lossy] sleeping 1 second
Sep 22 23:16:31.385 WARN returning error on write!
Sep 22 23:16:31.385 DEBG Write     :1200 deps:[JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:false
Sep 22 23:16:31.385 INFO [lossy] skipping 1201
Sep 22 23:16:31.385 INFO [lossy] skipping 1207
Sep 22 23:16:31.415 DEBG Write     :1200 deps:[JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:31.416 WARN returning error on flush!
Sep 22 23:16:31.416 DEBG Flush     :1201 extent_limit None deps:[JobId(1200), JobId(1199), JobId(1198)] res:false f:84 g:1
Sep 22 23:16:31.424 DEBG Flush     :1201 extent_limit None deps:[JobId(1200), JobId(1199), JobId(1198)] res:true f:84 g:1
Sep 22 23:16:31.424 INFO [lossy] sleeping 1 second
Sep 22 23:16:31.425 INFO [lossy] skipping 1202
Sep 22 23:16:31.425 WARN returning error on write!
Sep 22 23:16:31.425 DEBG Write     :1203 deps:[JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:false
Sep 22 23:16:31.426 INFO [lossy] skipping 1205
Sep 22 23:16:31.426 INFO [lossy] skipping 1208
Sep 22 23:16:31.456 DEBG Write     :1202 deps:[JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:31.487 DEBG Write     :1203 deps:[JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:31.496 DEBG Flush     :1204 extent_limit None deps:[JobId(1203), JobId(1202), JobId(1201)] res:true f:85 g:1
Sep 22 23:16:31.526 DEBG Write     :1205 deps:[JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:31.528 WARN returning error on write!
Sep 22 23:16:31.528 DEBG Write     :1206 deps:[JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:false
Sep 22 23:16:31.528 WARN 1207 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:31.528 WARN 1208 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:31.558 DEBG Write     :1206 deps:[JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:31.567 DEBG Flush     :1207 extent_limit None deps:[JobId(1206), JobId(1205), JobId(1204)] res:true f:86 g:1
Sep 22 23:16:31.597 DEBG Write     :1208 deps:[JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:31.598 INFO [lossy] sleeping 1 second
Sep 22 23:16:32.616 INFO [lossy] skipping 1202
Sep 22 23:16:32.647 DEBG Write     :1203 deps:[JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:32.648 INFO [lossy] skipping 1204
Sep 22 23:16:32.648 WARN 1205 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:32.648 WARN 1206 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:32.649 WARN returning error on write!
Sep 22 23:16:32.649 DEBG Write     :1202 deps:[JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:false
Sep 22 23:16:32.649 WARN 1204 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:32.679 DEBG Write     :1202 deps:[JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:32.680 INFO [lossy] skipping 1204
Sep 22 23:16:32.680 INFO [lossy] skipping 1206
Sep 22 23:16:32.680 INFO [lossy] skipping 1208
Sep 22 23:16:32.688 DEBG Flush     :1204 extent_limit None deps:[JobId(1203), JobId(1202), JobId(1201)] res:true f:85 g:1
Sep 22 23:16:32.719 DEBG Write     :1206 deps:[JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:32.720 WARN 1208 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:32.720 INFO [lossy] skipping 1205
Sep 22 23:16:32.720 WARN 1207 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:32.720 WARN 1209 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:32.720 WARN 1211 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:16:32.720 WARN 1212 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:16:32.720 WARN 1214 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:16:32.750 DEBG Write     :1205 deps:[JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:32.759 DEBG Flush     :1207 extent_limit None deps:[JobId(1206), JobId(1205), JobId(1204)] res:true f:86 g:1
Sep 22 23:16:32.759 INFO [lossy] skipping 1208
Sep 22 23:16:32.760 WARN returning error on write!
Sep 22 23:16:32.760 DEBG Write     :1209 deps:[JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:false
Sep 22 23:16:32.761 WARN 1210 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:16:32.761 INFO [lossy] skipping 1211
Sep 22 23:16:32.761 WARN 1212 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:32.761 WARN 1214 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:16:32.762 WARN returning error on write!
Sep 22 23:16:32.762 DEBG Write     :1208 deps:[JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:false
Sep 22 23:16:32.792 DEBG Write     :1209 deps:[JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:32.793 INFO [lossy] skipping 1211
Sep 22 23:16:32.823 DEBG Write     :1208 deps:[JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:32.825 WARN 1211 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:32.825 WARN returning error on flush!
Sep 22 23:16:32.825 DEBG Flush     :1210 extent_limit None deps:[JobId(1209), JobId(1208), JobId(1207)] res:false f:87 g:1
Sep 22 23:16:32.825 INFO [lossy] skipping 1212
Sep 22 23:16:32.825 INFO [lossy] skipping 1213
Sep 22 23:16:32.833 DEBG Flush     :1210 extent_limit None deps:[JobId(1209), JobId(1208), JobId(1207)] res:true f:87 g:1
Sep 22 23:16:32.833 INFO [lossy] skipping 1212
Sep 22 23:16:32.833 WARN 1213 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:16:32.833 INFO [lossy] skipping 1212
Sep 22 23:16:32.833 INFO [lossy] skipping 1212
Sep 22 23:16:32.834 WARN returning error on write!
Sep 22 23:16:32.834 DEBG Write     :1212 deps:[JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:false
Sep 22 23:16:32.834 INFO [lossy] skipping 1212
Sep 22 23:16:32.864 DEBG Write     :1212 deps:[JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:32.865 INFO [lossy] sleeping 1 second
Sep 22 23:16:32.896 DEBG Write     :1209 deps:[JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:32.897 WARN returning error on flush!
Sep 22 23:16:32.897 DEBG Flush     :1210 extent_limit None deps:[JobId(1209), JobId(1208), JobId(1207)] res:false f:87 g:1
Sep 22 23:16:32.897 INFO [lossy] skipping 1211
Sep 22 23:16:32.897 INFO [lossy] skipping 1214
Sep 22 23:16:32.897 INFO [lossy] skipping 1210
Sep 22 23:16:32.897 INFO [lossy] skipping 1214
Sep 22 23:16:32.905 DEBG Flush     :1210 extent_limit None deps:[JobId(1209), JobId(1208), JobId(1207)] res:true f:87 g:1
Sep 22 23:16:32.905 INFO [lossy] skipping 1211
Sep 22 23:16:32.936 DEBG Write     :1212 deps:[JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:32.937 WARN 1213 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:32.937 INFO [lossy] skipping 1214
Sep 22 23:16:32.937 INFO [lossy] skipping 1215
Sep 22 23:16:32.967 DEBG Write     :1211 deps:[JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:32.969 WARN 1215 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:32.969 WARN returning error on flush!
Sep 22 23:16:32.969 DEBG Flush     :1213 extent_limit None deps:[JobId(1212), JobId(1211), JobId(1210)] res:false f:88 g:1
Sep 22 23:16:32.969 INFO [lossy] skipping 1215
Sep 22 23:16:32.977 DEBG Flush     :1213 extent_limit None deps:[JobId(1212), JobId(1211), JobId(1210)] res:true f:88 g:1
Sep 22 23:16:32.978 WARN returning error on write!
Sep 22 23:16:32.978 DEBG Write     :1215 deps:[JobId(1213), JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:false
Sep 22 23:16:33.008 DEBG Write     :1215 deps:[JobId(1213), JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:33.040 DEBG Write     :1214 deps:[JobId(1213), JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:33.041 WARN returning error on flush!
Sep 22 23:16:33.041 DEBG Flush     :1216 extent_limit None deps:[JobId(1215), JobId(1214), JobId(1213)] res:false f:89 g:1
Sep 22 23:16:33.049 DEBG Flush     :1216 extent_limit None deps:[JobId(1215), JobId(1214), JobId(1213)] res:true f:89 g:1
Sep 22 23:16:33.049 INFO [lossy] sleeping 1 second
Sep 22 23:16:34.094 DEBG Write     :1211 deps:[JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:34.095 INFO [lossy] skipping 1213
Sep 22 23:16:34.095 WARN 1214 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:34.095 INFO [lossy] skipping 1213
Sep 22 23:16:34.103 DEBG Flush     :1213 extent_limit None deps:[JobId(1212), JobId(1211), JobId(1210)] res:true f:88 g:1
Sep 22 23:16:34.104 WARN returning error on write!
Sep 22 23:16:34.104 DEBG Write     :1214 deps:[JobId(1213), JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:false
Sep 22 23:16:34.106 WARN returning error on write!
Sep 22 23:16:34.106 DEBG Write     :1215 deps:[JobId(1213), JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:false
Sep 22 23:16:34.106 WARN 1216 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:16:34.106 WARN 1217 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:34.106 WARN 1218 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:34.106 WARN 1220 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:16:34.106 WARN 1221 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:16:34.106 INFO [lossy] skipping 1222
Sep 22 23:16:34.136 DEBG Write     :1214 deps:[JobId(1213), JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:34.167 DEBG Write     :1215 deps:[JobId(1213), JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:34.176 DEBG Flush     :1216 extent_limit None deps:[JobId(1215), JobId(1214), JobId(1213)] res:true f:89 g:1
Sep 22 23:16:34.177 WARN returning error on write!
Sep 22 23:16:34.177 DEBG Write     :1217 deps:[JobId(1216), JobId(1213), JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:false
Sep 22 23:16:34.207 DEBG Write     :1218 deps:[JobId(1216), JobId(1213), JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:34.208 INFO [lossy] skipping 1219
Sep 22 23:16:34.208 WARN 1220 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:34.209 WARN 1221 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:34.238 DEBG Write     :1217 deps:[JobId(1216), JobId(1213), JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:34.240 INFO [lossy] skipping 1219
Sep 22 23:16:34.240 INFO [lossy] skipping 1219
Sep 22 23:16:34.240 INFO [lossy] skipping 1219
Sep 22 23:16:34.248 DEBG Flush     :1219 extent_limit None deps:[JobId(1218), JobId(1217), JobId(1216)] res:true f:90 g:1
Sep 22 23:16:34.248 INFO [lossy] skipping 1220
Sep 22 23:16:34.248 INFO [lossy] skipping 1221
Sep 22 23:16:34.248 WARN 1222 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:16:34.249 WARN returning error on write!
Sep 22 23:16:34.249 DEBG Write     :1220 deps:[JobId(1219), JobId(1216), JobId(1213), JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:false
Sep 22 23:16:34.249 INFO [lossy] skipping 1221
Sep 22 23:16:34.279 DEBG Write     :1220 deps:[JobId(1219), JobId(1216), JobId(1213), JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:34.281 INFO [lossy] skipping 1221
Sep 22 23:16:34.311 DEBG Write     :1221 deps:[JobId(1219), JobId(1216), JobId(1213), JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:34.320 DEBG Flush     :1222 extent_limit None deps:[JobId(1221), JobId(1220), JobId(1219)] res:true f:91 g:1
Sep 22 23:16:34.320 INFO [lossy] sleeping 1 second
Sep 22 23:16:34.321 WARN returning error on write!
Sep 22 23:16:34.321 DEBG Write     :1217 deps:[JobId(1216), JobId(1213), JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:false
Sep 22 23:16:34.321 INFO [lossy] skipping 1218
Sep 22 23:16:34.321 INFO [lossy] skipping 1220
Sep 22 23:16:34.321 INFO [lossy] skipping 1222
Sep 22 23:16:34.321 INFO [lossy] skipping 1217
Sep 22 23:16:34.352 DEBG Write     :1218 deps:[JobId(1216), JobId(1213), JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:34.383 DEBG Write     :1217 deps:[JobId(1216), JobId(1213), JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:34.384 INFO [lossy] skipping 1219
Sep 22 23:16:34.384 INFO [lossy] skipping 1223
Sep 22 23:16:34.384 INFO [lossy] skipping 1219
Sep 22 23:16:34.384 WARN returning error on flush!
Sep 22 23:16:34.384 DEBG Flush     :1219 extent_limit None deps:[JobId(1218), JobId(1217), JobId(1216)] res:false f:90 g:1
Sep 22 23:16:34.384 INFO [lossy] skipping 1219
Sep 22 23:16:34.384 INFO [lossy] skipping 1219
Sep 22 23:16:34.392 DEBG Flush     :1219 extent_limit None deps:[JobId(1218), JobId(1217), JobId(1216)] res:true f:90 g:1
Sep 22 23:16:34.422 DEBG Write     :1220 deps:[JobId(1219), JobId(1216), JobId(1213), JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:34.424 WARN returning error on write!
Sep 22 23:16:34.424 DEBG Write     :1221 deps:[JobId(1219), JobId(1216), JobId(1213), JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:false
Sep 22 23:16:34.424 WARN 1222 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:34.424 WARN 1223 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:34.454 DEBG Write     :1221 deps:[JobId(1219), JobId(1216), JobId(1213), JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:34.455 INFO [lossy] sleeping 1 second
Sep 22 23:16:35.507 DEBG Write     :1223 deps:[JobId(1222), JobId(1219), JobId(1216), JobId(1213), JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:35.538 DEBG Write     :1224 deps:[JobId(1222), JobId(1219), JobId(1216), JobId(1213), JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:35.539 WARN returning error on flush!
Sep 22 23:16:35.539 DEBG Flush     :1225 extent_limit None deps:[JobId(1224), JobId(1223), JobId(1222)] res:false f:92 g:1
Sep 22 23:16:35.539 INFO [lossy] skipping 1227
Sep 22 23:16:35.539 INFO [lossy] skipping 1228
Sep 22 23:16:35.547 DEBG Flush     :1225 extent_limit None deps:[JobId(1224), JobId(1223), JobId(1222)] res:true f:92 g:1
Sep 22 23:16:35.578 DEBG Write     :1227 deps:[JobId(1225), JobId(1222), JobId(1219), JobId(1216), JobId(1213), JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:35.609 DEBG Write     :1226 deps:[JobId(1225), JobId(1222), JobId(1219), JobId(1216), JobId(1213), JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:35.618 DEBG Flush     :1228 extent_limit None deps:[JobId(1227), JobId(1226), JobId(1225)] res:true f:93 g:1
Sep 22 23:16:35.648 DEBG Write     :1229 deps:[JobId(1228), JobId(1225), JobId(1222), JobId(1219), JobId(1216), JobId(1213), JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:35.649 INFO [lossy] sleeping 1 second
Sep 22 23:16:35.658 DEBG Flush     :1222 extent_limit None deps:[JobId(1221), JobId(1220), JobId(1219)] res:true f:91 g:1
Sep 22 23:16:35.688 DEBG Write     :1223 deps:[JobId(1222), JobId(1219), JobId(1216), JobId(1213), JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:35.689 INFO [lossy] skipping 1224
Sep 22 23:16:35.689 INFO [lossy] skipping 1228
Sep 22 23:16:35.689 INFO [lossy] skipping 1229
Sep 22 23:16:35.720 DEBG Write     :1224 deps:[JobId(1222), JobId(1219), JobId(1216), JobId(1213), JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:35.729 DEBG Flush     :1225 extent_limit None deps:[JobId(1224), JobId(1223), JobId(1222)] res:true f:92 g:1
Sep 22 23:16:35.759 DEBG Write     :1226 deps:[JobId(1225), JobId(1222), JobId(1219), JobId(1216), JobId(1213), JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:35.791 DEBG Write     :1227 deps:[JobId(1225), JobId(1222), JobId(1219), JobId(1216), JobId(1213), JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:35.792 INFO [lossy] skipping 1228
Sep 22 23:16:35.792 WARN 1229 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:35.792 INFO [lossy] skipping 1230
Sep 22 23:16:35.792 INFO [lossy] skipping 1231
Sep 22 23:16:35.792 INFO [lossy] skipping 1228
Sep 22 23:16:35.792 INFO [lossy] skipping 1230
Sep 22 23:16:35.792 INFO [lossy] skipping 1231
Sep 22 23:16:35.792 INFO [lossy] skipping 1228
Sep 22 23:16:35.792 WARN 1230 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:35.792 INFO [lossy] skipping 1231
Sep 22 23:16:35.800 DEBG Flush     :1228 extent_limit None deps:[JobId(1227), JobId(1226), JobId(1225)] res:true f:93 g:1
Sep 22 23:16:35.800 INFO [lossy] skipping 1231
Sep 22 23:16:35.800 INFO [lossy] skipping 1231
Sep 22 23:16:35.800 INFO [lossy] skipping 1231
Sep 22 23:16:35.800 INFO [lossy] skipping 1231
Sep 22 23:16:35.800 WARN 1231 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:16:35.801 WARN returning error on write!
Sep 22 23:16:35.801 DEBG Write     :1229 deps:[JobId(1228), JobId(1225), JobId(1222), JobId(1219), JobId(1216), JobId(1213), JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:false
Sep 22 23:16:35.802 WARN returning error on write!
Sep 22 23:16:35.802 DEBG Write     :1230 deps:[JobId(1228), JobId(1225), JobId(1222), JobId(1219), JobId(1216), JobId(1213), JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:false
Sep 22 23:16:35.833 DEBG Write     :1229 deps:[JobId(1228), JobId(1225), JobId(1222), JobId(1219), JobId(1216), JobId(1213), JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:35.834 INFO [lossy] skipping 1230
Sep 22 23:16:35.864 DEBG Write     :1230 deps:[JobId(1228), JobId(1225), JobId(1222), JobId(1219), JobId(1216), JobId(1213), JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:35.865 INFO [lossy] skipping 1231
Sep 22 23:16:35.865 INFO [lossy] skipping 1231
Sep 22 23:16:35.865 WARN returning error on flush!
Sep 22 23:16:35.865 DEBG Flush     :1231 extent_limit None deps:[JobId(1230), JobId(1229), JobId(1228)] res:false f:94 g:1
Sep 22 23:16:35.865 INFO [lossy] skipping 1231
Sep 22 23:16:35.865 WARN returning error on flush!
Sep 22 23:16:35.865 DEBG Flush     :1231 extent_limit None deps:[JobId(1230), JobId(1229), JobId(1228)] res:false f:94 g:1
Sep 22 23:16:35.865 WARN returning error on flush!
Sep 22 23:16:35.865 DEBG Flush     :1231 extent_limit None deps:[JobId(1230), JobId(1229), JobId(1228)] res:false f:94 g:1
Sep 22 23:16:35.865 INFO [lossy] skipping 1231
Sep 22 23:16:35.873 DEBG Flush     :1231 extent_limit None deps:[JobId(1230), JobId(1229), JobId(1228)] res:true f:94 g:1
Sep 22 23:16:35.873 INFO [lossy] sleeping 1 second
Sep 22 23:16:36.428 ERRO [0] job id 1173 saw error GenericError("test error")
Sep 22 23:16:36.428 ERRO [0] job id 1173 saw error GenericError("test error")
Sep 22 23:16:36.428 ERRO [0] job id 1176 saw error GenericError("test error")
Sep 22 23:16:36.428 ERRO [0] job id 1176 saw error GenericError("test error")
Sep 22 23:16:36.428 ERRO [0] job id 1177 saw error GenericError("test error")
Sep 22 23:16:36.428 ERRO [0] job id 1188 saw error GenericError("test error")
Sep 22 23:16:36.428 ERRO [0] job id 1190 saw error GenericError("test error")
Sep 22 23:16:36.428 ERRO [0] job id 1190 saw error GenericError("test error")
Sep 22 23:16:36.429 ERRO [1] job id 1174 saw error GenericError("test error")
Sep 22 23:16:36.429 ERRO [1] job id 1174 saw error GenericError("test error")
Sep 22 23:16:36.429 ERRO [1] job id 1177 saw error GenericError("test error")
Sep 22 23:16:36.429 ERRO [1] job id 1178 saw error GenericError("test error")
Sep 22 23:16:36.429 ERRO [1] job id 1183 saw error GenericError("test error")
Sep 22 23:16:36.429 ERRO [1] job id 1187 saw error GenericError("test error")
Sep 22 23:16:36.429 ERRO [1] job id 1188 saw error GenericError("test error")
Sep 22 23:16:36.429 ERRO [1] job id 1188 saw error GenericError("test error")
Sep 22 23:16:36.430 ERRO [0] job id 1199 saw error GenericError("test error")
Sep 22 23:16:36.430 ERRO [0] job id 1203 saw error GenericError("test error")
Sep 22 23:16:36.430 ERRO [0] job id 1206 saw error GenericError("test error")
Sep 22 23:16:36.430 ERRO [0] job id 1210 saw error GenericError("test error")
Sep 22 23:16:36.430 DEBG up_ds_listen was notified
Sep 22 23:16:36.430 DEBG up_ds_listen process 1174
Sep 22 23:16:36.430 DEBG [A] ack job 1174:175, : downstairs
Sep 22 23:16:36.430 DEBG up_ds_listen process 1177
Sep 22 23:16:36.430 DEBG [A] ack job 1177:178, : downstairs
Sep 22 23:16:36.430 DEBG up_ds_listen process 1180
Sep 22 23:16:36.430 DEBG [A] ack job 1180:181, : downstairs
Sep 22 23:16:36.430 DEBG up_ds_listen process 1183
Sep 22 23:16:36.430 DEBG [A] ack job 1183:184, : downstairs
Sep 22 23:16:36.430 DEBG up_ds_listen process 1186
Sep 22 23:16:36.430 DEBG [A] ack job 1186:187, : downstairs
Sep 22 23:16:36.430 DEBG up_ds_listen checked 5 jobs, back to waiting
Sep 22 23:16:36.430 DEBG up_ds_listen was notified
Sep 22 23:16:36.430 DEBG up_ds_listen checked 0 jobs, back to waiting
Sep 22 23:16:36.430 DEBG up_ds_listen was notified
Sep 22 23:16:36.430 DEBG up_ds_listen checked 0 jobs, back to waiting
Sep 22 23:16:36.430 DEBG up_ds_listen was notified
Sep 22 23:16:36.430 DEBG up_ds_listen checked 0 jobs, back to waiting
Sep 22 23:16:36.430 DEBG up_ds_listen was notified
Sep 22 23:16:36.430 DEBG up_ds_listen checked 0 jobs, back to waiting
Sep 22 23:16:36.430 ERRO [1] job id 1190 saw error GenericError("test error")
Sep 22 23:16:36.430 ERRO [1] job id 1195 saw error GenericError("test error")
Sep 22 23:16:36.430 ERRO [1] job id 1200 saw error GenericError("test error")
Sep 22 23:16:36.430 ERRO [1] job id 1201 saw error GenericError("test error")
Sep 22 23:16:36.431 ERRO [1] job id 1202 saw error GenericError("test error")
Sep 22 23:16:36.431 ERRO [0] job id 1213 saw error GenericError("test error")
Sep 22 23:16:36.431 ERRO [0] job id 1215 saw error GenericError("test error")
Sep 22 23:16:36.431 ERRO [0] job id 1216 saw error GenericError("test error")
Sep 22 23:16:36.431 ERRO [0] job id 1217 saw error GenericError("test error")
Sep 22 23:16:36.431 ERRO [0] job id 1219 saw error GenericError("test error")
Sep 22 23:16:36.431 ERRO [0] job id 1221 saw error GenericError("test error")
Sep 22 23:16:36.431 ERRO [0] job id 1229 saw error GenericError("test error")
Sep 22 23:16:36.431 ERRO [0] job id 1230 saw error GenericError("test error")
Sep 22 23:16:36.431 DEBG up_ds_listen was notified
Sep 22 23:16:36.431 DEBG up_ds_listen process 1189
Sep 22 23:16:36.431 DEBG [A] ack job 1189:190, : downstairs
Sep 22 23:16:36.431 DEBG up_ds_listen process 1192
Sep 22 23:16:36.431 DEBG [A] ack job 1192:193, : downstairs
Sep 22 23:16:36.431 DEBG up_ds_listen process 1195
Sep 22 23:16:36.431 DEBG [A] ack job 1195:196, : downstairs
Sep 22 23:16:36.431 DEBG up_ds_listen process 1198
Sep 22 23:16:36.431 DEBG [A] ack job 1198:199, : downstairs
Sep 22 23:16:36.431 DEBG up_ds_listen process 1201
Sep 22 23:16:36.431 DEBG [A] ack job 1201:202, : downstairs
Sep 22 23:16:36.431 DEBG up_ds_listen process 1204
Sep 22 23:16:36.431 DEBG [A] ack job 1204:205, : downstairs
Sep 22 23:16:36.431 DEBG up_ds_listen process 1207
Sep 22 23:16:36.431 DEBG [A] ack job 1207:208, : downstairs
Sep 22 23:16:36.431 DEBG up_ds_listen checked 7 jobs, back to waiting
Sep 22 23:16:36.431 DEBG up_ds_listen was notified
Sep 22 23:16:36.431 DEBG up_ds_listen checked 0 jobs, back to waiting
Sep 22 23:16:36.431 DEBG up_ds_listen was notified
Sep 22 23:16:36.431 DEBG up_ds_listen checked 0 jobs, back to waiting
Sep 22 23:16:36.431 DEBG up_ds_listen was notified
Sep 22 23:16:36.431 DEBG up_ds_listen checked 0 jobs, back to waiting
Sep 22 23:16:36.431 DEBG up_ds_listen was notified
Sep 22 23:16:36.431 DEBG up_ds_listen checked 0 jobs, back to waiting
Sep 22 23:16:36.431 DEBG up_ds_listen was notified
Sep 22 23:16:36.431 DEBG up_ds_listen checked 0 jobs, back to waiting
Sep 22 23:16:36.431 DEBG up_ds_listen was notified
Sep 22 23:16:36.431 DEBG up_ds_listen checked 0 jobs, back to waiting
Sep 22 23:16:36.431 ERRO [1] job id 1209 saw error GenericError("test error")
Sep 22 23:16:36.431 ERRO [1] job id 1208 saw error GenericError("test error")
Sep 22 23:16:36.431 ERRO [1] job id 1210 saw error GenericError("test error")
Sep 22 23:16:36.431 ERRO [1] job id 1212 saw error GenericError("test error")
Sep 22 23:16:36.431 ERRO [1] job id 1214 saw error GenericError("test error")
Sep 22 23:16:36.431 ERRO [1] job id 1215 saw error GenericError("test error")
Sep 22 23:16:36.432 ERRO [1] job id 1217 saw error GenericError("test error")
Sep 22 23:16:36.432 ERRO [1] job id 1220 saw error GenericError("test error")
Sep 22 23:16:36.432 DEBG up_ds_listen was notified
Sep 22 23:16:36.432 DEBG up_ds_listen process 1210
Sep 22 23:16:36.432 DEBG [A] ack job 1210:211, : downstairs
Sep 22 23:16:36.432 DEBG up_ds_listen process 1213
Sep 22 23:16:36.432 DEBG [A] ack job 1213:214, : downstairs
Sep 22 23:16:36.432 DEBG up_ds_listen process 1216
Sep 22 23:16:36.432 DEBG [A] ack job 1216:217, : downstairs
Sep 22 23:16:36.432 DEBG up_ds_listen process 1219
Sep 22 23:16:36.432 DEBG [A] ack job 1219:220, : downstairs
Sep 22 23:16:36.432 DEBG up_ds_listen process 1222
Sep 22 23:16:36.432 DEBG [A] ack job 1222:223, : downstairs
Sep 22 23:16:36.432 DEBG up_ds_listen checked 5 jobs, back to waiting
Sep 22 23:16:36.432 DEBG up_ds_listen was notified
Sep 22 23:16:36.432 DEBG up_ds_listen checked 0 jobs, back to waiting
Sep 22 23:16:36.432 DEBG up_ds_listen was notified
Sep 22 23:16:36.432 DEBG up_ds_listen checked 0 jobs, back to waiting
Sep 22 23:16:36.432 DEBG up_ds_listen was notified
Sep 22 23:16:36.432 DEBG up_ds_listen checked 0 jobs, back to waiting
Sep 22 23:16:36.432 DEBG up_ds_listen was notified
Sep 22 23:16:36.432 DEBG up_ds_listen checked 0 jobs, back to waiting
Sep 22 23:16:36.432 ERRO [1] job id 1225 saw error GenericError("test error")
Sep 22 23:16:36.432 ERRO [0] job id 1231 saw error GenericError("test error")
Sep 22 23:16:36.432 ERRO [0] job id 1231 saw error GenericError("test error")
Sep 22 23:16:36.432 DEBG up_ds_listen was notified
Sep 22 23:16:36.432 DEBG up_ds_listen process 1225
Sep 22 23:16:36.432 DEBG [A] ack job 1225:226, : downstairs
Sep 22 23:16:36.432 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:36.432 ERRO [0] job id 1231 saw error GenericError("test error")
Sep 22 23:16:36.432 DEBG up_ds_listen was notified
Sep 22 23:16:36.432 DEBG up_ds_listen process 1228
Sep 22 23:16:36.432 DEBG [A] ack job 1228:229, : downstairs
Sep 22 23:16:36.432 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:36.440 INFO [lossy] sleeping 1 second
Sep 22 23:16:36.848 DEBG Write     :1230 deps:[JobId(1228), JobId(1225), JobId(1222), JobId(1219), JobId(1216), JobId(1213), JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:36.849 WARN returning error on flush!
Sep 22 23:16:36.849 DEBG Flush     :1231 extent_limit None deps:[JobId(1230), JobId(1229), JobId(1228)] res:false f:94 g:1
Sep 22 23:16:36.849 INFO [lossy] skipping 1233
Sep 22 23:16:36.857 DEBG Flush     :1231 extent_limit None deps:[JobId(1230), JobId(1229), JobId(1228)] res:true f:94 g:1
Sep 22 23:16:36.858 WARN returning error on write!
Sep 22 23:16:36.858 DEBG Write     :1233 deps:[JobId(1231), JobId(1228), JobId(1225), JobId(1222), JobId(1219), JobId(1216), JobId(1213), JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163), JobId(1162)] res:false
Sep 22 23:16:36.859 WARN returning error on write!
Sep 22 23:16:36.859 DEBG Write     :1233 deps:[JobId(1231), JobId(1228), JobId(1225), JobId(1222), JobId(1219), JobId(1216), JobId(1213), JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163), JobId(1162)] res:false
Sep 22 23:16:36.890 DEBG Write     :1233 deps:[JobId(1231), JobId(1228), JobId(1225), JobId(1222), JobId(1219), JobId(1216), JobId(1213), JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163), JobId(1162)] res:true
Sep 22 23:16:36.921 DEBG Write     :1232 deps:[JobId(1231), JobId(1228), JobId(1225), JobId(1222), JobId(1219), JobId(1216), JobId(1213), JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:36.922 WARN returning error on flush!
Sep 22 23:16:36.922 DEBG Flush     :1234 extent_limit None deps:[JobId(1233), JobId(1232), JobId(1231)] res:false f:95 g:1
Sep 22 23:16:36.922 INFO [lossy] skipping 1235
Sep 22 23:16:36.922 INFO [lossy] skipping 1237
Sep 22 23:16:36.922 WARN returning error on flush!
Sep 22 23:16:36.922 DEBG Flush     :1234 extent_limit None deps:[JobId(1233), JobId(1232), JobId(1231)] res:false f:95 g:1
Sep 22 23:16:36.922 WARN 1235 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:36.922 WARN returning error on flush!
Sep 22 23:16:36.922 DEBG Flush     :1234 extent_limit None deps:[JobId(1233), JobId(1232), JobId(1231)] res:false f:95 g:1
Sep 22 23:16:36.930 DEBG Flush     :1234 extent_limit None deps:[JobId(1233), JobId(1232), JobId(1231)] res:true f:95 g:1
Sep 22 23:16:36.930 INFO [lossy] sleeping 1 second
Sep 22 23:16:36.931 WARN returning error on write!
Sep 22 23:16:36.932 DEBG Write     :1232 deps:[JobId(1231), JobId(1228), JobId(1225), JobId(1222), JobId(1219), JobId(1216), JobId(1213), JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:false
Sep 22 23:16:36.962 DEBG Write     :1233 deps:[JobId(1231), JobId(1228), JobId(1225), JobId(1222), JobId(1219), JobId(1216), JobId(1213), JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163), JobId(1162)] res:true
Sep 22 23:16:36.963 INFO [lossy] skipping 1232
Sep 22 23:16:36.993 DEBG Write     :1232 deps:[JobId(1231), JobId(1228), JobId(1225), JobId(1222), JobId(1219), JobId(1216), JobId(1213), JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:36.995 ERRO [1] job id 1231 saw error GenericError("test error")
Sep 22 23:16:36.995 ERRO [1] job id 1233 saw error GenericError("test error")
Sep 22 23:16:36.995 ERRO [1] job id 1233 saw error GenericError("test error")
Sep 22 23:16:36.995 ERRO [1] job id 1234 saw error GenericError("test error")
Sep 22 23:16:36.995 ERRO [1] job id 1234 saw error GenericError("test error")
Sep 22 23:16:36.995 ERRO [1] job id 1234 saw error GenericError("test error")
Sep 22 23:16:37.003 DEBG up_ds_listen was notified
Sep 22 23:16:37.003 DEBG up_ds_listen process 1231
Sep 22 23:16:37.003 DEBG [A] ack job 1231:232, : downstairs
Sep 22 23:16:37.003 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:37.003 DEBG Flush     :1234 extent_limit None deps:[JobId(1233), JobId(1232), JobId(1231)] res:true f:95 g:1
Sep 22 23:16:37.003 INFO [lossy] skipping 1235
Sep 22 23:16:37.003 WARN 1236 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:37.033 DEBG Write     :1235 deps:[JobId(1234), JobId(1231), JobId(1228), JobId(1225), JobId(1222), JobId(1219), JobId(1216), JobId(1213), JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1164), JobId(1163)] res:true
Sep 22 23:16:37.038 DEBG Flush     :1236 extent_limit None deps:[JobId(1235), JobId(1234)] res:true f:96 g:1
Sep 22 23:16:37.044 DEBG Read      :1237 deps:[JobId(1236)] res:true
Sep 22 23:16:37.066 ERRO [0] job id 1232 saw error GenericError("test error")
Sep 22 23:16:37.066 DEBG up_ds_listen was notified
Sep 22 23:16:37.066 DEBG up_ds_listen process 1234
Sep 22 23:16:37.066 DEBG [A] ack job 1234:235, : downstairs
Sep 22 23:16:37.066 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:37.067 DEBG Flush     :1238 extent_limit None deps:[JobId(1237), JobId(1236)] res:true f:97 g:1
Sep 22 23:16:37.068 INFO [lossy] sleeping 1 second
Sep 22 23:16:37.443 DEBG [0] Read AckReady 1237, : downstairs
Sep 22 23:16:37.444 DEBG up_ds_listen was notified
Sep 22 23:16:37.444 DEBG up_ds_listen process 1237
Sep 22 23:16:37.444 DEBG [A] ack job 1237:238, : downstairs
Sep 22 23:16:37.496 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:37.497 INFO [lossy] skipping 1165
Sep 22 23:16:37.497 INFO [lossy] skipping 1169
Sep 22 23:16:37.497 INFO [lossy] skipping 1170
Sep 22 23:16:37.497 INFO [lossy] skipping 1171
Sep 22 23:16:37.497 INFO [lossy] skipping 1177
Sep 22 23:16:37.497 INFO [lossy] skipping 1178
Sep 22 23:16:37.497 INFO [lossy] skipping 1180
Sep 22 23:16:37.497 INFO [lossy] skipping 1183
Sep 22 23:16:37.497 INFO [lossy] skipping 1184
Sep 22 23:16:37.497 INFO [lossy] skipping 1185
Sep 22 23:16:37.497 INFO [lossy] skipping 1186
Sep 22 23:16:37.497 INFO [lossy] skipping 1187
Sep 22 23:16:37.497 INFO [lossy] skipping 1189
Sep 22 23:16:37.497 INFO [lossy] skipping 1191
Sep 22 23:16:37.497 INFO [lossy] skipping 1192
Sep 22 23:16:37.497 INFO [lossy] skipping 1195
Sep 22 23:16:37.497 INFO [lossy] skipping 1200
Sep 22 23:16:37.497 INFO [lossy] skipping 1204
Sep 22 23:16:37.497 INFO [lossy] skipping 1206
Sep 22 23:16:37.497 INFO [lossy] skipping 1208
Sep 22 23:16:37.497 INFO [lossy] skipping 1219
Sep 22 23:16:37.497 INFO [lossy] skipping 1222
Sep 22 23:16:37.497 INFO [lossy] skipping 1227
Sep 22 23:16:37.497 INFO [lossy] skipping 1228
Sep 22 23:16:37.497 INFO [lossy] skipping 1229
Sep 22 23:16:37.498 INFO [lossy] skipping 1234
Sep 22 23:16:37.498 INFO [lossy] skipping 1236
Sep 22 23:16:37.498 INFO [lossy] skipping 1238
Sep 22 23:16:37.498 WARN returning error on flush!
Sep 22 23:16:37.498 DEBG Flush     :1165 extent_limit None deps:[JobId(1164), JobId(1163)] res:false f:72 g:1
Sep 22 23:16:37.498 INFO [lossy] skipping 1170
Sep 22 23:16:37.498 INFO [lossy] skipping 1171
Sep 22 23:16:37.498 INFO [lossy] skipping 1178
Sep 22 23:16:37.498 INFO [lossy] skipping 1180
Sep 22 23:16:37.498 INFO [lossy] skipping 1185
Sep 22 23:16:37.498 INFO [lossy] skipping 1191
Sep 22 23:16:37.498 INFO [lossy] skipping 1222
Sep 22 23:16:37.498 INFO [lossy] skipping 1229
Sep 22 23:16:37.498 INFO [lossy] skipping 1238
Sep 22 23:16:37.498 DEBG Flush     :1165 extent_limit None deps:[JobId(1164), JobId(1163)] res:true f:72 g:1
Sep 22 23:16:37.498 INFO [lossy] skipping 1171
Sep 22 23:16:37.498 INFO [lossy] skipping 1185
Sep 22 23:16:37.529 DEBG Write     :1166 deps:[JobId(1165), JobId(1163)] res:true
Sep 22 23:16:37.560 DEBG Write     :1167 deps:[JobId(1165), JobId(1163)] res:true
Sep 22 23:16:37.561 WARN returning error on flush!
Sep 22 23:16:37.561 DEBG Flush     :1168 extent_limit None deps:[JobId(1167), JobId(1166), JobId(1165)] res:false f:73 g:1
Sep 22 23:16:37.561 WARN 1169 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:37.561 INFO [lossy] skipping 1172
Sep 22 23:16:37.561 WARN 1173 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:16:37.561 WARN 1175 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:16:37.561 WARN 1176 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:16:37.561 WARN 1179 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:16:37.561 WARN 1181 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 5 deps, role: work
Sep 22 23:16:37.561 WARN 1182 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 5 deps, role: work
Sep 22 23:16:37.561 WARN 1184 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 6 deps, role: work
Sep 22 23:16:37.561 WARN 1187 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 7 deps, role: work
Sep 22 23:16:37.561 WARN 1188 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 7 deps, role: work
Sep 22 23:16:37.561 WARN 1190 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 8 deps, role: work
Sep 22 23:16:37.561 WARN 1193 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 9 deps, role: work
Sep 22 23:16:37.561 WARN 1194 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 9 deps, role: work
Sep 22 23:16:37.561 WARN 1196 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 10 deps, role: work
Sep 22 23:16:37.561 WARN 1197 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 10 deps, role: work
Sep 22 23:16:37.561 WARN 1199 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 11 deps, role: work
Sep 22 23:16:37.561 WARN 1200 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 11 deps, role: work
Sep 22 23:16:37.561 INFO [lossy] skipping 1201
Sep 22 23:16:37.561 WARN 1202 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 12 deps, role: work
Sep 22 23:16:37.561 WARN 1203 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 12 deps, role: work
Sep 22 23:16:37.561 INFO [lossy] skipping 1204
Sep 22 23:16:37.561 INFO [lossy] skipping 1205
Sep 22 23:16:37.561 WARN 1206 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 13 deps, role: work
Sep 22 23:16:37.561 INFO [lossy] skipping 1208
Sep 22 23:16:37.561 WARN 1209 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 14 deps, role: work
Sep 22 23:16:37.561 WARN 1211 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 15 deps, role: work
Sep 22 23:16:37.561 INFO [lossy] skipping 1212
Sep 22 23:16:37.561 WARN 1214 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 16 deps, role: work
Sep 22 23:16:37.561 WARN 1215 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 16 deps, role: work
Sep 22 23:16:37.561 WARN 1217 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 17 deps, role: work
Sep 22 23:16:37.561 INFO [lossy] skipping 1218
Sep 22 23:16:37.562 WARN 1220 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 18 deps, role: work
Sep 22 23:16:37.562 WARN 1221 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 18 deps, role: work
Sep 22 23:16:37.562 WARN 1223 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 19 deps, role: work
Sep 22 23:16:37.562 WARN 1224 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 19 deps, role: work
Sep 22 23:16:37.562 WARN 1226 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 20 deps, role: work
Sep 22 23:16:37.562 WARN 1227 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 20 deps, role: work
Sep 22 23:16:37.562 INFO [lossy] skipping 1228
Sep 22 23:16:37.562 INFO [lossy] skipping 1229
Sep 22 23:16:37.562 WARN 1230 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 21 deps, role: work
Sep 22 23:16:37.562 INFO [lossy] skipping 1231
Sep 22 23:16:37.562 INFO [lossy] skipping 1232
Sep 22 23:16:37.562 INFO [lossy] skipping 1233
Sep 22 23:16:37.562 WARN 1235 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 23 deps, role: work
Sep 22 23:16:37.562 INFO [lossy] skipping 1237
Sep 22 23:16:37.570 DEBG Flush     :1168 extent_limit None deps:[JobId(1167), JobId(1166), JobId(1165)] res:true f:73 g:1
Sep 22 23:16:37.570 WARN 1172 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:37.570 INFO [lossy] skipping 1204
Sep 22 23:16:37.570 WARN 1205 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 12 deps, role: work
Sep 22 23:16:37.570 WARN 1208 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 13 deps, role: work
Sep 22 23:16:37.570 WARN 1212 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 14 deps, role: work
Sep 22 23:16:37.570 INFO [lossy] skipping 1218
Sep 22 23:16:37.570 WARN 1229 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 20 deps, role: work
Sep 22 23:16:37.570 WARN 1232 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 21 deps, role: work
Sep 22 23:16:37.570 WARN 1233 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 21 deps, role: work
Sep 22 23:16:37.570 INFO [lossy] skipping 1237
Sep 22 23:16:37.570 INFO [lossy] skipping 1218
Sep 22 23:16:37.570 INFO [lossy] skipping 1237
Sep 22 23:16:37.570 WARN 1218 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 16 deps, role: work
Sep 22 23:16:37.570 INFO [lossy] skipping 1237
Sep 22 23:16:37.570 INFO [lossy] skipping 1169
Sep 22 23:16:37.600 DEBG Write     :1170 deps:[JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:37.601 WARN 1171 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:37.601 INFO [lossy] skipping 1172
Sep 22 23:16:37.601 WARN 1173 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:37.601 WARN 1175 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:16:37.601 WARN 1176 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:16:37.601 WARN 1178 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:16:37.601 WARN 1179 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:16:37.601 INFO [lossy] skipping 1180
Sep 22 23:16:37.601 WARN 1181 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:16:37.601 WARN 1182 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:16:37.601 INFO [lossy] skipping 1184
Sep 22 23:16:37.601 INFO [lossy] skipping 1185
Sep 22 23:16:37.601 WARN 1187 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 6 deps, role: work
Sep 22 23:16:37.601 WARN 1188 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 6 deps, role: work
Sep 22 23:16:37.601 INFO [lossy] skipping 1189
Sep 22 23:16:37.601 WARN 1190 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 7 deps, role: work
Sep 22 23:16:37.601 INFO [lossy] skipping 1191
Sep 22 23:16:37.601 WARN 1193 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 8 deps, role: work
Sep 22 23:16:37.602 WARN 1194 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 8 deps, role: work
Sep 22 23:16:37.602 INFO [lossy] skipping 1195
Sep 22 23:16:37.602 WARN 1196 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 9 deps, role: work
Sep 22 23:16:37.602 WARN 1197 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 9 deps, role: work
Sep 22 23:16:37.602 WARN 1199 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 10 deps, role: work
Sep 22 23:16:37.602 WARN 1200 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 10 deps, role: work
Sep 22 23:16:37.602 WARN 1202 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 11 deps, role: work
Sep 22 23:16:37.602 WARN 1203 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 11 deps, role: work
Sep 22 23:16:37.602 INFO [lossy] skipping 1204
Sep 22 23:16:37.602 WARN 1206 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 12 deps, role: work
Sep 22 23:16:37.602 WARN 1209 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 13 deps, role: work
Sep 22 23:16:37.602 WARN 1211 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 14 deps, role: work
Sep 22 23:16:37.602 INFO [lossy] skipping 1213
Sep 22 23:16:37.602 WARN 1214 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 15 deps, role: work
Sep 22 23:16:37.602 WARN 1215 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 15 deps, role: work
Sep 22 23:16:37.602 INFO [lossy] skipping 1217
Sep 22 23:16:37.602 WARN 1220 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 17 deps, role: work
Sep 22 23:16:37.602 WARN 1221 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 17 deps, role: work
Sep 22 23:16:37.602 WARN 1223 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 18 deps, role: work
Sep 22 23:16:37.602 WARN 1224 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 18 deps, role: work
Sep 22 23:16:37.602 INFO [lossy] skipping 1226
Sep 22 23:16:37.602 WARN 1227 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 19 deps, role: work
Sep 22 23:16:37.602 WARN 1230 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 20 deps, role: work
Sep 22 23:16:37.602 INFO [lossy] skipping 1231
Sep 22 23:16:37.602 INFO [lossy] skipping 1232
Sep 22 23:16:37.602 INFO [lossy] skipping 1233
Sep 22 23:16:37.602 INFO [lossy] skipping 1234
Sep 22 23:16:37.602 WARN 1235 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 22 deps, role: work
Sep 22 23:16:37.632 DEBG Write     :1169 deps:[JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:37.634 INFO [lossy] skipping 1180
Sep 22 23:16:37.634 INFO [lossy] skipping 1184
Sep 22 23:16:37.634 INFO [lossy] skipping 1185
Sep 22 23:16:37.634 WARN 1191 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 7 deps, role: work
Sep 22 23:16:37.634 WARN 1217 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 16 deps, role: work
Sep 22 23:16:37.634 INFO [lossy] skipping 1226
Sep 22 23:16:37.634 WARN 1184 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 5 deps, role: work
Sep 22 23:16:37.634 WARN 1185 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 5 deps, role: work
Sep 22 23:16:37.634 WARN 1226 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 19 deps, role: work
Sep 22 23:16:37.634 INFO [lossy] skipping 1171
Sep 22 23:16:37.634 INFO [lossy] skipping 1172
Sep 22 23:16:37.634 INFO [lossy] skipping 1173
Sep 22 23:16:37.634 INFO [lossy] skipping 1175
Sep 22 23:16:37.634 INFO [lossy] skipping 1181
Sep 22 23:16:37.634 INFO [lossy] skipping 1182
Sep 22 23:16:37.634 INFO [lossy] skipping 1185
Sep 22 23:16:37.634 INFO [lossy] skipping 1188
Sep 22 23:16:37.634 INFO [lossy] skipping 1201
Sep 22 23:16:37.634 INFO [lossy] skipping 1202
Sep 22 23:16:37.634 INFO [lossy] skipping 1203
Sep 22 23:16:37.634 INFO [lossy] skipping 1204
Sep 22 23:16:37.634 INFO [lossy] skipping 1210
Sep 22 23:16:37.634 INFO [lossy] skipping 1212
Sep 22 23:16:37.634 INFO [lossy] skipping 1214
Sep 22 23:16:37.634 INFO [lossy] skipping 1222
Sep 22 23:16:37.634 INFO [lossy] skipping 1225
Sep 22 23:16:37.634 INFO [lossy] skipping 1231
Sep 22 23:16:37.634 INFO [lossy] skipping 1234
Sep 22 23:16:37.642 DEBG Flush     :1171 extent_limit None deps:[JobId(1170), JobId(1169), JobId(1168)] res:true f:74 g:1
Sep 22 23:16:37.642 INFO [lossy] skipping 1172
Sep 22 23:16:37.642 INFO [lossy] skipping 1173
Sep 22 23:16:37.642 WARN 1175 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:37.642 WARN 1181 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:16:37.642 WARN 1182 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:16:37.642 WARN 1185 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:16:37.642 WARN 1188 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 5 deps, role: work
Sep 22 23:16:37.642 WARN 1202 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 10 deps, role: work
Sep 22 23:16:37.642 WARN 1203 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 10 deps, role: work
Sep 22 23:16:37.642 WARN 1212 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 13 deps, role: work
Sep 22 23:16:37.642 WARN 1214 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 14 deps, role: work
Sep 22 23:16:37.642 INFO [lossy] skipping 1222
Sep 22 23:16:37.642 INFO [lossy] skipping 1231
Sep 22 23:16:37.673 DEBG Write     :1172 deps:[JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:37.704 DEBG Write     :1173 deps:[JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:37.713 DEBG Flush     :1174 extent_limit None deps:[JobId(1173), JobId(1172), JobId(1171)] res:true f:75 g:1
Sep 22 23:16:37.743 DEBG Write     :1175 deps:[JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:37.744 INFO [lossy] skipping 1176
Sep 22 23:16:37.744 INFO [lossy] skipping 1177
Sep 22 23:16:37.744 WARN 1178 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:37.744 INFO [lossy] skipping 1179
Sep 22 23:16:37.744 INFO [lossy] skipping 1180
Sep 22 23:16:37.744 WARN 1181 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:16:37.744 WARN 1182 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:16:37.744 INFO [lossy] skipping 1183
Sep 22 23:16:37.744 WARN 1184 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:16:37.744 INFO [lossy] skipping 1185
Sep 22 23:16:37.744 WARN 1187 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:16:37.744 WARN 1188 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:16:37.744 INFO [lossy] skipping 1189
Sep 22 23:16:37.744 INFO [lossy] skipping 1190
Sep 22 23:16:37.744 INFO [lossy] skipping 1191
Sep 22 23:16:37.744 WARN 1193 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 6 deps, role: work
Sep 22 23:16:37.744 WARN 1194 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 6 deps, role: work
Sep 22 23:16:37.744 WARN 1196 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 7 deps, role: work
Sep 22 23:16:37.744 INFO [lossy] skipping 1197
Sep 22 23:16:37.744 WARN 1199 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 8 deps, role: work
Sep 22 23:16:37.744 INFO [lossy] skipping 1200
Sep 22 23:16:37.744 INFO [lossy] skipping 1202
Sep 22 23:16:37.744 WARN 1203 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 9 deps, role: work
Sep 22 23:16:37.744 WARN 1205 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 10 deps, role: work
Sep 22 23:16:37.744 WARN 1206 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 10 deps, role: work
Sep 22 23:16:37.744 INFO [lossy] skipping 1208
Sep 22 23:16:37.744 WARN 1209 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 11 deps, role: work
Sep 22 23:16:37.744 WARN 1211 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 12 deps, role: work
Sep 22 23:16:37.744 INFO [lossy] skipping 1212
Sep 22 23:16:37.744 WARN 1214 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 13 deps, role: work
Sep 22 23:16:37.745 INFO [lossy] skipping 1215
Sep 22 23:16:37.745 WARN 1217 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 14 deps, role: work
Sep 22 23:16:37.745 INFO [lossy] skipping 1218
Sep 22 23:16:37.745 WARN 1220 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 15 deps, role: work
Sep 22 23:16:37.745 WARN 1221 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 15 deps, role: work
Sep 22 23:16:37.745 WARN 1223 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 16 deps, role: work
Sep 22 23:16:37.745 WARN 1224 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 16 deps, role: work
Sep 22 23:16:37.745 WARN 1226 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 17 deps, role: work
Sep 22 23:16:37.745 WARN 1227 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 17 deps, role: work
Sep 22 23:16:37.745 WARN 1229 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 18 deps, role: work
Sep 22 23:16:37.745 WARN 1230 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 18 deps, role: work
Sep 22 23:16:37.745 INFO [lossy] skipping 1232
Sep 22 23:16:37.745 WARN 1233 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 19 deps, role: work
Sep 22 23:16:37.745 INFO [lossy] skipping 1234
Sep 22 23:16:37.745 WARN 1235 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 20 deps, role: work
Sep 22 23:16:37.745 INFO [lossy] skipping 1236
Sep 22 23:16:37.775 DEBG Write     :1176 deps:[JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:37.776 WARN returning error on flush!
Sep 22 23:16:37.776 DEBG Flush     :1177 extent_limit None deps:[JobId(1176), JobId(1175), JobId(1174)] res:false f:76 g:1
Sep 22 23:16:37.776 WARN 1179 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:37.776 INFO [lossy] skipping 1180
Sep 22 23:16:37.776 INFO [lossy] skipping 1183
Sep 22 23:16:37.776 INFO [lossy] skipping 1185
Sep 22 23:16:37.776 WARN 1190 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 5 deps, role: work
Sep 22 23:16:37.776 WARN 1191 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 5 deps, role: work
Sep 22 23:16:37.776 WARN 1197 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 7 deps, role: work
Sep 22 23:16:37.776 INFO [lossy] skipping 1200
Sep 22 23:16:37.776 WARN 1202 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 9 deps, role: work
Sep 22 23:16:37.776 WARN 1208 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 11 deps, role: work
Sep 22 23:16:37.776 WARN 1212 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 12 deps, role: work
Sep 22 23:16:37.776 WARN 1215 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 13 deps, role: work
Sep 22 23:16:37.776 WARN 1218 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 14 deps, role: work
Sep 22 23:16:37.776 WARN 1232 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 19 deps, role: work
Sep 22 23:16:37.776 INFO [lossy] skipping 1234
Sep 22 23:16:37.784 DEBG Flush     :1177 extent_limit None deps:[JobId(1176), JobId(1175), JobId(1174)] res:true f:76 g:1
Sep 22 23:16:37.784 WARN 1180 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:16:37.784 WARN 1185 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:16:37.784 INFO [lossy] skipping 1200
Sep 22 23:16:37.784 WARN 1200 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 7 deps, role: work
Sep 22 23:16:37.784 INFO [lossy] sleeping 1 second
Sep 22 23:16:37.785 DEBG IO Read  1239 has deps [JobId(1238)]
Sep 22 23:16:37.785 ERRO [2] job id 1165 saw error GenericError("test error")
Sep 22 23:16:37.785 DEBG [rc] retire 1165 clears [JobId(1164), JobId(1165)], : downstairs
Sep 22 23:16:37.785 ERRO [2] job id 1168 saw error GenericError("test error")
Sep 22 23:16:37.788 DEBG [rc] retire 1168 clears [JobId(1166), JobId(1167), JobId(1168)], : downstairs
Sep 22 23:16:37.791 DEBG [rc] retire 1171 clears [JobId(1169), JobId(1170), JobId(1171)], : downstairs
Sep 22 23:16:37.793 DEBG [rc] retire 1174 clears [JobId(1172), JobId(1173), JobId(1174)], : downstairs
Sep 22 23:16:37.793 ERRO [2] job id 1177 saw error GenericError("test error")
Sep 22 23:16:37.796 DEBG [rc] retire 1177 clears [JobId(1175), JobId(1176), JobId(1177)], : downstairs
Sep 22 23:16:37.962 DEBG Write     :1235 deps:[JobId(1234), JobId(1231), JobId(1228), JobId(1225), JobId(1222), JobId(1219), JobId(1216), JobId(1213), JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1164), JobId(1163)] res:true
Sep 22 23:16:37.967 DEBG Flush     :1236 extent_limit None deps:[JobId(1235), JobId(1234)] res:true f:96 g:1
Sep 22 23:16:37.967 WARN returning error on read!
Sep 22 23:16:37.967 DEBG Read      :1237 deps:[JobId(1236)] res:false
Sep 22 23:16:37.968 WARN 1238 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:37.973 DEBG Read      :1237 deps:[JobId(1236)] res:true
Sep 22 23:16:37.995 ERRO [1] job id 1237 saw error GenericError("test error")
Sep 22 23:16:37.995 DEBG up_ds_listen was notified
Sep 22 23:16:37.995 DEBG up_ds_listen process 1236
Sep 22 23:16:37.995 DEBG [A] ack job 1236:237, : downstairs
Sep 22 23:16:37.995 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:37.996 DEBG Flush     :1238 extent_limit None deps:[JobId(1237), JobId(1236)] res:true f:97 g:1
Sep 22 23:16:38.002 DEBG Read      :1239 deps:[JobId(1238)] res:true
Sep 22 23:16:38.071 DEBG IO Flush 1240 has deps [JobId(1239), JobId(1238)]
Sep 22 23:16:38.401 DEBG up_ds_listen was notified
Sep 22 23:16:38.401 DEBG up_ds_listen process 1238
Sep 22 23:16:38.401 DEBG [A] ack job 1238:239, : downstairs
Sep 22 23:16:38.401 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:38.407 DEBG Read      :1239 deps:[JobId(1238)] res:true
Sep 22 23:16:38.430 WARN returning error on flush!
Sep 22 23:16:38.430 DEBG Flush     :1240 extent_limit None deps:[JobId(1239), JobId(1238)] res:false f:98 g:1
Sep 22 23:16:38.430 INFO [lossy] skipping 1240
Sep 22 23:16:38.430 DEBG Flush     :1240 extent_limit None deps:[JobId(1239), JobId(1238)] res:true f:98 g:1
Sep 22 23:16:38.430 INFO [lossy] sleeping 1 second
Sep 22 23:16:38.805 DEBG [1] Read AckReady 1239, : downstairs
Sep 22 23:16:38.805 ERRO [1] job id 1240 saw error GenericError("test error")
Sep 22 23:16:38.805 DEBG up_ds_listen was notified
Sep 22 23:16:38.806 DEBG up_ds_listen process 1239
Sep 22 23:16:38.806 DEBG [A] ack job 1239:240, : downstairs
Sep 22 23:16:38.858 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:38.860 DEBG Flush     :1240 extent_limit None deps:[JobId(1239), JobId(1238)] res:true f:98 g:1
Sep 22 23:16:38.860 INFO [lossy] sleeping 1 second
Sep 22 23:16:38.891 DEBG Write     :1178 deps:[JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:38.922 DEBG Write     :1179 deps:[JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:38.924 INFO [lossy] skipping 1180
Sep 22 23:16:38.924 WARN 1181 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:38.924 WARN 1182 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:38.924 WARN 1184 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:16:38.924 INFO [lossy] skipping 1185
Sep 22 23:16:38.924 INFO [lossy] skipping 1186
Sep 22 23:16:38.924 INFO [lossy] skipping 1187
Sep 22 23:16:38.924 WARN 1188 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:16:38.924 WARN 1190 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:16:38.924 WARN 1191 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:16:38.924 WARN 1193 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 5 deps, role: work
Sep 22 23:16:38.924 WARN 1194 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 5 deps, role: work
Sep 22 23:16:38.924 WARN 1196 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 6 deps, role: work
Sep 22 23:16:38.924 WARN 1197 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 6 deps, role: work
Sep 22 23:16:38.924 INFO [lossy] skipping 1199
Sep 22 23:16:38.924 WARN 1202 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 8 deps, role: work
Sep 22 23:16:38.924 INFO [lossy] skipping 1203
Sep 22 23:16:38.924 WARN 1205 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 9 deps, role: work
Sep 22 23:16:38.924 WARN 1206 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 9 deps, role: work
Sep 22 23:16:38.924 WARN 1208 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 10 deps, role: work
Sep 22 23:16:38.924 WARN 1209 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 10 deps, role: work
Sep 22 23:16:38.924 INFO [lossy] skipping 1210
Sep 22 23:16:38.924 WARN 1211 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 11 deps, role: work
Sep 22 23:16:38.924 INFO [lossy] skipping 1212
Sep 22 23:16:38.924 WARN 1214 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 12 deps, role: work
Sep 22 23:16:38.924 WARN 1215 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 12 deps, role: work
Sep 22 23:16:38.924 WARN 1217 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 13 deps, role: work
Sep 22 23:16:38.924 WARN 1218 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 13 deps, role: work
Sep 22 23:16:38.924 INFO [lossy] skipping 1220
Sep 22 23:16:38.924 WARN 1221 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 14 deps, role: work
Sep 22 23:16:38.924 WARN 1223 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 15 deps, role: work
Sep 22 23:16:38.924 WARN 1224 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 15 deps, role: work
Sep 22 23:16:38.924 WARN 1226 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 16 deps, role: work
Sep 22 23:16:38.924 INFO [lossy] skipping 1227
Sep 22 23:16:38.924 WARN 1229 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 17 deps, role: work
Sep 22 23:16:38.924 WARN 1230 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 17 deps, role: work
Sep 22 23:16:38.924 WARN 1232 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 18 deps, role: work
Sep 22 23:16:38.924 WARN 1233 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 18 deps, role: work
Sep 22 23:16:38.924 WARN 1235 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 19 deps, role: work
Sep 22 23:16:38.932 DEBG Flush     :1180 extent_limit None deps:[JobId(1179), JobId(1178), JobId(1177)] res:true f:77 g:1
Sep 22 23:16:38.932 WARN 1185 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:38.932 WARN 1187 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:16:38.932 WARN 1199 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 6 deps, role: work
Sep 22 23:16:38.933 WARN 1203 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 7 deps, role: work
Sep 22 23:16:38.933 WARN 1212 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 10 deps, role: work
Sep 22 23:16:38.933 WARN 1220 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 13 deps, role: work
Sep 22 23:16:38.933 WARN 1227 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 15 deps, role: work
Sep 22 23:16:38.933 INFO [lossy] sleeping 1 second
Sep 22 23:16:38.980 DEBG IO Read  1241 has deps [JobId(1240)]
Sep 22 23:16:39.313 DEBG [rc] retire 1180 clears [JobId(1178), JobId(1179), JobId(1180)], : downstairs
Sep 22 23:16:39.318 DEBG up_ds_listen was notified
Sep 22 23:16:39.318 DEBG up_ds_listen process 1240
Sep 22 23:16:39.318 DEBG [A] ack job 1240:241, : downstairs
Sep 22 23:16:39.318 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:39.361 DEBG IO Flush 1242 has deps [JobId(1241), JobId(1240)]
Sep 22 23:16:39.432 INFO [lossy] skipping 1241
Sep 22 23:16:39.432 INFO [lossy] skipping 1242
Sep 22 23:16:39.438 DEBG Read      :1241 deps:[JobId(1240)] res:true
Sep 22 23:16:39.460 INFO [lossy] skipping 1242
Sep 22 23:16:39.460 DEBG Flush     :1242 extent_limit None deps:[JobId(1241), JobId(1240)] res:true f:99 g:1
Sep 22 23:16:39.460 INFO [lossy] sleeping 1 second
Sep 22 23:16:39.835 DEBG [1] Read AckReady 1241, : downstairs
Sep 22 23:16:39.836 DEBG up_ds_listen was notified
Sep 22 23:16:39.836 DEBG up_ds_listen process 1241
Sep 22 23:16:39.836 DEBG [A] ack job 1241:242, : downstairs
Sep 22 23:16:39.889 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:39.895 DEBG Read      :1241 deps:[JobId(1240)] res:true
Sep 22 23:16:39.917 DEBG IO Read  1243 has deps [JobId(1242)]
Sep 22 23:16:39.930 DEBG Flush     :1242 extent_limit None deps:[JobId(1241), JobId(1240)] res:true f:99 g:1
Sep 22 23:16:39.936 DEBG Read      :1243 deps:[JobId(1242)] res:true
Sep 22 23:16:40.035 DEBG Write     :1181 deps:[JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:40.037 WARN returning error on write!
Sep 22 23:16:40.037 DEBG Write     :1182 deps:[JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:false
Sep 22 23:16:40.038 WARN 1183 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:40.038 INFO [lossy] skipping 1184
Sep 22 23:16:40.038 INFO [lossy] skipping 1188
Sep 22 23:16:40.038 WARN 1190 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:16:40.038 WARN 1191 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:16:40.038 INFO [lossy] skipping 1193
Sep 22 23:16:40.038 WARN 1194 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:16:40.038 INFO [lossy] skipping 1195
Sep 22 23:16:40.038 WARN 1196 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 5 deps, role: work
Sep 22 23:16:40.038 WARN 1197 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 5 deps, role: work
Sep 22 23:16:40.038 WARN 1200 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 6 deps, role: work
Sep 22 23:16:40.038 WARN 1202 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 7 deps, role: work
Sep 22 23:16:40.038 INFO [lossy] skipping 1203
Sep 22 23:16:40.038 INFO [lossy] skipping 1204
Sep 22 23:16:40.038 WARN 1205 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 8 deps, role: work
Sep 22 23:16:40.038 WARN 1206 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 8 deps, role: work
Sep 22 23:16:40.038 INFO [lossy] skipping 1207
Sep 22 23:16:40.038 WARN 1208 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 9 deps, role: work
Sep 22 23:16:40.038 INFO [lossy] skipping 1209
Sep 22 23:16:40.038 WARN 1211 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 10 deps, role: work
Sep 22 23:16:40.038 WARN 1214 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 11 deps, role: work
Sep 22 23:16:40.038 WARN 1215 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 11 deps, role: work
Sep 22 23:16:40.038 WARN 1217 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 12 deps, role: work
Sep 22 23:16:40.038 WARN 1218 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 12 deps, role: work
Sep 22 23:16:40.038 WARN 1221 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 13 deps, role: work
Sep 22 23:16:40.038 INFO [lossy] skipping 1222
Sep 22 23:16:40.038 INFO [lossy] skipping 1223
Sep 22 23:16:40.038 INFO [lossy] skipping 1224
Sep 22 23:16:40.038 WARN 1226 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 15 deps, role: work
Sep 22 23:16:40.038 WARN 1229 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 16 deps, role: work
Sep 22 23:16:40.038 INFO [lossy] skipping 1230
Sep 22 23:16:40.038 INFO [lossy] skipping 1231
Sep 22 23:16:40.038 INFO [lossy] skipping 1232
Sep 22 23:16:40.038 WARN 1233 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 17 deps, role: work
Sep 22 23:16:40.038 WARN 1235 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 18 deps, role: work
Sep 22 23:16:40.038 INFO [lossy] skipping 1238
Sep 22 23:16:40.038 INFO [lossy] skipping 1242
Sep 22 23:16:40.038 INFO [lossy] skipping 1182
Sep 22 23:16:40.038 WARN 1184 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:40.038 WARN 1188 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:16:40.038 WARN 1193 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:16:40.038 WARN 1209 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 9 deps, role: work
Sep 22 23:16:40.038 INFO [lossy] skipping 1223
Sep 22 23:16:40.038 WARN 1224 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 14 deps, role: work
Sep 22 23:16:40.038 INFO [lossy] skipping 1230
Sep 22 23:16:40.038 WARN 1232 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 17 deps, role: work
Sep 22 23:16:40.039 INFO [lossy] skipping 1242
Sep 22 23:16:40.068 DEBG Write     :1182 deps:[JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:40.069 WARN 1223 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 14 deps, role: work
Sep 22 23:16:40.069 INFO [lossy] skipping 1230
Sep 22 23:16:40.069 INFO [lossy] skipping 1230
Sep 22 23:16:40.069 WARN 1230 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 16 deps, role: work
Sep 22 23:16:40.069 INFO [lossy] sleeping 1 second
Sep 22 23:16:40.399 DEBG up_ds_listen was notified
Sep 22 23:16:40.399 DEBG up_ds_listen process 1242
Sep 22 23:16:40.399 DEBG [A] ack job 1242:243, : downstairs
Sep 22 23:16:40.399 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:40.400 ERRO [2] job id 1182 saw error GenericError("test error")
Sep 22 23:16:40.402 INFO [lossy] sleeping 1 second
Sep 22 23:16:40.777 DEBG [0] Read AckReady 1243, : downstairs
Sep 22 23:16:40.777 DEBG up_ds_listen was notified
Sep 22 23:16:40.777 DEBG up_ds_listen process 1243
Sep 22 23:16:40.777 DEBG [A] ack job 1243:244, : downstairs
Sep 22 23:16:40.830 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:40.831 DEBG IO Flush 1244 has deps [JobId(1243), JobId(1242)]
Sep 22 23:16:40.831 WARN returning error on read!
Sep 22 23:16:40.831 DEBG Read      :1243 deps:[JobId(1242)] res:false
Sep 22 23:16:40.837 DEBG Read      :1243 deps:[JobId(1242)] res:true
Sep 22 23:16:40.859 DEBG IO Read  1245 has deps [JobId(1244)]
Sep 22 23:16:40.859 ERRO [1] job id 1243 saw error GenericError("test error")
Sep 22 23:16:40.872 INFO [lossy] skipping 1244
Sep 22 23:16:40.872 DEBG Flush     :1244 extent_limit None deps:[JobId(1243), JobId(1242)] res:true f:100 g:1
Sep 22 23:16:40.872 WARN returning error on read!
Sep 22 23:16:40.872 DEBG Read      :1245 deps:[JobId(1244)] res:false
Sep 22 23:16:40.878 DEBG Read      :1245 deps:[JobId(1244)] res:true
Sep 22 23:16:41.277 ERRO [1] job id 1245 saw error GenericError("test error")
Sep 22 23:16:41.285 DEBG Flush     :1183 extent_limit None deps:[JobId(1182), JobId(1181), JobId(1180)] res:true f:78 g:1
Sep 22 23:16:41.316 DEBG Write     :1184 deps:[JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:41.347 DEBG Write     :1185 deps:[JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:41.356 DEBG Flush     :1186 extent_limit None deps:[JobId(1185), JobId(1184), JobId(1183)] res:true f:79 g:1
Sep 22 23:16:41.357 WARN returning error on write!
Sep 22 23:16:41.357 DEBG Write     :1187 deps:[JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:false
Sep 22 23:16:41.388 DEBG Write     :1188 deps:[JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:41.389 WARN 1189 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:41.389 WARN 1190 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:41.389 INFO [lossy] skipping 1191
Sep 22 23:16:41.389 WARN 1193 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:16:41.389 INFO [lossy] skipping 1194
Sep 22 23:16:41.389 INFO [lossy] skipping 1195
Sep 22 23:16:41.389 INFO [lossy] skipping 1196
Sep 22 23:16:41.389 WARN 1197 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:16:41.389 INFO [lossy] skipping 1198
Sep 22 23:16:41.389 INFO [lossy] skipping 1199
Sep 22 23:16:41.389 WARN 1200 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:16:41.389 WARN 1202 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 5 deps, role: work
Sep 22 23:16:41.389 WARN 1203 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 5 deps, role: work
Sep 22 23:16:41.389 INFO [lossy] skipping 1204
Sep 22 23:16:41.389 INFO [lossy] skipping 1205
Sep 22 23:16:41.389 WARN 1206 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 6 deps, role: work
Sep 22 23:16:41.389 WARN 1208 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 7 deps, role: work
Sep 22 23:16:41.389 INFO [lossy] skipping 1209
Sep 22 23:16:41.389 INFO [lossy] skipping 1210
Sep 22 23:16:41.389 INFO [lossy] skipping 1211
Sep 22 23:16:41.389 WARN 1212 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 8 deps, role: work
Sep 22 23:16:41.389 WARN 1214 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 9 deps, role: work
Sep 22 23:16:41.389 WARN 1215 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 9 deps, role: work
Sep 22 23:16:41.389 INFO [lossy] skipping 1216
Sep 22 23:16:41.389 WARN 1217 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 10 deps, role: work
Sep 22 23:16:41.389 INFO [lossy] skipping 1218
Sep 22 23:16:41.389 WARN 1220 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 11 deps, role: work
Sep 22 23:16:41.389 INFO [lossy] skipping 1221
Sep 22 23:16:41.389 WARN 1223 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 12 deps, role: work
Sep 22 23:16:41.389 INFO [lossy] skipping 1224
Sep 22 23:16:41.389 INFO [lossy] skipping 1225
Sep 22 23:16:41.389 INFO [lossy] skipping 1226
Sep 22 23:16:41.389 INFO [lossy] skipping 1227
Sep 22 23:16:41.389 INFO [lossy] skipping 1228
Sep 22 23:16:41.390 WARN 1229 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 14 deps, role: work
Sep 22 23:16:41.390 WARN 1230 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 14 deps, role: work
Sep 22 23:16:41.390 INFO [lossy] skipping 1231
Sep 22 23:16:41.390 WARN 1232 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 15 deps, role: work
Sep 22 23:16:41.390 WARN 1233 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 15 deps, role: work
Sep 22 23:16:41.390 WARN 1235 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 16 deps, role: work
Sep 22 23:16:41.390 INFO [lossy] skipping 1236
Sep 22 23:16:41.390 INFO [lossy] skipping 1237
Sep 22 23:16:41.390 INFO [lossy] skipping 1242
Sep 22 23:16:41.390 INFO [lossy] skipping 1244
Sep 22 23:16:41.420 DEBG Write     :1187 deps:[JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:41.421 INFO [lossy] skipping 1191
Sep 22 23:16:41.421 WARN 1194 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:16:41.421 WARN 1196 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:16:41.421 WARN 1199 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:16:41.421 WARN 1205 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 6 deps, role: work
Sep 22 23:16:41.421 INFO [lossy] skipping 1209
Sep 22 23:16:41.421 INFO [lossy] skipping 1210
Sep 22 23:16:41.421 WARN 1211 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 8 deps, role: work
Sep 22 23:16:41.421 INFO [lossy] skipping 1216
Sep 22 23:16:41.421 WARN 1218 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 10 deps, role: work
Sep 22 23:16:41.421 WARN 1221 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 11 deps, role: work
Sep 22 23:16:41.421 WARN 1224 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 12 deps, role: work
Sep 22 23:16:41.421 WARN 1226 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 13 deps, role: work
Sep 22 23:16:41.421 WARN 1227 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 13 deps, role: work
Sep 22 23:16:41.421 INFO [lossy] skipping 1191
Sep 22 23:16:41.421 WARN 1209 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 7 deps, role: work
Sep 22 23:16:41.421 WARN 1191 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:41.421 INFO [lossy] sleeping 1 second
Sep 22 23:16:41.422 DEBG IO Flush 1246 has deps [JobId(1245), JobId(1244)]
Sep 22 23:16:41.422 DEBG Flush     :1244 extent_limit None deps:[JobId(1243), JobId(1242)] res:true f:100 g:1
Sep 22 23:16:41.422 WARN returning error on read!
Sep 22 23:16:41.422 DEBG Read      :1245 deps:[JobId(1244)] res:false
Sep 22 23:16:41.422 WARN returning error on read!
Sep 22 23:16:41.422 DEBG Read      :1245 deps:[JobId(1244)] res:false
Sep 22 23:16:41.428 DEBG Read      :1245 deps:[JobId(1244)] res:true
Sep 22 23:16:41.452 DEBG [rc] retire 1183 clears [JobId(1181), JobId(1182), JobId(1183)], : downstairs
Sep 22 23:16:41.455 DEBG [rc] retire 1186 clears [JobId(1184), JobId(1185), JobId(1186)], : downstairs
Sep 22 23:16:41.455 ERRO [2] job id 1187 saw error GenericError("test error")
Sep 22 23:16:41.456 ERRO [0] job id 1245 saw error GenericError("test error")
Sep 22 23:16:41.456 ERRO [0] job id 1245 saw error GenericError("test error")
Sep 22 23:16:41.456 DEBG up_ds_listen was notified
Sep 22 23:16:41.456 DEBG up_ds_listen process 1244
Sep 22 23:16:41.456 DEBG [A] ack job 1244:245, : downstairs
Sep 22 23:16:41.456 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:41.459 INFO [lossy] skipping 1246
Sep 22 23:16:41.459 INFO [lossy] skipping 1246
Sep 22 23:16:41.459 WARN returning error on flush!
Sep 22 23:16:41.459 DEBG Flush     :1246 extent_limit None deps:[JobId(1245), JobId(1244)] res:false f:101 g:1
Sep 22 23:16:41.459 INFO [lossy] skipping 1246
Sep 22 23:16:41.459 INFO [lossy] skipping 1246
Sep 22 23:16:41.459 WARN returning error on flush!
Sep 22 23:16:41.459 DEBG Flush     :1246 extent_limit None deps:[JobId(1245), JobId(1244)] res:false f:101 g:1
Sep 22 23:16:41.459 DEBG Flush     :1246 extent_limit None deps:[JobId(1245), JobId(1244)] res:true f:101 g:1
Sep 22 23:16:41.459 INFO [lossy] sleeping 1 second
Sep 22 23:16:41.834 DEBG [1] Read AckReady 1245, : downstairs
Sep 22 23:16:41.835 ERRO [1] job id 1246 saw error GenericError("test error")
Sep 22 23:16:41.835 ERRO [1] job id 1246 saw error GenericError("test error")
Sep 22 23:16:41.835 DEBG up_ds_listen was notified
Sep 22 23:16:41.835 DEBG up_ds_listen process 1245
Sep 22 23:16:41.835 DEBG [A] ack job 1245:246, : downstairs
Sep 22 23:16:41.888 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:41.890 DEBG Flush     :1246 extent_limit None deps:[JobId(1245), JobId(1244)] res:true f:101 g:1
Sep 22 23:16:41.890 INFO [lossy] sleeping 1 second
Sep 22 23:16:41.890 DEBG IO Read  1247 has deps [JobId(1246)]
Sep 22 23:16:42.279 DEBG up_ds_listen was notified
Sep 22 23:16:42.279 DEBG up_ds_listen process 1246
Sep 22 23:16:42.279 DEBG [A] ack job 1246:247, : downstairs
Sep 22 23:16:42.279 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:42.279 DEBG IO Flush 1248 has deps [JobId(1247), JobId(1246)]
Sep 22 23:16:42.430 DEBG Flush     :1189 extent_limit None deps:[JobId(1188), JobId(1187), JobId(1186)] res:true f:80 g:1
Sep 22 23:16:42.430 INFO [lossy] skipping 1190
Sep 22 23:16:42.461 DEBG Write     :1191 deps:[JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:42.462 WARN 1192 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:42.462 WARN 1193 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:42.462 WARN 1194 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:42.462 WARN 1196 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:16:42.462 WARN 1197 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:16:42.462 WARN 1199 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:16:42.462 WARN 1200 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:16:42.462 WARN 1202 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:16:42.462 INFO [lossy] skipping 1203
Sep 22 23:16:42.462 WARN 1205 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 5 deps, role: work
Sep 22 23:16:42.462 WARN 1206 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 5 deps, role: work
Sep 22 23:16:42.462 INFO [lossy] skipping 1208
Sep 22 23:16:42.462 INFO [lossy] skipping 1209
Sep 22 23:16:42.462 WARN 1211 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 7 deps, role: work
Sep 22 23:16:42.462 WARN 1212 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 7 deps, role: work
Sep 22 23:16:42.462 INFO [lossy] skipping 1213
Sep 22 23:16:42.462 WARN 1214 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 8 deps, role: work
Sep 22 23:16:42.462 INFO [lossy] skipping 1215
Sep 22 23:16:42.462 INFO [lossy] skipping 1216
Sep 22 23:16:42.462 WARN 1217 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 9 deps, role: work
Sep 22 23:16:42.462 WARN 1218 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 9 deps, role: work
Sep 22 23:16:42.462 WARN 1220 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 10 deps, role: work
Sep 22 23:16:42.462 WARN 1221 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 10 deps, role: work
Sep 22 23:16:42.462 INFO [lossy] skipping 1222
Sep 22 23:16:42.462 INFO [lossy] skipping 1223
Sep 22 23:16:42.462 WARN 1224 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 11 deps, role: work
Sep 22 23:16:42.462 INFO [lossy] skipping 1225
Sep 22 23:16:42.462 WARN 1226 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 12 deps, role: work
Sep 22 23:16:42.462 WARN 1227 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 12 deps, role: work
Sep 22 23:16:42.462 INFO [lossy] skipping 1229
Sep 22 23:16:42.462 WARN 1230 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 13 deps, role: work
Sep 22 23:16:42.462 WARN 1232 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 14 deps, role: work
Sep 22 23:16:42.462 WARN 1233 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 14 deps, role: work
Sep 22 23:16:42.462 WARN 1235 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 15 deps, role: work
Sep 22 23:16:42.462 INFO [lossy] skipping 1237
Sep 22 23:16:42.462 INFO [lossy] skipping 1247
Sep 22 23:16:42.462 INFO [lossy] skipping 1190
Sep 22 23:16:42.462 WARN 1203 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:16:42.462 WARN 1208 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 6 deps, role: work
Sep 22 23:16:42.463 WARN 1209 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 6 deps, role: work
Sep 22 23:16:42.463 INFO [lossy] skipping 1213
Sep 22 23:16:42.463 WARN 1215 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 8 deps, role: work
Sep 22 23:16:42.463 INFO [lossy] skipping 1223
Sep 22 23:16:42.463 WARN 1229 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 13 deps, role: work
Sep 22 23:16:42.463 INFO [lossy] skipping 1190
Sep 22 23:16:42.463 WARN 1223 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 11 deps, role: work
Sep 22 23:16:42.493 DEBG Write     :1190 deps:[JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:42.494 INFO [lossy] sleeping 1 second
Sep 22 23:16:42.494 INFO [lossy] skipping 1247
Sep 22 23:16:42.494 INFO [lossy] skipping 1248
Sep 22 23:16:42.494 INFO [lossy] skipping 1247
Sep 22 23:16:42.494 INFO [lossy] skipping 1248
Sep 22 23:16:42.500 DEBG Read      :1247 deps:[JobId(1246)] res:true
Sep 22 23:16:42.524 DEBG [rc] retire 1189 clears [JobId(1187), JobId(1188), JobId(1189)], : downstairs
Sep 22 23:16:42.526 WARN returning error on flush!
Sep 22 23:16:42.526 DEBG Flush     :1248 extent_limit None deps:[JobId(1247), JobId(1246)] res:false f:102 g:1
Sep 22 23:16:42.526 DEBG Flush     :1248 extent_limit None deps:[JobId(1247), JobId(1246)] res:true f:102 g:1
Sep 22 23:16:42.526 INFO [lossy] sleeping 1 second
Sep 22 23:16:42.901 DEBG [1] Read AckReady 1247, : downstairs
Sep 22 23:16:42.902 ERRO [1] job id 1248 saw error GenericError("test error")
Sep 22 23:16:42.902 DEBG up_ds_listen was notified
Sep 22 23:16:42.902 DEBG up_ds_listen process 1247
Sep 22 23:16:42.902 DEBG [A] ack job 1247:248, : downstairs
Sep 22 23:16:42.955 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:42.961 DEBG Read      :1247 deps:[JobId(1246)] res:true
Sep 22 23:16:42.983 DEBG IO Read  1249 has deps [JobId(1248)]
Sep 22 23:16:42.996 DEBG Flush     :1248 extent_limit None deps:[JobId(1247), JobId(1246)] res:true f:102 g:1
Sep 22 23:16:42.996 INFO [lossy] sleeping 1 second
Sep 22 23:16:43.373 DEBG up_ds_listen was notified
Sep 22 23:16:43.373 DEBG up_ds_listen process 1248
Sep 22 23:16:43.373 DEBG [A] ack job 1248:249, : downstairs
Sep 22 23:16:43.373 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:43.456 DEBG IO Flush 1250 has deps [JobId(1249), JobId(1248)]
Sep 22 23:16:43.495 WARN returning error on flush!
Sep 22 23:16:43.495 DEBG Flush     :1192 extent_limit None deps:[JobId(1191), JobId(1190), JobId(1189)] res:false f:81 g:1
Sep 22 23:16:43.495 INFO [lossy] skipping 1193
Sep 22 23:16:43.496 INFO [lossy] skipping 1196
Sep 22 23:16:43.496 INFO [lossy] skipping 1198
Sep 22 23:16:43.496 INFO [lossy] skipping 1203
Sep 22 23:16:43.496 INFO [lossy] skipping 1204
Sep 22 23:16:43.496 INFO [lossy] skipping 1212
Sep 22 23:16:43.496 INFO [lossy] skipping 1216
Sep 22 23:16:43.496 INFO [lossy] skipping 1217
Sep 22 23:16:43.496 INFO [lossy] skipping 1219
Sep 22 23:16:43.496 INFO [lossy] skipping 1223
Sep 22 23:16:43.496 INFO [lossy] skipping 1228
Sep 22 23:16:43.496 INFO [lossy] skipping 1231
Sep 22 23:16:43.496 INFO [lossy] skipping 1233
Sep 22 23:16:43.496 INFO [lossy] skipping 1240
Sep 22 23:16:43.504 DEBG Flush     :1192 extent_limit None deps:[JobId(1191), JobId(1190), JobId(1189)] res:true f:81 g:1
Sep 22 23:16:43.504 INFO [lossy] skipping 1193
Sep 22 23:16:43.504 INFO [lossy] skipping 1196
Sep 22 23:16:43.504 INFO [lossy] skipping 1203
Sep 22 23:16:43.504 INFO [lossy] skipping 1212
Sep 22 23:16:43.504 WARN 1217 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 8 deps, role: work
Sep 22 23:16:43.504 INFO [lossy] skipping 1223
Sep 22 23:16:43.504 WARN 1233 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 13 deps, role: work
Sep 22 23:16:43.535 DEBG Write     :1193 deps:[JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:43.536 WARN 1196 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:43.536 WARN 1203 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:16:43.536 WARN 1212 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 6 deps, role: work
Sep 22 23:16:43.536 WARN 1223 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 10 deps, role: work
Sep 22 23:16:43.567 DEBG Write     :1194 deps:[JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:43.568 INFO [lossy] skipping 1195
Sep 22 23:16:43.568 WARN 1197 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:43.568 WARN 1199 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:16:43.568 WARN 1200 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:16:43.568 WARN 1202 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:16:43.568 INFO [lossy] skipping 1205
Sep 22 23:16:43.568 WARN 1206 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:16:43.568 WARN 1208 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 5 deps, role: work
Sep 22 23:16:43.568 WARN 1209 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 5 deps, role: work
Sep 22 23:16:43.568 INFO [lossy] skipping 1210
Sep 22 23:16:43.568 WARN 1211 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 6 deps, role: work
Sep 22 23:16:43.568 INFO [lossy] skipping 1212
Sep 22 23:16:43.568 INFO [lossy] skipping 1214
Sep 22 23:16:43.568 WARN 1215 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 7 deps, role: work
Sep 22 23:16:43.568 INFO [lossy] skipping 1217
Sep 22 23:16:43.568 WARN 1218 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 8 deps, role: work
Sep 22 23:16:43.568 WARN 1220 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 9 deps, role: work
Sep 22 23:16:43.568 INFO [lossy] skipping 1221
Sep 22 23:16:43.568 WARN 1224 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 10 deps, role: work
Sep 22 23:16:43.568 INFO [lossy] skipping 1226
Sep 22 23:16:43.568 WARN 1227 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 11 deps, role: work
Sep 22 23:16:43.568 WARN 1229 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 12 deps, role: work
Sep 22 23:16:43.568 WARN 1230 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 12 deps, role: work
Sep 22 23:16:43.568 WARN 1232 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 13 deps, role: work
Sep 22 23:16:43.568 WARN 1235 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 14 deps, role: work
Sep 22 23:16:43.568 INFO [lossy] skipping 1239
Sep 22 23:16:43.568 INFO [lossy] skipping 1242
Sep 22 23:16:43.568 INFO [lossy] skipping 1249
Sep 22 23:16:43.568 WARN returning error on flush!
Sep 22 23:16:43.568 DEBG Flush     :1195 extent_limit None deps:[JobId(1194), JobId(1193), JobId(1192)] res:false f:82 g:1
Sep 22 23:16:43.568 WARN 1205 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:16:43.568 INFO [lossy] skipping 1212
Sep 22 23:16:43.568 INFO [lossy] skipping 1214
Sep 22 23:16:43.568 WARN 1221 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 9 deps, role: work
Sep 22 23:16:43.568 WARN 1226 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 11 deps, role: work
Sep 22 23:16:43.576 DEBG Flush     :1195 extent_limit None deps:[JobId(1194), JobId(1193), JobId(1192)] res:true f:82 g:1
Sep 22 23:16:43.576 INFO [lossy] skipping 1212
Sep 22 23:16:43.576 WARN 1214 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 6 deps, role: work
Sep 22 23:16:43.576 INFO [lossy] skipping 1212
Sep 22 23:16:43.576 WARN 1212 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 5 deps, role: work
Sep 22 23:16:43.577 INFO [lossy] skipping 1196
Sep 22 23:16:43.607 DEBG Write     :1197 deps:[JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:43.608 WARN 1198 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:43.608 WARN 1199 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:43.608 WARN 1200 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:43.608 WARN 1202 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:16:43.608 WARN 1203 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:16:43.608 WARN 1205 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:16:43.608 INFO [lossy] skipping 1206
Sep 22 23:16:43.608 INFO [lossy] skipping 1207
Sep 22 23:16:43.608 INFO [lossy] skipping 1208
Sep 22 23:16:43.608 WARN 1209 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:16:43.608 WARN 1211 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 5 deps, role: work
Sep 22 23:16:43.608 INFO [lossy] skipping 1212
Sep 22 23:16:43.608 WARN 1215 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 6 deps, role: work
Sep 22 23:16:43.608 WARN 1217 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 7 deps, role: work
Sep 22 23:16:43.608 INFO [lossy] skipping 1218
Sep 22 23:16:43.608 WARN 1220 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 8 deps, role: work
Sep 22 23:16:43.608 WARN 1221 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 8 deps, role: work
Sep 22 23:16:43.608 WARN 1223 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 9 deps, role: work
Sep 22 23:16:43.608 INFO [lossy] skipping 1224
Sep 22 23:16:43.608 WARN 1226 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 10 deps, role: work
Sep 22 23:16:43.608 INFO [lossy] skipping 1227
Sep 22 23:16:43.608 WARN 1229 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 11 deps, role: work
Sep 22 23:16:43.608 WARN 1230 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 11 deps, role: work
Sep 22 23:16:43.608 INFO [lossy] skipping 1232
Sep 22 23:16:43.608 INFO [lossy] skipping 1233
Sep 22 23:16:43.608 WARN 1235 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 13 deps, role: work
Sep 22 23:16:43.608 INFO [lossy] skipping 1240
Sep 22 23:16:43.609 INFO [lossy] skipping 1249
Sep 22 23:16:43.609 INFO [lossy] skipping 1250
Sep 22 23:16:43.639 DEBG Write     :1196 deps:[JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:43.640 WARN 1206 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:16:43.640 WARN 1208 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:16:43.640 WARN 1218 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 7 deps, role: work
Sep 22 23:16:43.640 WARN 1224 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 9 deps, role: work
Sep 22 23:16:43.640 INFO [lossy] skipping 1227
Sep 22 23:16:43.640 INFO [lossy] skipping 1232
Sep 22 23:16:43.640 WARN 1233 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 12 deps, role: work
Sep 22 23:16:43.640 INFO [lossy] skipping 1240
Sep 22 23:16:43.640 WARN 1227 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 10 deps, role: work
Sep 22 23:16:43.640 WARN 1232 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 12 deps, role: work
Sep 22 23:16:43.648 DEBG Flush     :1198 extent_limit None deps:[JobId(1197), JobId(1196), JobId(1195)] res:true f:83 g:1
Sep 22 23:16:43.649 WARN returning error on write!
Sep 22 23:16:43.649 DEBG Write     :1199 deps:[JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:false
Sep 22 23:16:43.680 DEBG Write     :1200 deps:[JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:43.681 WARN 1201 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:43.681 WARN 1202 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:43.681 WARN 1203 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:43.681 INFO [lossy] skipping 1204
Sep 22 23:16:43.681 WARN 1205 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:16:43.681 WARN 1206 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:16:43.681 WARN 1208 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:16:43.681 INFO [lossy] skipping 1209
Sep 22 23:16:43.681 WARN 1211 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:16:43.681 WARN 1212 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:16:43.681 INFO [lossy] skipping 1214
Sep 22 23:16:43.681 INFO [lossy] skipping 1215
Sep 22 23:16:43.681 WARN 1217 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 6 deps, role: work
Sep 22 23:16:43.681 WARN 1218 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 6 deps, role: work
Sep 22 23:16:43.681 INFO [lossy] skipping 1219
Sep 22 23:16:43.681 WARN 1220 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 7 deps, role: work
Sep 22 23:16:43.681 WARN 1221 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 7 deps, role: work
Sep 22 23:16:43.681 INFO [lossy] skipping 1222
Sep 22 23:16:43.681 WARN 1223 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 8 deps, role: work
Sep 22 23:16:43.681 WARN 1224 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 8 deps, role: work
Sep 22 23:16:43.681 INFO [lossy] skipping 1226
Sep 22 23:16:43.681 WARN 1227 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 9 deps, role: work
Sep 22 23:16:43.682 WARN 1229 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 10 deps, role: work
Sep 22 23:16:43.682 WARN 1230 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 10 deps, role: work
Sep 22 23:16:43.682 WARN 1232 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 11 deps, role: work
Sep 22 23:16:43.682 WARN 1233 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 11 deps, role: work
Sep 22 23:16:43.682 WARN 1235 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 12 deps, role: work
Sep 22 23:16:43.682 INFO [lossy] skipping 1238
Sep 22 23:16:43.682 INFO [lossy] skipping 1240
Sep 22 23:16:43.682 INFO [lossy] skipping 1241
Sep 22 23:16:43.682 INFO [lossy] skipping 1243
Sep 22 23:16:43.682 INFO [lossy] skipping 1247
Sep 22 23:16:43.682 INFO [lossy] skipping 1249
Sep 22 23:16:43.682 INFO [lossy] skipping 1199
Sep 22 23:16:43.682 INFO [lossy] skipping 1209
Sep 22 23:16:43.682 INFO [lossy] skipping 1214
Sep 22 23:16:43.682 WARN 1215 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 5 deps, role: work
Sep 22 23:16:43.682 WARN 1226 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 9 deps, role: work
Sep 22 23:16:43.682 INFO [lossy] skipping 1241
Sep 22 23:16:43.682 INFO [lossy] skipping 1249
Sep 22 23:16:43.712 DEBG Write     :1199 deps:[JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:43.713 WARN 1209 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:16:43.713 WARN 1214 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 5 deps, role: work
Sep 22 23:16:43.713 INFO [lossy] skipping 1241
Sep 22 23:16:43.713 INFO [lossy] sleeping 1 second
Sep 22 23:16:43.719 DEBG Read      :1249 deps:[JobId(1248)] res:true
Sep 22 23:16:43.740 ERRO [2] job id 1192 saw error GenericError("test error")
Sep 22 23:16:43.743 DEBG [rc] retire 1192 clears [JobId(1190), JobId(1191), JobId(1192)], : downstairs
Sep 22 23:16:43.743 ERRO [2] job id 1195 saw error GenericError("test error")
Sep 22 23:16:43.746 DEBG [rc] retire 1195 clears [JobId(1193), JobId(1194), JobId(1195)], : downstairs
Sep 22 23:16:43.749 DEBG [rc] retire 1198 clears [JobId(1196), JobId(1197), JobId(1198)], : downstairs
Sep 22 23:16:43.749 ERRO [2] job id 1199 saw error GenericError("test error")
Sep 22 23:16:43.751 DEBG Flush     :1250 extent_limit None deps:[JobId(1249), JobId(1248)] res:true f:103 g:1
Sep 22 23:16:43.751 INFO [lossy] sleeping 1 second
Sep 22 23:16:44.126 DEBG [1] Read AckReady 1249, : downstairs
Sep 22 23:16:44.127 DEBG up_ds_listen was notified
Sep 22 23:16:44.127 DEBG up_ds_listen process 1249
Sep 22 23:16:44.127 DEBG [A] ack job 1249:250, : downstairs
Sep 22 23:16:44.180 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:44.181 WARN returning error on read!
Sep 22 23:16:44.181 DEBG Read      :1249 deps:[JobId(1248)] res:false
Sep 22 23:16:44.187 DEBG Read      :1249 deps:[JobId(1248)] res:true
Sep 22 23:16:44.209 DEBG IO Read  1251 has deps [JobId(1250)]
Sep 22 23:16:44.209 ERRO [0] job id 1249 saw error GenericError("test error")
Sep 22 23:16:44.222 INFO [lossy] sleeping 1 second
Sep 22 23:16:44.682 DEBG IO Flush 1252 has deps [JobId(1251), JobId(1250)]
Sep 22 23:16:44.722 DEBG Flush     :1201 extent_limit None deps:[JobId(1200), JobId(1199), JobId(1198)] res:true f:84 g:1
Sep 22 23:16:44.722 INFO [lossy] skipping 1202
Sep 22 23:16:44.723 WARN returning error on write!
Sep 22 23:16:44.723 DEBG Write     :1203 deps:[JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:false
Sep 22 23:16:44.723 WARN 1204 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:16:44.723 WARN 1205 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:44.724 WARN 1206 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:44.724 WARN 1208 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:16:44.724 WARN 1209 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:16:44.724 WARN 1211 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:16:44.724 WARN 1212 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:16:44.724 WARN 1214 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:16:44.724 WARN 1215 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:16:44.724 INFO [lossy] skipping 1216
Sep 22 23:16:44.724 WARN 1217 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 5 deps, role: work
Sep 22 23:16:44.724 WARN 1218 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 5 deps, role: work
Sep 22 23:16:44.724 INFO [lossy] skipping 1220
Sep 22 23:16:44.724 INFO [lossy] skipping 1221
Sep 22 23:16:44.724 INFO [lossy] skipping 1223
Sep 22 23:16:44.724 INFO [lossy] skipping 1224
Sep 22 23:16:44.724 WARN 1226 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 8 deps, role: work
Sep 22 23:16:44.724 WARN 1227 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 8 deps, role: work
Sep 22 23:16:44.724 INFO [lossy] skipping 1229
Sep 22 23:16:44.724 WARN 1230 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 9 deps, role: work
Sep 22 23:16:44.724 INFO [lossy] skipping 1232
Sep 22 23:16:44.724 WARN 1233 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 10 deps, role: work
Sep 22 23:16:44.724 INFO [lossy] skipping 1234
Sep 22 23:16:44.724 WARN 1235 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 11 deps, role: work
Sep 22 23:16:44.724 INFO [lossy] skipping 1236
Sep 22 23:16:44.724 INFO [lossy] skipping 1237
Sep 22 23:16:44.724 INFO [lossy] skipping 1240
Sep 22 23:16:44.724 INFO [lossy] skipping 1243
Sep 22 23:16:44.724 INFO [lossy] skipping 1244
Sep 22 23:16:44.724 INFO [lossy] skipping 1250
Sep 22 23:16:44.725 WARN returning error on write!
Sep 22 23:16:44.725 DEBG Write     :1202 deps:[JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:false
Sep 22 23:16:44.726 WARN returning error on write!
Sep 22 23:16:44.726 DEBG Write     :1203 deps:[JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:false
Sep 22 23:16:44.726 INFO [lossy] skipping 1216
Sep 22 23:16:44.726 WARN 1220 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 6 deps, role: work
Sep 22 23:16:44.726 WARN 1221 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 6 deps, role: work
Sep 22 23:16:44.726 WARN 1223 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 7 deps, role: work
Sep 22 23:16:44.726 INFO [lossy] skipping 1224
Sep 22 23:16:44.726 INFO [lossy] skipping 1229
Sep 22 23:16:44.726 WARN 1232 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 10 deps, role: work
Sep 22 23:16:44.726 INFO [lossy] skipping 1234
Sep 22 23:16:44.726 INFO [lossy] skipping 1236
Sep 22 23:16:44.726 INFO [lossy] skipping 1237
Sep 22 23:16:44.726 INFO [lossy] skipping 1240
Sep 22 23:16:44.756 DEBG Write     :1202 deps:[JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:44.757 INFO [lossy] skipping 1203
Sep 22 23:16:44.757 INFO [lossy] skipping 1224
Sep 22 23:16:44.757 WARN 1229 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 9 deps, role: work
Sep 22 23:16:44.757 INFO [lossy] skipping 1234
Sep 22 23:16:44.757 INFO [lossy] skipping 1237
Sep 22 23:16:44.757 INFO [lossy] skipping 1240
Sep 22 23:16:44.757 INFO [lossy] skipping 1203
Sep 22 23:16:44.757 INFO [lossy] skipping 1224
Sep 22 23:16:44.757 INFO [lossy] skipping 1234
Sep 22 23:16:44.758 WARN returning error on write!
Sep 22 23:16:44.758 DEBG Write     :1203 deps:[JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:false
Sep 22 23:16:44.758 INFO [lossy] skipping 1224
Sep 22 23:16:44.759 INFO [lossy] skipping 1234
Sep 22 23:16:44.759 INFO [lossy] skipping 1203
Sep 22 23:16:44.759 INFO [lossy] skipping 1224
Sep 22 23:16:44.788 DEBG Write     :1203 deps:[JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:44.789 INFO [lossy] skipping 1224
Sep 22 23:16:44.789 WARN 1224 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 7 deps, role: work
Sep 22 23:16:44.797 DEBG Flush     :1204 extent_limit None deps:[JobId(1203), JobId(1202), JobId(1201)] res:true f:85 g:1
Sep 22 23:16:44.828 DEBG Write     :1205 deps:[JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:44.859 DEBG Write     :1206 deps:[JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:44.868 DEBG Flush     :1207 extent_limit None deps:[JobId(1206), JobId(1205), JobId(1204)] res:true f:86 g:1
Sep 22 23:16:44.898 DEBG Write     :1208 deps:[JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:44.930 DEBG Write     :1209 deps:[JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:44.939 DEBG Flush     :1210 extent_limit None deps:[JobId(1209), JobId(1208), JobId(1207)] res:true f:87 g:1
Sep 22 23:16:44.939 INFO [lossy] skipping 1211
Sep 22 23:16:44.969 DEBG Write     :1212 deps:[JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:44.971 WARN 1213 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:44.971 INFO [lossy] skipping 1214
Sep 22 23:16:44.971 WARN 1215 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:44.971 WARN 1217 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:16:44.971 WARN 1218 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:16:44.971 INFO [lossy] skipping 1219
Sep 22 23:16:44.971 WARN 1220 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:16:44.971 WARN 1221 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:16:44.971 INFO [lossy] skipping 1222
Sep 22 23:16:44.971 WARN 1223 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:16:44.971 WARN 1224 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:16:44.971 WARN 1226 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 5 deps, role: work
Sep 22 23:16:44.971 WARN 1227 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 5 deps, role: work
Sep 22 23:16:44.971 INFO [lossy] skipping 1229
Sep 22 23:16:44.971 WARN 1230 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 6 deps, role: work
Sep 22 23:16:44.971 WARN 1232 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 7 deps, role: work
Sep 22 23:16:44.971 WARN 1233 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 7 deps, role: work
Sep 22 23:16:44.971 INFO [lossy] skipping 1234
Sep 22 23:16:44.971 INFO [lossy] skipping 1235
Sep 22 23:16:44.971 INFO [lossy] skipping 1238
Sep 22 23:16:44.971 INFO [lossy] skipping 1243
Sep 22 23:16:44.971 INFO [lossy] skipping 1244
Sep 22 23:16:44.971 INFO [lossy] skipping 1247
Sep 22 23:16:45.001 DEBG Write     :1211 deps:[JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:45.003 INFO [lossy] skipping 1214
Sep 22 23:16:45.003 INFO [lossy] skipping 1222
Sep 22 23:16:45.003 INFO [lossy] skipping 1229
Sep 22 23:16:45.003 WARN 1235 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 8 deps, role: work
Sep 22 23:16:45.003 INFO [lossy] skipping 1244
Sep 22 23:16:45.003 WARN 1214 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:45.003 WARN 1229 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 6 deps, role: work
Sep 22 23:16:45.003 WARN returning error on flush!
Sep 22 23:16:45.003 DEBG Flush     :1213 extent_limit None deps:[JobId(1212), JobId(1211), JobId(1210)] res:false f:88 g:1
Sep 22 23:16:45.003 INFO [lossy] skipping 1215
Sep 22 23:16:45.003 INFO [lossy] skipping 1226
Sep 22 23:16:45.003 INFO [lossy] skipping 1232
Sep 22 23:16:45.003 INFO [lossy] skipping 1235
Sep 22 23:16:45.003 INFO [lossy] skipping 1238
Sep 22 23:16:45.003 INFO [lossy] skipping 1241
Sep 22 23:16:45.003 INFO [lossy] skipping 1245
Sep 22 23:16:45.003 INFO [lossy] skipping 1249
Sep 22 23:16:45.003 INFO [lossy] skipping 1250
Sep 22 23:16:45.011 DEBG Flush     :1213 extent_limit None deps:[JobId(1212), JobId(1211), JobId(1210)] res:true f:88 g:1
Sep 22 23:16:45.042 DEBG Write     :1215 deps:[JobId(1213), JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:45.043 WARN 1226 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:16:45.043 WARN 1232 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 6 deps, role: work
Sep 22 23:16:45.043 INFO [lossy] skipping 1235
Sep 22 23:16:45.043 INFO [lossy] skipping 1238
Sep 22 23:16:45.043 INFO [lossy] skipping 1249
Sep 22 23:16:45.043 WARN 1235 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 7 deps, role: work
Sep 22 23:16:45.074 DEBG Write     :1214 deps:[JobId(1213), JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:45.083 DEBG Flush     :1216 extent_limit None deps:[JobId(1215), JobId(1214), JobId(1213)] res:true f:89 g:1
Sep 22 23:16:45.113 DEBG Write     :1217 deps:[JobId(1216), JobId(1213), JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:45.145 DEBG Write     :1218 deps:[JobId(1216), JobId(1213), JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:45.146 WARN returning error on flush!
Sep 22 23:16:45.146 DEBG Flush     :1219 extent_limit None deps:[JobId(1218), JobId(1217), JobId(1216)] res:false f:90 g:1
Sep 22 23:16:45.146 WARN 1220 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:45.146 WARN 1221 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:45.146 WARN 1223 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:16:45.146 WARN 1224 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:16:45.146 WARN 1226 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:16:45.146 WARN 1227 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:16:45.146 INFO [lossy] skipping 1228
Sep 22 23:16:45.146 WARN 1229 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:16:45.146 WARN 1230 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:16:45.146 WARN 1232 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 5 deps, role: work
Sep 22 23:16:45.146 WARN 1233 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 5 deps, role: work
Sep 22 23:16:45.146 WARN 1235 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 6 deps, role: work
Sep 22 23:16:45.146 INFO [lossy] skipping 1244
Sep 22 23:16:45.154 DEBG Flush     :1219 extent_limit None deps:[JobId(1218), JobId(1217), JobId(1216)] res:true f:90 g:1
Sep 22 23:16:45.154 INFO [lossy] skipping 1228
Sep 22 23:16:45.154 INFO [lossy] sleeping 1 second
Sep 22 23:16:45.155 WARN returning error on read!
Sep 22 23:16:45.155 DEBG Read      :1251 deps:[JobId(1250)] res:false
Sep 22 23:16:45.161 DEBG Read      :1251 deps:[JobId(1250)] res:true
Sep 22 23:16:45.185 DEBG [rc] retire 1201 clears [JobId(1199), JobId(1200), JobId(1201)], : downstairs
Sep 22 23:16:45.185 ERRO [2] job id 1203 saw error GenericError("test error")
Sep 22 23:16:45.185 ERRO [2] job id 1202 saw error GenericError("test error")
Sep 22 23:16:45.185 ERRO [2] job id 1203 saw error GenericError("test error")
Sep 22 23:16:45.185 ERRO [2] job id 1203 saw error GenericError("test error")
Sep 22 23:16:45.188 DEBG [rc] retire 1204 clears [JobId(1202), JobId(1203), JobId(1204)], : downstairs
Sep 22 23:16:45.191 DEBG [rc] retire 1207 clears [JobId(1205), JobId(1206), JobId(1207)], : downstairs
Sep 22 23:16:45.194 DEBG [rc] retire 1210 clears [JobId(1208), JobId(1209), JobId(1210)], : downstairs
Sep 22 23:16:45.194 ERRO [2] job id 1213 saw error GenericError("test error")
Sep 22 23:16:45.197 DEBG [rc] retire 1213 clears [JobId(1211), JobId(1212), JobId(1213)], : downstairs
Sep 22 23:16:45.200 DEBG [rc] retire 1216 clears [JobId(1214), JobId(1215), JobId(1216)], : downstairs
Sep 22 23:16:45.200 ERRO [2] job id 1219 saw error GenericError("test error")
Sep 22 23:16:45.203 DEBG [rc] retire 1219 clears [JobId(1217), JobId(1218), JobId(1219)], : downstairs
Sep 22 23:16:45.203 ERRO [1] job id 1251 saw error GenericError("test error")
Sep 22 23:16:45.205 DEBG Flush     :1252 extent_limit None deps:[JobId(1251), JobId(1250)] res:true f:104 g:1
Sep 22 23:16:45.205 INFO [lossy] sleeping 1 second
Sep 22 23:16:45.581 DEBG [1] Read AckReady 1251, : downstairs
Sep 22 23:16:45.582 DEBG up_ds_listen was notified
Sep 22 23:16:45.582 DEBG up_ds_listen process 1251
Sep 22 23:16:45.582 DEBG [A] ack job 1251:252, : downstairs
Sep 22 23:16:45.635 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:45.636 DEBG Flush     :1250 extent_limit None deps:[JobId(1249), JobId(1248)] res:true f:103 g:1
Sep 22 23:16:45.641 DEBG Read      :1251 deps:[JobId(1250)] res:true
Sep 22 23:16:45.663 DEBG IO Read  1253 has deps [JobId(1252)]
Sep 22 23:16:45.668 DEBG up_ds_listen was notified
Sep 22 23:16:45.668 DEBG up_ds_listen process 1250
Sep 22 23:16:45.668 DEBG [A] ack job 1250:251, : downstairs
Sep 22 23:16:45.668 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:45.676 DEBG Flush     :1252 extent_limit None deps:[JobId(1251), JobId(1250)] res:true f:104 g:1
Sep 22 23:16:45.682 DEBG Read      :1253 deps:[JobId(1252)] res:true
Sep 22 23:16:46.082 DEBG up_ds_listen was notified
Sep 22 23:16:46.082 DEBG up_ds_listen process 1252
Sep 22 23:16:46.082 DEBG [A] ack job 1252:253, : downstairs
Sep 22 23:16:46.082 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:46.082 DEBG IO Flush 1254 has deps [JobId(1253), JobId(1252)]
Sep 22 23:16:46.084 INFO [lossy] sleeping 1 second
Sep 22 23:16:46.460 DEBG [0] Read AckReady 1253, : downstairs
Sep 22 23:16:46.460 DEBG up_ds_listen was notified
Sep 22 23:16:46.460 DEBG up_ds_listen process 1253
Sep 22 23:16:46.460 DEBG [A] ack job 1253:254, : downstairs
Sep 22 23:16:46.513 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:46.544 DEBG Write     :1220 deps:[JobId(1219), JobId(1216), JobId(1213), JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:46.575 DEBG Write     :1221 deps:[JobId(1219), JobId(1216), JobId(1213), JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:46.582 DEBG Read      :1253 deps:[JobId(1252)] res:true
Sep 22 23:16:46.611 DEBG Flush     :1222 extent_limit None deps:[JobId(1221), JobId(1220), JobId(1219)] res:true f:91 g:1
Sep 22 23:16:46.641 DEBG Write     :1223 deps:[JobId(1222), JobId(1219), JobId(1216), JobId(1213), JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:46.643 WARN returning error on write!
Sep 22 23:16:46.644 DEBG Write     :1224 deps:[JobId(1222), JobId(1219), JobId(1216), JobId(1213), JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:false
Sep 22 23:16:46.644 WARN 1225 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:46.644 INFO [lossy] skipping 1226
Sep 22 23:16:46.644 WARN 1227 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:46.644 INFO [lossy] skipping 1229
Sep 22 23:16:46.644 WARN 1230 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:16:46.644 INFO [lossy] skipping 1231
Sep 22 23:16:46.644 WARN 1232 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:16:46.644 WARN 1233 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:16:46.644 WARN 1235 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:16:46.644 INFO [lossy] skipping 1236
Sep 22 23:16:46.644 INFO [lossy] skipping 1240
Sep 22 23:16:46.644 INFO [lossy] skipping 1242
Sep 22 23:16:46.644 INFO [lossy] skipping 1248
Sep 22 23:16:46.674 DEBG Write     :1224 deps:[JobId(1222), JobId(1219), JobId(1216), JobId(1213), JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:46.675 WARN 1226 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:46.675 WARN 1229 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:16:46.675 INFO [lossy] skipping 1236
Sep 22 23:16:46.675 INFO [lossy] skipping 1248
Sep 22 23:16:46.675 INFO [lossy] skipping 1225
Sep 22 23:16:46.675 INFO [lossy] skipping 1228
Sep 22 23:16:46.675 INFO [lossy] skipping 1238
Sep 22 23:16:46.675 INFO [lossy] skipping 1249
Sep 22 23:16:46.675 INFO [lossy] skipping 1251
Sep 22 23:16:46.675 INFO [lossy] skipping 1225
Sep 22 23:16:46.675 INFO [lossy] skipping 1251
Sep 22 23:16:46.675 WARN returning error on flush!
Sep 22 23:16:46.675 DEBG Flush     :1225 extent_limit None deps:[JobId(1224), JobId(1223), JobId(1222)] res:false f:92 g:1
Sep 22 23:16:46.675 INFO [lossy] skipping 1251
Sep 22 23:16:46.683 DEBG Flush     :1225 extent_limit None deps:[JobId(1224), JobId(1223), JobId(1222)] res:true f:92 g:1
Sep 22 23:16:46.683 INFO [lossy] skipping 1251
Sep 22 23:16:46.683 INFO [lossy] skipping 1251
Sep 22 23:16:46.714 DEBG Write     :1226 deps:[JobId(1225), JobId(1222), JobId(1219), JobId(1216), JobId(1213), JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:46.715 WARN returning error on write!
Sep 22 23:16:46.716 DEBG Write     :1227 deps:[JobId(1225), JobId(1222), JobId(1219), JobId(1216), JobId(1213), JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:false
Sep 22 23:16:46.716 WARN 1228 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:46.716 WARN 1229 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:46.716 WARN 1230 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:46.716 INFO [lossy] skipping 1232
Sep 22 23:16:46.716 WARN 1233 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:16:46.716 INFO [lossy] skipping 1235
Sep 22 23:16:46.716 INFO [lossy] skipping 1238
Sep 22 23:16:46.716 INFO [lossy] skipping 1239
Sep 22 23:16:46.716 INFO [lossy] skipping 1243
Sep 22 23:16:46.716 INFO [lossy] skipping 1244
Sep 22 23:16:46.716 INFO [lossy] skipping 1249
Sep 22 23:16:46.716 INFO [lossy] skipping 1250
Sep 22 23:16:46.746 DEBG Write     :1227 deps:[JobId(1225), JobId(1222), JobId(1219), JobId(1216), JobId(1213), JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:46.747 WARN 1232 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:16:46.747 WARN 1235 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:16:46.747 INFO [lossy] skipping 1239
Sep 22 23:16:46.748 INFO [lossy] skipping 1228
Sep 22 23:16:46.748 INFO [lossy] skipping 1230
Sep 22 23:16:46.748 INFO [lossy] skipping 1231
Sep 22 23:16:46.748 INFO [lossy] skipping 1232
Sep 22 23:16:46.748 INFO [lossy] skipping 1238
Sep 22 23:16:46.748 INFO [lossy] skipping 1239
Sep 22 23:16:46.748 INFO [lossy] skipping 1243
Sep 22 23:16:46.748 INFO [lossy] skipping 1244
Sep 22 23:16:46.748 INFO [lossy] skipping 1247
Sep 22 23:16:46.748 INFO [lossy] skipping 1248
Sep 22 23:16:46.748 INFO [lossy] skipping 1253
Sep 22 23:16:46.756 DEBG Flush     :1228 extent_limit None deps:[JobId(1227), JobId(1226), JobId(1225)] res:true f:93 g:1
Sep 22 23:16:46.757 WARN returning error on write!
Sep 22 23:16:46.757 DEBG Write     :1230 deps:[JobId(1228), JobId(1225), JobId(1222), JobId(1219), JobId(1216), JobId(1213), JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:false
Sep 22 23:16:46.757 WARN 1231 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:16:46.757 WARN 1232 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:46.757 INFO [lossy] skipping 1239
Sep 22 23:16:46.757 INFO [lossy] skipping 1244
Sep 22 23:16:46.757 INFO [lossy] skipping 1247
Sep 22 23:16:46.757 INFO [lossy] skipping 1248
Sep 22 23:16:46.787 DEBG Write     :1230 deps:[JobId(1228), JobId(1225), JobId(1222), JobId(1219), JobId(1216), JobId(1213), JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:46.819 DEBG Write     :1229 deps:[JobId(1228), JobId(1225), JobId(1222), JobId(1219), JobId(1216), JobId(1213), JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:46.820 INFO [lossy] skipping 1231
Sep 22 23:16:46.820 WARN 1233 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:46.820 WARN 1235 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:16:46.820 INFO [lossy] skipping 1236
Sep 22 23:16:46.820 INFO [lossy] skipping 1237
Sep 22 23:16:46.820 INFO [lossy] skipping 1242
Sep 22 23:16:46.820 INFO [lossy] skipping 1243
Sep 22 23:16:46.820 INFO [lossy] skipping 1244
Sep 22 23:16:46.820 INFO [lossy] skipping 1248
Sep 22 23:16:46.820 INFO [lossy] skipping 1249
Sep 22 23:16:46.820 INFO [lossy] skipping 1250
Sep 22 23:16:46.828 DEBG Flush     :1231 extent_limit None deps:[JobId(1230), JobId(1229), JobId(1228)] res:true f:94 g:1
Sep 22 23:16:46.828 INFO [lossy] skipping 1236
Sep 22 23:16:46.828 INFO [lossy] skipping 1237
Sep 22 23:16:46.828 INFO [lossy] skipping 1236
Sep 22 23:16:46.828 INFO [lossy] skipping 1237
Sep 22 23:16:46.828 INFO [lossy] skipping 1236
Sep 22 23:16:46.858 DEBG Write     :1232 deps:[JobId(1231), JobId(1228), JobId(1225), JobId(1222), JobId(1219), JobId(1216), JobId(1213), JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163)] res:true
Sep 22 23:16:46.861 WARN returning error on write!
Sep 22 23:16:46.861 DEBG Write     :1233 deps:[JobId(1231), JobId(1228), JobId(1225), JobId(1222), JobId(1219), JobId(1216), JobId(1213), JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163), JobId(1162)] res:false
Sep 22 23:16:46.861 WARN 1234 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:46.861 WARN 1235 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:46.861 INFO [lossy] skipping 1240
Sep 22 23:16:46.861 INFO [lossy] skipping 1245
Sep 22 23:16:46.861 INFO [lossy] skipping 1253
Sep 22 23:16:46.861 INFO [lossy] skipping 1254
Sep 22 23:16:46.891 DEBG Write     :1233 deps:[JobId(1231), JobId(1228), JobId(1225), JobId(1222), JobId(1219), JobId(1216), JobId(1213), JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1163), JobId(1162)] res:true
Sep 22 23:16:46.900 DEBG Flush     :1234 extent_limit None deps:[JobId(1233), JobId(1232), JobId(1231)] res:true f:95 g:1
Sep 22 23:16:46.931 DEBG Write     :1235 deps:[JobId(1234), JobId(1231), JobId(1228), JobId(1225), JobId(1222), JobId(1219), JobId(1216), JobId(1213), JobId(1210), JobId(1207), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1189), JobId(1186), JobId(1183), JobId(1180), JobId(1177), JobId(1174), JobId(1171), JobId(1168), JobId(1165), JobId(1164), JobId(1163)] res:true
Sep 22 23:16:46.936 DEBG Flush     :1236 extent_limit None deps:[JobId(1235), JobId(1234)] res:true f:96 g:1
Sep 22 23:16:46.942 DEBG Read      :1237 deps:[JobId(1236)] res:true
Sep 22 23:16:46.964 DEBG IO Read  1255 has deps [JobId(1254)]
Sep 22 23:16:46.964 DEBG IO Flush 1256 has deps [JobId(1255), JobId(1254)]
Sep 22 23:16:46.966 DEBG [rc] retire 1222 clears [JobId(1220), JobId(1221), JobId(1222)], : downstairs
Sep 22 23:16:46.967 ERRO [2] job id 1224 saw error GenericError("test error")
Sep 22 23:16:46.967 ERRO [2] job id 1225 saw error GenericError("test error")
Sep 22 23:16:46.970 DEBG [rc] retire 1225 clears [JobId(1223), JobId(1224), JobId(1225)], : downstairs
Sep 22 23:16:46.970 ERRO [2] job id 1227 saw error GenericError("test error")
Sep 22 23:16:46.973 DEBG [rc] retire 1228 clears [JobId(1226), JobId(1227), JobId(1228)], : downstairs
Sep 22 23:16:46.973 ERRO [2] job id 1230 saw error GenericError("test error")
Sep 22 23:16:46.977 DEBG [rc] retire 1231 clears [JobId(1229), JobId(1230), JobId(1231)], : downstairs
Sep 22 23:16:46.977 ERRO [2] job id 1233 saw error GenericError("test error")
Sep 22 23:16:46.980 DEBG [rc] retire 1234 clears [JobId(1232), JobId(1233), JobId(1234)], : downstairs
Sep 22 23:16:46.981 DEBG [rc] retire 1236 clears [JobId(1235), JobId(1236)], : downstairs
Sep 22 23:16:46.996 DEBG Flush     :1254 extent_limit None deps:[JobId(1253), JobId(1252)] res:true f:105 g:1
Sep 22 23:16:46.996 INFO [lossy] skipping 1255
Sep 22 23:16:46.997 WARN returning error on read!
Sep 22 23:16:46.997 DEBG Read      :1255 deps:[JobId(1254)] res:false
Sep 22 23:16:47.002 DEBG Read      :1255 deps:[JobId(1254)] res:true
Sep 22 23:16:47.025 WARN returning error on flush!
Sep 22 23:16:47.025 DEBG Flush     :1238 extent_limit None deps:[JobId(1237), JobId(1236)] res:false f:97 g:1
Sep 22 23:16:47.025 INFO [lossy] skipping 1246
Sep 22 23:16:47.025 INFO [lossy] skipping 1247
Sep 22 23:16:47.025 INFO [lossy] skipping 1248
Sep 22 23:16:47.025 INFO [lossy] skipping 1251
Sep 22 23:16:47.025 DEBG Flush     :1238 extent_limit None deps:[JobId(1237), JobId(1236)] res:true f:97 g:1
Sep 22 23:16:47.025 INFO [lossy] skipping 1246
Sep 22 23:16:47.025 INFO [lossy] skipping 1248
Sep 22 23:16:47.025 INFO [lossy] skipping 1239
Sep 22 23:16:47.025 WARN 1240 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:47.025 INFO [lossy] skipping 1243
Sep 22 23:16:47.025 INFO [lossy] skipping 1248
Sep 22 23:16:47.025 INFO [lossy] skipping 1251
Sep 22 23:16:47.031 DEBG Read      :1239 deps:[JobId(1238)] res:true
Sep 22 23:16:47.480 ERRO [2] job id 1238 saw error GenericError("test error")
Sep 22 23:16:47.480 DEBG [rc] retire 1238 clears [JobId(1237), JobId(1238)], : downstairs
Sep 22 23:16:47.813 ERRO [1] job id 1255 saw error GenericError("test error")
Sep 22 23:16:47.813 INFO [lossy] skipping 1254
Sep 22 23:16:47.813 DEBG Flush     :1254 extent_limit None deps:[JobId(1253), JobId(1252)] res:true f:105 g:1
Sep 22 23:16:47.819 DEBG Read      :1255 deps:[JobId(1254)] res:true
Sep 22 23:16:47.840 DEBG up_ds_listen was notified
Sep 22 23:16:47.840 DEBG up_ds_listen process 1254
Sep 22 23:16:47.840 DEBG [A] ack job 1254:255, : downstairs
Sep 22 23:16:47.840 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:47.843 DEBG Flush     :1256 extent_limit None deps:[JobId(1255), JobId(1254)] res:true f:106 g:1
Sep 22 23:16:47.843 INFO [lossy] sleeping 1 second
Sep 22 23:16:47.845 INFO [lossy] skipping 1243
Sep 22 23:16:47.845 INFO [lossy] skipping 1248
Sep 22 23:16:47.845 DEBG Flush     :1240 extent_limit None deps:[JobId(1239), JobId(1238)] res:true f:98 g:1
Sep 22 23:16:47.851 DEBG Read      :1241 deps:[JobId(1240)] res:true
Sep 22 23:16:48.302 DEBG [rc] retire 1240 clears [JobId(1239), JobId(1240)], : downstairs
Sep 22 23:16:48.632 DEBG [1] Read AckReady 1255, : downstairs
Sep 22 23:16:48.633 DEBG up_ds_listen was notified
Sep 22 23:16:48.633 DEBG up_ds_listen process 1255
Sep 22 23:16:48.633 DEBG [A] ack job 1255:256, : downstairs
Sep 22 23:16:48.686 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:48.688 DEBG Flush     :1256 extent_limit None deps:[JobId(1255), JobId(1254)] res:true f:106 g:1
Sep 22 23:16:48.688 INFO [lossy] sleeping 1 second
Sep 22 23:16:48.736 DEBG IO Read  1257 has deps [JobId(1256)]
Sep 22 23:16:49.074 DEBG up_ds_listen was notified
Sep 22 23:16:49.074 DEBG up_ds_listen process 1256
Sep 22 23:16:49.074 DEBG [A] ack job 1256:257, : downstairs
Sep 22 23:16:49.074 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:49.081 INFO [lossy] sleeping 1 second
Sep 22 23:16:49.083 WARN returning error on flush!
Sep 22 23:16:49.083 DEBG Flush     :1242 extent_limit None deps:[JobId(1241), JobId(1240)] res:false f:99 g:1
Sep 22 23:16:49.083 INFO [lossy] skipping 1244
Sep 22 23:16:49.083 INFO [lossy] skipping 1247
Sep 22 23:16:49.083 INFO [lossy] skipping 1252
Sep 22 23:16:49.083 INFO [lossy] skipping 1253
Sep 22 23:16:49.083 INFO [lossy] skipping 1255
Sep 22 23:16:49.083 DEBG Flush     :1242 extent_limit None deps:[JobId(1241), JobId(1240)] res:true f:99 g:1
Sep 22 23:16:49.083 WARN 1244 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:49.083 INFO [lossy] skipping 1253
Sep 22 23:16:49.084 INFO [lossy] skipping 1255
Sep 22 23:16:49.084 INFO [lossy] skipping 1253
Sep 22 23:16:49.084 INFO [lossy] skipping 1255
Sep 22 23:16:49.084 INFO [lossy] skipping 1253
Sep 22 23:16:49.090 DEBG Read      :1243 deps:[JobId(1242)] res:true
Sep 22 23:16:49.490 ERRO [2] job id 1242 saw error GenericError("test error")
Sep 22 23:16:49.490 DEBG [rc] retire 1242 clears [JobId(1241), JobId(1242)], : downstairs
Sep 22 23:16:49.490 DEBG IO Flush 1258 has deps [JobId(1257), JobId(1256)]
Sep 22 23:16:49.493 WARN returning error on flush!
Sep 22 23:16:49.493 DEBG Flush     :1244 extent_limit None deps:[JobId(1243), JobId(1242)] res:false f:100 g:1
Sep 22 23:16:49.493 INFO [lossy] skipping 1245
Sep 22 23:16:49.493 INFO [lossy] skipping 1246
Sep 22 23:16:49.493 INFO [lossy] skipping 1249
Sep 22 23:16:49.493 INFO [lossy] skipping 1254
Sep 22 23:16:49.493 DEBG Flush     :1244 extent_limit None deps:[JobId(1243), JobId(1242)] res:true f:100 g:1
Sep 22 23:16:49.499 DEBG Read      :1245 deps:[JobId(1244)] res:true
Sep 22 23:16:49.899 ERRO [2] job id 1244 saw error GenericError("test error")
Sep 22 23:16:49.899 DEBG [rc] retire 1244 clears [JobId(1243), JobId(1244)], : downstairs
Sep 22 23:16:49.905 DEBG Read      :1257 deps:[JobId(1256)] res:true
Sep 22 23:16:49.929 WARN returning error on flush!
Sep 22 23:16:49.929 DEBG Flush     :1246 extent_limit None deps:[JobId(1245), JobId(1244)] res:false f:101 g:1
Sep 22 23:16:49.929 INFO [lossy] skipping 1254
Sep 22 23:16:49.929 WARN returning error on flush!
Sep 22 23:16:49.929 DEBG Flush     :1246 extent_limit None deps:[JobId(1245), JobId(1244)] res:false f:101 g:1
Sep 22 23:16:49.929 DEBG Flush     :1246 extent_limit None deps:[JobId(1245), JobId(1244)] res:true f:101 g:1
Sep 22 23:16:49.929 INFO [lossy] skipping 1247
Sep 22 23:16:49.929 WARN 1248 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:49.929 INFO [lossy] skipping 1257
Sep 22 23:16:49.929 INFO [lossy] skipping 1258
Sep 22 23:16:49.935 DEBG Read      :1247 deps:[JobId(1246)] res:true
Sep 22 23:16:50.336 ERRO [2] job id 1246 saw error GenericError("test error")
Sep 22 23:16:50.336 ERRO [2] job id 1246 saw error GenericError("test error")
Sep 22 23:16:50.336 DEBG [rc] retire 1246 clears [JobId(1245), JobId(1246)], : downstairs
Sep 22 23:16:50.338 DEBG Flush     :1258 extent_limit None deps:[JobId(1257), JobId(1256)] res:true f:107 g:1
Sep 22 23:16:50.338 INFO [lossy] sleeping 1 second
Sep 22 23:16:50.344 DEBG Read      :1257 deps:[JobId(1256)] res:true
Sep 22 23:16:50.743 DEBG [0] Read AckReady 1257, : downstairs
Sep 22 23:16:50.744 DEBG up_ds_listen was notified
Sep 22 23:16:50.744 DEBG up_ds_listen process 1257
Sep 22 23:16:50.744 DEBG [A] ack job 1257:258, : downstairs
Sep 22 23:16:50.797 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:50.798 DEBG IO Read  1259 has deps [JobId(1258)]
Sep 22 23:16:50.813 INFO [lossy] skipping 1258
Sep 22 23:16:50.813 INFO [lossy] skipping 1258
Sep 22 23:16:50.813 DEBG Flush     :1258 extent_limit None deps:[JobId(1257), JobId(1256)] res:true f:107 g:1
Sep 22 23:16:50.819 DEBG Read      :1259 deps:[JobId(1258)] res:true
Sep 22 23:16:50.842 WARN returning error on flush!
Sep 22 23:16:50.842 DEBG Flush     :1248 extent_limit None deps:[JobId(1247), JobId(1246)] res:false f:102 g:1
Sep 22 23:16:50.842 INFO [lossy] skipping 1249
Sep 22 23:16:50.842 INFO [lossy] skipping 1252
Sep 22 23:16:50.842 INFO [lossy] skipping 1258
Sep 22 23:16:50.842 DEBG Flush     :1248 extent_limit None deps:[JobId(1247), JobId(1246)] res:true f:102 g:1
Sep 22 23:16:50.842 INFO [lossy] skipping 1249
Sep 22 23:16:50.842 INFO [lossy] skipping 1252
Sep 22 23:16:50.842 INFO [lossy] skipping 1249
Sep 22 23:16:50.842 INFO [lossy] skipping 1252
Sep 22 23:16:50.848 DEBG Read      :1249 deps:[JobId(1248)] res:true
Sep 22 23:16:50.965 DEBG IO Flush 1260 has deps [JobId(1259), JobId(1258)]
Sep 22 23:16:51.297 ERRO [2] job id 1248 saw error GenericError("test error")
Sep 22 23:16:51.298 DEBG [rc] retire 1248 clears [JobId(1247), JobId(1248)], : downstairs
Sep 22 23:16:51.629 DEBG up_ds_listen was notified
Sep 22 23:16:51.630 DEBG up_ds_listen process 1258
Sep 22 23:16:51.630 DEBG [A] ack job 1258:259, : downstairs
Sep 22 23:16:51.630 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:51.636 DEBG Read      :1259 deps:[JobId(1258)] res:true
Sep 22 23:16:51.660 INFO [lossy] sleeping 1 second
Sep 22 23:16:51.662 DEBG Flush     :1250 extent_limit None deps:[JobId(1249), JobId(1248)] res:true f:103 g:1
Sep 22 23:16:51.662 WARN returning error on read!
Sep 22 23:16:51.662 DEBG Read      :1251 deps:[JobId(1250)] res:false
Sep 22 23:16:51.662 WARN 1252 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:51.662 INFO [lossy] skipping 1253
Sep 22 23:16:51.662 WARN returning error on read!
Sep 22 23:16:51.662 DEBG Read      :1251 deps:[JobId(1250)] res:false
Sep 22 23:16:51.662 INFO [lossy] skipping 1251
Sep 22 23:16:51.663 INFO [lossy] skipping 1251
Sep 22 23:16:51.663 INFO [lossy] skipping 1251
Sep 22 23:16:51.663 WARN returning error on read!
Sep 22 23:16:51.663 DEBG Read      :1251 deps:[JobId(1250)] res:false
Sep 22 23:16:51.669 DEBG Read      :1251 deps:[JobId(1250)] res:true
Sep 22 23:16:52.118 DEBG [rc] retire 1250 clears [JobId(1249), JobId(1250)], : downstairs
Sep 22 23:16:52.118 ERRO [2] job id 1251 saw error GenericError("test error")
Sep 22 23:16:52.118 ERRO [2] job id 1251 saw error GenericError("test error")
Sep 22 23:16:52.118 ERRO [2] job id 1251 saw error GenericError("test error")
Sep 22 23:16:52.448 DEBG [1] Read AckReady 1259, : downstairs
Sep 22 23:16:52.449 DEBG up_ds_listen was notified
Sep 22 23:16:52.449 DEBG up_ds_listen process 1259
Sep 22 23:16:52.449 DEBG [A] ack job 1259:260, : downstairs
Sep 22 23:16:52.503 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:52.505 INFO [lossy] sleeping 1 second
Sep 22 23:16:52.553 DEBG IO Read  1261 has deps [JobId(1260)]
Sep 22 23:16:52.896 DEBG Flush     :1260 extent_limit None deps:[JobId(1259), JobId(1258)] res:true f:108 g:1
Sep 22 23:16:52.896 INFO [lossy] sleeping 1 second
Sep 22 23:16:52.899 DEBG Flush     :1252 extent_limit None deps:[JobId(1251), JobId(1250)] res:true f:104 g:1
Sep 22 23:16:52.905 DEBG Read      :1253 deps:[JobId(1252)] res:true
Sep 22 23:16:53.305 DEBG [rc] retire 1252 clears [JobId(1251), JobId(1252)], : downstairs
Sep 22 23:16:53.305 DEBG IO Flush 1262 has deps [JobId(1261), JobId(1260)]
Sep 22 23:16:53.308 DEBG Flush     :1254 extent_limit None deps:[JobId(1253), JobId(1252)] res:true f:105 g:1
Sep 22 23:16:53.309 WARN returning error on read!
Sep 22 23:16:53.309 DEBG Read      :1255 deps:[JobId(1254)] res:false
Sep 22 23:16:53.309 WARN 1256 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:53.309 INFO [lossy] skipping 1257
Sep 22 23:16:53.309 INFO [lossy] skipping 1258
Sep 22 23:16:53.309 INFO [lossy] skipping 1260
Sep 22 23:16:53.315 DEBG Read      :1255 deps:[JobId(1254)] res:true
Sep 22 23:16:53.717 DEBG [rc] retire 1254 clears [JobId(1253), JobId(1254)], : downstairs
Sep 22 23:16:53.717 ERRO [2] job id 1255 saw error GenericError("test error")
Sep 22 23:16:53.717 DEBG Flush     :1260 extent_limit None deps:[JobId(1259), JobId(1258)] res:true f:108 g:1
Sep 22 23:16:53.723 DEBG Read      :1261 deps:[JobId(1260)] res:true
Sep 22 23:16:53.744 DEBG up_ds_listen was notified
Sep 22 23:16:53.745 DEBG up_ds_listen process 1260
Sep 22 23:16:53.745 DEBG [A] ack job 1260:261, : downstairs
Sep 22 23:16:53.745 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:53.747 INFO [lossy] skipping 1256
Sep 22 23:16:53.747 INFO [lossy] skipping 1257
Sep 22 23:16:53.747 INFO [lossy] skipping 1262
Sep 22 23:16:53.747 INFO [lossy] skipping 1256
Sep 22 23:16:53.747 INFO [lossy] skipping 1257
Sep 22 23:16:53.747 DEBG Flush     :1256 extent_limit None deps:[JobId(1255), JobId(1254)] res:true f:106 g:1
Sep 22 23:16:53.747 INFO [lossy] skipping 1257
Sep 22 23:16:53.747 INFO [lossy] skipping 1257
Sep 22 23:16:53.753 DEBG Read      :1257 deps:[JobId(1256)] res:true
Sep 22 23:16:54.153 DEBG [rc] retire 1256 clears [JobId(1255), JobId(1256)], : downstairs
Sep 22 23:16:54.155 DEBG Flush     :1262 extent_limit None deps:[JobId(1261), JobId(1260)] res:true f:109 g:1
Sep 22 23:16:54.155 INFO [lossy] sleeping 1 second
Sep 22 23:16:54.161 DEBG Read      :1261 deps:[JobId(1260)] res:true
Sep 22 23:16:54.560 DEBG [0] Read AckReady 1261, : downstairs
Sep 22 23:16:54.561 DEBG up_ds_listen was notified
Sep 22 23:16:54.561 DEBG up_ds_listen process 1261
Sep 22 23:16:54.561 DEBG [A] ack job 1261:262, : downstairs
Sep 22 23:16:54.614 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:54.615 DEBG IO Read  1263 has deps [JobId(1262)]
Sep 22 23:16:54.630 INFO [lossy] skipping 1258
Sep 22 23:16:54.630 INFO [lossy] skipping 1259
Sep 22 23:16:54.630 INFO [lossy] skipping 1263
Sep 22 23:16:54.630 DEBG Flush     :1258 extent_limit None deps:[JobId(1257), JobId(1256)] res:true f:107 g:1
Sep 22 23:16:54.636 DEBG Read      :1259 deps:[JobId(1258)] res:true
Sep 22 23:16:54.705 DEBG IO Flush 1264 has deps [JobId(1263), JobId(1262)]
Sep 22 23:16:55.037 DEBG [rc] retire 1258 clears [JobId(1257), JobId(1258)], : downstairs
Sep 22 23:16:55.039 DEBG Flush     :1262 extent_limit None deps:[JobId(1261), JobId(1260)] res:true f:109 g:1
Sep 22 23:16:55.039 INFO [lossy] skipping 1263
Sep 22 23:16:55.039 WARN returning error on read!
Sep 22 23:16:55.039 DEBG Read      :1263 deps:[JobId(1262)] res:false
Sep 22 23:16:55.045 DEBG Read      :1263 deps:[JobId(1262)] res:true
Sep 22 23:16:55.445 ERRO [1] job id 1263 saw error GenericError("test error")
Sep 22 23:16:55.445 DEBG up_ds_listen was notified
Sep 22 23:16:55.445 DEBG up_ds_listen process 1262
Sep 22 23:16:55.445 DEBG [A] ack job 1262:263, : downstairs
Sep 22 23:16:55.445 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:55.446 INFO [lossy] skipping 1263
Sep 22 23:16:55.446 WARN returning error on read!
Sep 22 23:16:55.446 DEBG Read      :1263 deps:[JobId(1262)] res:false
Sep 22 23:16:55.446 WARN returning error on read!
Sep 22 23:16:55.446 DEBG Read      :1263 deps:[JobId(1262)] res:false
Sep 22 23:16:55.452 DEBG Read      :1263 deps:[JobId(1262)] res:true
Sep 22 23:16:55.474 ERRO [0] job id 1263 saw error GenericError("test error")
Sep 22 23:16:55.474 ERRO [0] job id 1263 saw error GenericError("test error")
Sep 22 23:16:55.476 INFO [lossy] sleeping 1 second
Sep 22 23:16:55.477 INFO [lossy] skipping 1260
Sep 22 23:16:55.477 INFO [lossy] skipping 1262
Sep 22 23:16:55.478 DEBG Flush     :1260 extent_limit None deps:[JobId(1259), JobId(1258)] res:true f:108 g:1
Sep 22 23:16:55.478 INFO [lossy] skipping 1262
Sep 22 23:16:55.478 WARN 1262 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:16:55.478 INFO [lossy] skipping 1261
Sep 22 23:16:55.478 INFO [lossy] skipping 1264
Sep 22 23:16:55.483 DEBG Read      :1261 deps:[JobId(1260)] res:true
Sep 22 23:16:55.935 DEBG [rc] retire 1260 clears [JobId(1259), JobId(1260)], : downstairs
Sep 22 23:16:56.269 DEBG [1] Read AckReady 1263, : downstairs
Sep 22 23:16:56.270 DEBG up_ds_listen was notified
Sep 22 23:16:56.270 DEBG up_ds_listen process 1263
Sep 22 23:16:56.270 DEBG [A] ack job 1263:264, : downstairs
Sep 22 23:16:56.324 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:56.327 DEBG Flush     :1264 extent_limit None deps:[JobId(1263), JobId(1262)] res:true f:110 g:1
Sep 22 23:16:56.327 INFO [lossy] sleeping 1 second
Sep 22 23:16:56.375 DEBG IO Read  1265 has deps [JobId(1264)]
Sep 22 23:16:56.722 WARN returning error on flush!
Sep 22 23:16:56.722 DEBG Flush     :1264 extent_limit None deps:[JobId(1263), JobId(1262)] res:false f:110 g:1
Sep 22 23:16:56.722 DEBG Flush     :1264 extent_limit None deps:[JobId(1263), JobId(1262)] res:true f:110 g:1
Sep 22 23:16:56.722 INFO [lossy] sleeping 1 second
Sep 22 23:16:56.723 ERRO [1] job id 1264 saw error GenericError("test error")
Sep 22 23:16:56.723 DEBG up_ds_listen was notified
Sep 22 23:16:56.723 DEBG up_ds_listen process 1264
Sep 22 23:16:56.723 DEBG [A] ack job 1264:265, : downstairs
Sep 22 23:16:56.723 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:56.725 DEBG Flush     :1262 extent_limit None deps:[JobId(1261), JobId(1260)] res:true f:109 g:1
Sep 22 23:16:56.731 DEBG Read      :1263 deps:[JobId(1262)] res:true
Sep 22 23:16:57.135 DEBG [rc] retire 1262 clears [JobId(1261), JobId(1262)], : downstairs
Sep 22 23:16:57.135 DEBG IO Flush 1266 has deps [JobId(1265), JobId(1264)]
Sep 22 23:16:57.138 INFO [lossy] skipping 1264
Sep 22 23:16:57.138 INFO [lossy] skipping 1264
Sep 22 23:16:57.138 INFO [lossy] skipping 1264
Sep 22 23:16:57.138 WARN returning error on flush!
Sep 22 23:16:57.138 DEBG Flush     :1264 extent_limit None deps:[JobId(1263), JobId(1262)] res:false f:110 g:1
Sep 22 23:16:57.138 INFO [lossy] skipping 1264
Sep 22 23:16:57.138 DEBG Flush     :1264 extent_limit None deps:[JobId(1263), JobId(1262)] res:true f:110 g:1
Sep 22 23:16:57.144 DEBG Read      :1265 deps:[JobId(1264)] res:true
Sep 22 23:16:57.547 ERRO [2] job id 1264 saw error GenericError("test error")
Sep 22 23:16:57.547 DEBG [rc] retire 1264 clears [JobId(1263), JobId(1264)], : downstairs
Sep 22 23:16:57.553 DEBG Read      :1265 deps:[JobId(1264)] res:true
Sep 22 23:16:57.577 DEBG Flush     :1266 extent_limit None deps:[JobId(1265), JobId(1264)] res:true f:111 g:1
Sep 22 23:16:57.577 INFO [lossy] sleeping 1 second
Sep 22 23:16:57.579 INFO [lossy] skipping 1266
Sep 22 23:16:57.579 WARN returning error on flush!
Sep 22 23:16:57.579 DEBG Flush     :1266 extent_limit None deps:[JobId(1265), JobId(1264)] res:false f:111 g:1
Sep 22 23:16:57.579 DEBG Flush     :1266 extent_limit None deps:[JobId(1265), JobId(1264)] res:true f:111 g:1
Sep 22 23:16:57.579 INFO [lossy] sleeping 1 second
Sep 22 23:16:58.009 DEBG [0] Read AckReady 1265, : downstairs
Sep 22 23:16:58.010 ERRO [0] job id 1266 saw error GenericError("test error")
Sep 22 23:16:58.343 DEBG [2] Read already AckReady 1265, : downstairs
Sep 22 23:16:58.345 DEBG up_ds_listen was notified
Sep 22 23:16:58.345 DEBG up_ds_listen process 1265
Sep 22 23:16:58.345 DEBG [A] ack job 1265:266, : downstairs
Sep 22 23:16:58.399 DEBG up_ds_listen process 1266
Sep 22 23:16:58.399 DEBG [A] ack job 1266:267, : downstairs
Sep 22 23:16:58.399 DEBG up_ds_listen checked 2 jobs, back to waiting
Sep 22 23:16:58.399 DEBG up_ds_listen was notified
Sep 22 23:16:58.399 DEBG up_ds_listen checked 0 jobs, back to waiting
Sep 22 23:16:58.406 DEBG Read      :1265 deps:[JobId(1264)] res:true
Sep 22 23:16:58.428 DEBG IO Read  1267 has deps [JobId(1266)]
Sep 22 23:16:58.441 DEBG Flush     :1266 extent_limit None deps:[JobId(1265), JobId(1264)] res:true f:111 g:1
Sep 22 23:16:58.447 DEBG Read      :1267 deps:[JobId(1266)] res:true
Sep 22 23:16:58.852 DEBG [rc] retire 1266 clears [JobId(1265), JobId(1266)], : downstairs
Sep 22 23:16:58.852 INFO [lossy] skipping 1267
Sep 22 23:16:58.852 WARN returning error on read!
Sep 22 23:16:58.852 DEBG Read      :1267 deps:[JobId(1266)] res:false
Sep 22 23:16:58.858 DEBG Read      :1267 deps:[JobId(1266)] res:true
Sep 22 23:16:58.880 INFO [lossy] skipping 1267
Sep 22 23:16:58.880 INFO [lossy] skipping 1267
Sep 22 23:16:58.880 INFO [lossy] skipping 1267
Sep 22 23:16:58.886 DEBG Read      :1267 deps:[JobId(1266)] res:true
Sep 22 23:16:58.907 DEBG IO Flush 1268 has deps [JobId(1267)]
Sep 22 23:16:58.908 ERRO [2] job id 1267 saw error GenericError("test error")
Sep 22 23:16:58.910 INFO [lossy] sleeping 1 second
Sep 22 23:16:59.291 DEBG [1] Read AckReady 1267, : downstairs
Sep 22 23:16:59.292 DEBG up_ds_listen was notified
Sep 22 23:16:59.292 DEBG up_ds_listen process 1267
Sep 22 23:16:59.292 DEBG [A] ack job 1267:268, : downstairs
Sep 22 23:16:59.345 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:16:59.347 INFO [lossy] sleeping 1 second
Sep 22 23:16:59.349 WARN returning error on flush!
Sep 22 23:16:59.349 DEBG Flush     :1268 extent_limit None deps:[JobId(1267)] res:false f:112 g:1
Sep 22 23:16:59.349 DEBG Flush     :1268 extent_limit None deps:[JobId(1267)] res:true f:112 g:1
Sep 22 23:16:59.349 INFO [lossy] sleeping 1 second
Sep 22 23:16:59.445 DEBG IO Read  1269 has deps [JobId(1268)]
Sep 22 23:16:59.778 ERRO [0] job id 1268 saw error GenericError("test error")
Sep 22 23:17:00.126 DEBG IO Flush 1270 has deps [JobId(1269), JobId(1268)]
Sep 22 23:17:00.126 DEBG Flush     :1268 extent_limit None deps:[JobId(1267)] res:true f:112 g:1
Sep 22 23:17:00.126 INFO [lossy] sleeping 1 second
Sep 22 23:17:00.127 DEBG up_ds_listen was notified
Sep 22 23:17:00.127 DEBG up_ds_listen process 1268
Sep 22 23:17:00.127 DEBG [A] ack job 1268:269, : downstairs
Sep 22 23:17:00.127 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:00.348 DEBG Flush     :1268 extent_limit None deps:[JobId(1267)] res:true f:112 g:1
Sep 22 23:17:00.348 INFO [lossy] skipping 1269
Sep 22 23:17:00.355 DEBG Read      :1269 deps:[JobId(1268)] res:true
Sep 22 23:17:00.377 WARN returning error on read!
Sep 22 23:17:00.377 DEBG Read      :1269 deps:[JobId(1268)] res:false
Sep 22 23:17:00.383 DEBG Read      :1269 deps:[JobId(1268)] res:true
Sep 22 23:17:00.405 DEBG [rc] retire 1268 clears [JobId(1267), JobId(1268)], : downstairs
Sep 22 23:17:00.405 ERRO [0] job id 1269 saw error GenericError("test error")
Sep 22 23:17:00.407 WARN returning error on flush!
Sep 22 23:17:00.407 DEBG Flush     :1270 extent_limit None deps:[JobId(1269), JobId(1268)] res:false f:113 g:1
Sep 22 23:17:00.408 DEBG Flush     :1270 extent_limit None deps:[JobId(1269), JobId(1268)] res:true f:113 g:1
Sep 22 23:17:00.408 INFO [lossy] sleeping 1 second
Sep 22 23:17:00.457 DEBG Flush     :1270 extent_limit None deps:[JobId(1269), JobId(1268)] res:true f:113 g:1
Sep 22 23:17:00.457 INFO [lossy] sleeping 1 second
Sep 22 23:17:00.789 DEBG [2] Read AckReady 1269, : downstairs
Sep 22 23:17:00.790 ERRO [2] job id 1270 saw error GenericError("test error")
Sep 22 23:17:00.790 DEBG up_ds_listen was notified
Sep 22 23:17:00.790 DEBG up_ds_listen process 1269
Sep 22 23:17:00.790 DEBG [A] ack job 1269:270, : downstairs
Sep 22 23:17:00.844 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:01.226 DEBG up_ds_listen was notified
Sep 22 23:17:01.226 DEBG up_ds_listen process 1270
Sep 22 23:17:01.226 DEBG [A] ack job 1270:271, : downstairs
Sep 22 23:17:01.226 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:01.233 DEBG Read      :1269 deps:[JobId(1268)] res:true
Sep 22 23:17:01.254 DEBG IO Read  1271 has deps [JobId(1270)]
Sep 22 23:17:01.268 INFO [lossy] skipping 1270
Sep 22 23:17:01.268 DEBG Flush     :1270 extent_limit None deps:[JobId(1269), JobId(1268)] res:true f:113 g:1
Sep 22 23:17:01.274 DEBG Read      :1271 deps:[JobId(1270)] res:true
Sep 22 23:17:01.678 DEBG [rc] retire 1270 clears [JobId(1269), JobId(1270)], : downstairs
Sep 22 23:17:01.678 DEBG IO Flush 1272 has deps [JobId(1271)]
Sep 22 23:17:01.678 WARN returning error on read!
Sep 22 23:17:01.678 DEBG Read      :1271 deps:[JobId(1270)] res:false
Sep 22 23:17:01.684 DEBG Read      :1271 deps:[JobId(1270)] res:true
Sep 22 23:17:01.712 DEBG Read      :1271 deps:[JobId(1270)] res:true
Sep 22 23:17:01.734 ERRO [2] job id 1271 saw error GenericError("test error")
Sep 22 23:17:01.737 INFO [lossy] sleeping 1 second
Sep 22 23:17:02.119 DEBG [1] Read AckReady 1271, : downstairs
Sep 22 23:17:02.120 DEBG up_ds_listen was notified
Sep 22 23:17:02.120 DEBG up_ds_listen process 1271
Sep 22 23:17:02.120 DEBG [A] ack job 1271:272, : downstairs
Sep 22 23:17:02.174 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:02.177 DEBG Flush     :1272 extent_limit None deps:[JobId(1271)] res:true f:114 g:1
Sep 22 23:17:02.177 INFO [lossy] sleeping 1 second
Sep 22 23:17:02.178 INFO [lossy] skipping 1272
Sep 22 23:17:02.178 INFO [lossy] skipping 1272
Sep 22 23:17:02.178 INFO [lossy] skipping 1272
Sep 22 23:17:02.178 DEBG Flush     :1272 extent_limit None deps:[JobId(1271)] res:true f:114 g:1
Sep 22 23:17:02.178 INFO [lossy] sleeping 1 second
Sep 22 23:17:02.276 DEBG IO Read  1273 has deps [JobId(1272)]
Sep 22 23:17:02.952 DEBG up_ds_listen was notified
Sep 22 23:17:02.952 DEBG up_ds_listen process 1272
Sep 22 23:17:02.952 DEBG [A] ack job 1272:273, : downstairs
Sep 22 23:17:02.952 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:02.959 DEBG IO Flush 1274 has deps [JobId(1273), JobId(1272)]
Sep 22 23:17:02.959 DEBG Flush     :1272 extent_limit None deps:[JobId(1271)] res:true f:114 g:1
Sep 22 23:17:02.959 INFO [lossy] sleeping 1 second
Sep 22 23:17:02.960 DEBG [rc] retire 1272 clears [JobId(1271), JobId(1272)], : downstairs
Sep 22 23:17:03.178 WARN returning error on read!
Sep 22 23:17:03.178 DEBG Read      :1273 deps:[JobId(1272)] res:false
Sep 22 23:17:03.178 INFO [lossy] skipping 1274
Sep 22 23:17:03.179 WARN returning error on read!
Sep 22 23:17:03.179 DEBG Read      :1273 deps:[JobId(1272)] res:false
Sep 22 23:17:03.185 DEBG Read      :1273 deps:[JobId(1272)] res:true
Sep 22 23:17:03.214 DEBG Read      :1273 deps:[JobId(1272)] res:true
Sep 22 23:17:03.236 ERRO [2] job id 1273 saw error GenericError("test error")
Sep 22 23:17:03.236 ERRO [2] job id 1273 saw error GenericError("test error")
Sep 22 23:17:03.239 DEBG Flush     :1274 extent_limit None deps:[JobId(1273), JobId(1272)] res:true f:115 g:1
Sep 22 23:17:03.239 INFO [lossy] sleeping 1 second
Sep 22 23:17:03.288 DEBG Flush     :1274 extent_limit None deps:[JobId(1273), JobId(1272)] res:true f:115 g:1
Sep 22 23:17:03.288 INFO [lossy] sleeping 1 second
Sep 22 23:17:03.621 DEBG [2] Read AckReady 1273, : downstairs
Sep 22 23:17:03.622 DEBG up_ds_listen was notified
Sep 22 23:17:03.622 DEBG up_ds_listen process 1273
Sep 22 23:17:03.622 DEBG [A] ack job 1273:274, : downstairs
Sep 22 23:17:03.675 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:04.060 DEBG up_ds_listen was notified
Sep 22 23:17:04.061 DEBG up_ds_listen process 1274
Sep 22 23:17:04.061 DEBG [A] ack job 1274:275, : downstairs
Sep 22 23:17:04.061 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:04.061 INFO [lossy] skipping 1273
Sep 22 23:17:04.067 DEBG Read      :1273 deps:[JobId(1272)] res:true
Sep 22 23:17:04.088 DEBG IO Read  1275 has deps [JobId(1274)]
Sep 22 23:17:04.102 INFO [lossy] skipping 1274
Sep 22 23:17:04.102 DEBG Flush     :1274 extent_limit None deps:[JobId(1273), JobId(1272)] res:true f:115 g:1
Sep 22 23:17:04.102 INFO [lossy] sleeping 1 second
Sep 22 23:17:04.484 DEBG [rc] retire 1274 clears [JobId(1273), JobId(1274)], : downstairs
Sep 22 23:17:04.484 DEBG IO Flush 1276 has deps [JobId(1275)]
Sep 22 23:17:04.485 INFO [lossy] skipping 1275
Sep 22 23:17:04.490 DEBG Read      :1275 deps:[JobId(1274)] res:true
Sep 22 23:17:04.512 INFO [lossy] skipping 1275
Sep 22 23:17:04.518 DEBG Read      :1275 deps:[JobId(1274)] res:true
Sep 22 23:17:04.542 DEBG Flush     :1276 extent_limit None deps:[JobId(1275)] res:true f:116 g:1
Sep 22 23:17:04.542 INFO [lossy] sleeping 1 second
Sep 22 23:17:04.543 INFO [lossy] sleeping 1 second
Sep 22 23:17:04.972 DEBG [0] Read AckReady 1275, : downstairs
Sep 22 23:17:05.305 DEBG [2] Read already AckReady 1275, : downstairs
Sep 22 23:17:05.307 DEBG up_ds_listen was notified
Sep 22 23:17:05.307 DEBG up_ds_listen process 1275
Sep 22 23:17:05.307 DEBG [A] ack job 1275:276, : downstairs
Sep 22 23:17:05.361 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:05.367 DEBG Read      :1275 deps:[JobId(1274)] res:true
Sep 22 23:17:05.389 DEBG IO Read  1277 has deps [JobId(1276)]
Sep 22 23:17:05.403 DEBG Flush     :1276 extent_limit None deps:[JobId(1275)] res:true f:116 g:1
Sep 22 23:17:05.408 DEBG Read      :1277 deps:[JobId(1276)] res:true
Sep 22 23:17:05.812 DEBG up_ds_listen was notified
Sep 22 23:17:05.813 DEBG up_ds_listen process 1276
Sep 22 23:17:05.813 DEBG [A] ack job 1276:277, : downstairs
Sep 22 23:17:05.813 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:05.813 INFO [lossy] skipping 1277
Sep 22 23:17:05.813 INFO [lossy] skipping 1277
Sep 22 23:17:05.819 DEBG Read      :1277 deps:[JobId(1276)] res:true
Sep 22 23:17:05.841 DEBG Flush     :1276 extent_limit None deps:[JobId(1275)] res:true f:116 g:1
Sep 22 23:17:05.848 DEBG Read      :1277 deps:[JobId(1276)] res:true
Sep 22 23:17:05.870 DEBG IO Flush 1278 has deps [JobId(1277), JobId(1276)]
Sep 22 23:17:05.870 DEBG [rc] retire 1276 clears [JobId(1275), JobId(1276)], : downstairs
Sep 22 23:17:05.873 INFO [lossy] sleeping 1 second
Sep 22 23:17:06.254 DEBG [1] Read AckReady 1277, : downstairs
Sep 22 23:17:06.255 DEBG up_ds_listen was notified
Sep 22 23:17:06.255 DEBG up_ds_listen process 1277
Sep 22 23:17:06.255 DEBG [A] ack job 1277:278, : downstairs
Sep 22 23:17:06.308 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:06.311 INFO [lossy] sleeping 1 second
Sep 22 23:17:06.312 INFO [lossy] sleeping 1 second
Sep 22 23:17:06.409 DEBG IO Read  1279 has deps [JobId(1278)]
Sep 22 23:17:07.091 DEBG IO Flush 1280 has deps [JobId(1279), JobId(1278)]
Sep 22 23:17:07.091 DEBG Flush     :1278 extent_limit None deps:[JobId(1277), JobId(1276)] res:true f:117 g:1
Sep 22 23:17:07.091 INFO [lossy] sleeping 1 second
Sep 22 23:17:07.312 DEBG Flush     :1278 extent_limit None deps:[JobId(1277), JobId(1276)] res:true f:117 g:1
Sep 22 23:17:07.312 INFO [lossy] skipping 1279
Sep 22 23:17:07.312 INFO [lossy] skipping 1280
Sep 22 23:17:07.312 INFO [lossy] skipping 1279
Sep 22 23:17:07.319 DEBG Read      :1279 deps:[JobId(1278)] res:true
Sep 22 23:17:07.340 DEBG Flush     :1278 extent_limit None deps:[JobId(1277), JobId(1276)] res:true f:117 g:1
Sep 22 23:17:07.340 INFO [lossy] skipping 1279
Sep 22 23:17:07.341 WARN returning error on read!
Sep 22 23:17:07.341 DEBG Read      :1279 deps:[JobId(1278)] res:false
Sep 22 23:17:07.341 INFO [lossy] skipping 1279
Sep 22 23:17:07.347 DEBG Read      :1279 deps:[JobId(1278)] res:true
Sep 22 23:17:07.368 DEBG up_ds_listen was notified
Sep 22 23:17:07.368 DEBG up_ds_listen process 1278
Sep 22 23:17:07.368 DEBG [A] ack job 1278:279, : downstairs
Sep 22 23:17:07.368 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:07.369 DEBG [rc] retire 1278 clears [JobId(1277), JobId(1278)], : downstairs
Sep 22 23:17:07.369 ERRO [0] job id 1279 saw error GenericError("test error")
Sep 22 23:17:07.371 INFO [lossy] sleeping 1 second
Sep 22 23:17:07.419 INFO [lossy] skipping 1280
Sep 22 23:17:07.419 DEBG Flush     :1280 extent_limit None deps:[JobId(1279), JobId(1278)] res:true f:118 g:1
Sep 22 23:17:07.420 INFO [lossy] sleeping 1 second
Sep 22 23:17:07.752 DEBG [2] Read AckReady 1279, : downstairs
Sep 22 23:17:07.753 DEBG up_ds_listen was notified
Sep 22 23:17:07.753 DEBG up_ds_listen process 1279
Sep 22 23:17:07.753 DEBG [A] ack job 1279:280, : downstairs
Sep 22 23:17:07.806 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:08.191 INFO [lossy] skipping 1279
Sep 22 23:17:08.191 INFO [lossy] skipping 1279
Sep 22 23:17:08.197 DEBG Read      :1279 deps:[JobId(1278)] res:true
Sep 22 23:17:08.219 DEBG IO Read  1281 has deps [JobId(1280)]
Sep 22 23:17:08.233 DEBG Flush     :1280 extent_limit None deps:[JobId(1279), JobId(1278)] res:true f:118 g:1
Sep 22 23:17:08.239 DEBG Read      :1281 deps:[JobId(1280)] res:true
Sep 22 23:17:08.643 DEBG up_ds_listen was notified
Sep 22 23:17:08.643 DEBG up_ds_listen process 1280
Sep 22 23:17:08.643 DEBG [A] ack job 1280:281, : downstairs
Sep 22 23:17:08.643 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:08.643 DEBG IO Flush 1282 has deps [JobId(1281), JobId(1280)]
Sep 22 23:17:08.644 WARN returning error on flush!
Sep 22 23:17:08.644 DEBG Flush     :1280 extent_limit None deps:[JobId(1279), JobId(1278)] res:false f:118 g:1
Sep 22 23:17:08.644 INFO [lossy] skipping 1281
Sep 22 23:17:08.644 INFO [lossy] skipping 1280
Sep 22 23:17:08.644 INFO [lossy] skipping 1280
Sep 22 23:17:08.644 INFO [lossy] skipping 1280
Sep 22 23:17:08.644 INFO [lossy] skipping 1280
Sep 22 23:17:08.644 DEBG Flush     :1280 extent_limit None deps:[JobId(1279), JobId(1278)] res:true f:118 g:1
Sep 22 23:17:08.650 DEBG Read      :1281 deps:[JobId(1280)] res:true
Sep 22 23:17:08.678 DEBG Read      :1281 deps:[JobId(1280)] res:true
Sep 22 23:17:08.700 ERRO [2] job id 1280 saw error GenericError("test error")
Sep 22 23:17:08.700 DEBG [rc] retire 1280 clears [JobId(1279), JobId(1280)], : downstairs
Sep 22 23:17:08.703 INFO [lossy] sleeping 1 second
Sep 22 23:17:09.086 DEBG [1] Read AckReady 1281, : downstairs
Sep 22 23:17:09.087 DEBG up_ds_listen was notified
Sep 22 23:17:09.087 DEBG up_ds_listen process 1281
Sep 22 23:17:09.087 DEBG [A] ack job 1281:282, : downstairs
Sep 22 23:17:09.141 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:09.143 INFO [lossy] sleeping 1 second
Sep 22 23:17:09.144 INFO [lossy] sleeping 1 second
Sep 22 23:17:09.241 DEBG IO Read  1283 has deps [JobId(1282)]
Sep 22 23:17:09.924 DEBG IO Flush 1284 has deps [JobId(1283), JobId(1282)]
Sep 22 23:17:09.924 DEBG Flush     :1282 extent_limit None deps:[JobId(1281), JobId(1280)] res:true f:119 g:1
Sep 22 23:17:09.924 INFO [lossy] sleeping 1 second
Sep 22 23:17:10.143 WARN returning error on flush!
Sep 22 23:17:10.143 DEBG Flush     :1282 extent_limit None deps:[JobId(1281), JobId(1280)] res:false f:119 g:1
Sep 22 23:17:10.143 DEBG Flush     :1282 extent_limit None deps:[JobId(1281), JobId(1280)] res:true f:119 g:1
Sep 22 23:17:10.150 DEBG Read      :1283 deps:[JobId(1282)] res:true
Sep 22 23:17:10.172 WARN returning error on flush!
Sep 22 23:17:10.172 DEBG Flush     :1282 extent_limit None deps:[JobId(1281), JobId(1280)] res:false f:119 g:1
Sep 22 23:17:10.172 INFO [lossy] skipping 1284
Sep 22 23:17:10.172 INFO [lossy] skipping 1282
Sep 22 23:17:10.172 DEBG Flush     :1282 extent_limit None deps:[JobId(1281), JobId(1280)] res:true f:119 g:1
Sep 22 23:17:10.173 WARN returning error on read!
Sep 22 23:17:10.173 DEBG Read      :1283 deps:[JobId(1282)] res:false
Sep 22 23:17:10.173 WARN 1284 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:17:10.173 INFO [lossy] skipping 1283
Sep 22 23:17:10.179 DEBG Read      :1283 deps:[JobId(1282)] res:true
Sep 22 23:17:10.200 ERRO [2] job id 1282 saw error GenericError("test error")
Sep 22 23:17:10.200 DEBG up_ds_listen was notified
Sep 22 23:17:10.200 DEBG up_ds_listen process 1282
Sep 22 23:17:10.200 DEBG [A] ack job 1282:283, : downstairs
Sep 22 23:17:10.200 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:10.200 ERRO [0] job id 1282 saw error GenericError("test error")
Sep 22 23:17:10.201 DEBG [rc] retire 1282 clears [JobId(1281), JobId(1282)], : downstairs
Sep 22 23:17:10.201 ERRO [0] job id 1283 saw error GenericError("test error")
Sep 22 23:17:10.203 DEBG Flush     :1284 extent_limit None deps:[JobId(1283), JobId(1282)] res:true f:120 g:1
Sep 22 23:17:10.203 INFO [lossy] sleeping 1 second
Sep 22 23:17:10.252 INFO [lossy] skipping 1284
Sep 22 23:17:10.252 INFO [lossy] skipping 1284
Sep 22 23:17:10.252 INFO [lossy] skipping 1284
Sep 22 23:17:10.252 DEBG Flush     :1284 extent_limit None deps:[JobId(1283), JobId(1282)] res:true f:120 g:1
Sep 22 23:17:10.252 INFO [lossy] sleeping 1 second
Sep 22 23:17:10.585 DEBG [2] Read AckReady 1283, : downstairs
Sep 22 23:17:10.586 DEBG up_ds_listen was notified
Sep 22 23:17:10.586 DEBG up_ds_listen process 1283
Sep 22 23:17:10.586 DEBG [A] ack job 1283:284, : downstairs
Sep 22 23:17:10.639 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:11.023 DEBG up_ds_listen was notified
Sep 22 23:17:11.023 DEBG up_ds_listen process 1284
Sep 22 23:17:11.023 DEBG [A] ack job 1284:285, : downstairs
Sep 22 23:17:11.023 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:11.023 INFO [lossy] skipping 1283
Sep 22 23:17:11.029 DEBG Read      :1283 deps:[JobId(1282)] res:true
Sep 22 23:17:11.051 DEBG IO Read  1285 has deps [JobId(1284)]
Sep 22 23:17:11.065 INFO [lossy] sleeping 1 second
Sep 22 23:17:11.447 DEBG IO Flush 1286 has deps [JobId(1285), JobId(1284)]
Sep 22 23:17:11.454 DEBG Read      :1285 deps:[JobId(1284)] res:true
Sep 22 23:17:11.481 DEBG Read      :1285 deps:[JobId(1284)] res:true
Sep 22 23:17:11.505 WARN returning error on flush!
Sep 22 23:17:11.505 DEBG Flush     :1286 extent_limit None deps:[JobId(1285), JobId(1284)] res:false f:121 g:1
Sep 22 23:17:11.505 INFO [lossy] skipping 1286
Sep 22 23:17:11.505 DEBG Flush     :1286 extent_limit None deps:[JobId(1285), JobId(1284)] res:true f:121 g:1
Sep 22 23:17:11.505 INFO [lossy] sleeping 1 second
Sep 22 23:17:11.506 WARN returning error on flush!
Sep 22 23:17:11.506 DEBG Flush     :1286 extent_limit None deps:[JobId(1285), JobId(1284)] res:false f:121 g:1
Sep 22 23:17:11.506 WARN returning error on flush!
Sep 22 23:17:11.506 DEBG Flush     :1286 extent_limit None deps:[JobId(1285), JobId(1284)] res:false f:121 g:1
Sep 22 23:17:11.506 DEBG Flush     :1286 extent_limit None deps:[JobId(1285), JobId(1284)] res:true f:121 g:1
Sep 22 23:17:11.506 INFO [lossy] sleeping 1 second
Sep 22 23:17:11.934 DEBG [0] Read AckReady 1285, : downstairs
Sep 22 23:17:11.934 ERRO [0] job id 1286 saw error GenericError("test error")
Sep 22 23:17:11.934 ERRO [0] job id 1286 saw error GenericError("test error")
Sep 22 23:17:12.266 DEBG [2] Read already AckReady 1285, : downstairs
Sep 22 23:17:12.268 ERRO [2] job id 1286 saw error GenericError("test error")
Sep 22 23:17:12.268 DEBG up_ds_listen was notified
Sep 22 23:17:12.268 DEBG up_ds_listen process 1285
Sep 22 23:17:12.268 DEBG [A] ack job 1285:286, : downstairs
Sep 22 23:17:12.322 DEBG up_ds_listen process 1286
Sep 22 23:17:12.322 DEBG [A] ack job 1286:287, : downstairs
Sep 22 23:17:12.322 DEBG up_ds_listen checked 2 jobs, back to waiting
Sep 22 23:17:12.322 DEBG up_ds_listen was notified
Sep 22 23:17:12.322 DEBG up_ds_listen checked 0 jobs, back to waiting
Sep 22 23:17:12.323 WARN returning error on flush!
Sep 22 23:17:12.323 DEBG Flush     :1284 extent_limit None deps:[JobId(1283), JobId(1282)] res:false f:120 g:1
Sep 22 23:17:12.323 INFO [lossy] skipping 1286
Sep 22 23:17:12.323 INFO [lossy] skipping 1284
Sep 22 23:17:12.323 INFO [lossy] skipping 1284
Sep 22 23:17:12.323 INFO [lossy] skipping 1284
Sep 22 23:17:12.323 DEBG Flush     :1284 extent_limit None deps:[JobId(1283), JobId(1282)] res:true f:120 g:1
Sep 22 23:17:12.329 DEBG Read      :1285 deps:[JobId(1284)] res:true
Sep 22 23:17:12.351 DEBG IO Read  1287 has deps [JobId(1286)]
Sep 22 23:17:12.351 ERRO [1] job id 1284 saw error GenericError("test error")
Sep 22 23:17:12.351 DEBG [rc] retire 1284 clears [JobId(1283), JobId(1284)], : downstairs
Sep 22 23:17:12.364 DEBG Flush     :1286 extent_limit None deps:[JobId(1285), JobId(1284)] res:true f:121 g:1
Sep 22 23:17:12.365 INFO [lossy] skipping 1287
Sep 22 23:17:12.365 INFO [lossy] skipping 1287
Sep 22 23:17:12.370 DEBG Read      :1287 deps:[JobId(1286)] res:true
Sep 22 23:17:12.773 DEBG [rc] retire 1286 clears [JobId(1285), JobId(1286)], : downstairs
Sep 22 23:17:12.773 WARN returning error on read!
Sep 22 23:17:12.774 DEBG Read      :1287 deps:[JobId(1286)] res:false
Sep 22 23:17:12.779 DEBG Read      :1287 deps:[JobId(1286)] res:true
Sep 22 23:17:12.801 WARN returning error on read!
Sep 22 23:17:12.801 DEBG Read      :1287 deps:[JobId(1286)] res:false
Sep 22 23:17:12.806 DEBG Read      :1287 deps:[JobId(1286)] res:true
Sep 22 23:17:12.828 DEBG IO Flush 1288 has deps [JobId(1287)]
Sep 22 23:17:12.828 ERRO [0] job id 1287 saw error GenericError("test error")
Sep 22 23:17:12.828 ERRO [2] job id 1287 saw error GenericError("test error")
Sep 22 23:17:12.831 DEBG Flush     :1288 extent_limit None deps:[JobId(1287)] res:true f:122 g:1
Sep 22 23:17:12.831 INFO [lossy] sleeping 1 second
Sep 22 23:17:13.211 DEBG [1] Read AckReady 1287, : downstairs
Sep 22 23:17:13.212 DEBG up_ds_listen was notified
Sep 22 23:17:13.212 DEBG up_ds_listen process 1287
Sep 22 23:17:13.212 DEBG [A] ack job 1287:288, : downstairs
Sep 22 23:17:13.265 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:13.267 WARN returning error on flush!
Sep 22 23:17:13.267 DEBG Flush     :1288 extent_limit None deps:[JobId(1287)] res:false f:122 g:1
Sep 22 23:17:13.267 DEBG Flush     :1288 extent_limit None deps:[JobId(1287)] res:true f:122 g:1
Sep 22 23:17:13.267 INFO [lossy] sleeping 1 second
Sep 22 23:17:13.269 DEBG Flush     :1288 extent_limit None deps:[JobId(1287)] res:true f:122 g:1
Sep 22 23:17:13.269 INFO [lossy] sleeping 1 second
Sep 22 23:17:13.364 DEBG IO Read  1289 has deps [JobId(1288)]
Sep 22 23:17:14.032 ERRO [2] job id 1288 saw error GenericError("test error")
Sep 22 23:17:14.037 DEBG up_ds_listen was notified
Sep 22 23:17:14.037 DEBG up_ds_listen process 1288
Sep 22 23:17:14.037 DEBG [A] ack job 1288:289, : downstairs
Sep 22 23:17:14.037 DEBG [rc] retire 1288 clears [JobId(1287), JobId(1288)], : downstairs
Sep 22 23:17:14.037 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:14.044 DEBG IO Flush 1290 has deps [JobId(1289)]
Sep 22 23:17:14.050 DEBG Read      :1289 deps:[JobId(1288)] res:true
Sep 22 23:17:14.073 INFO [lossy] skipping 1290
Sep 22 23:17:14.073 DEBG Flush     :1290 extent_limit None deps:[JobId(1289)] res:true f:123 g:1
Sep 22 23:17:14.452 DEBG [1] Read AckReady 1289, : downstairs
Sep 22 23:17:14.453 DEBG up_ds_listen was notified
Sep 22 23:17:14.453 DEBG up_ds_listen process 1289
Sep 22 23:17:14.453 DEBG [A] ack job 1289:290, : downstairs
Sep 22 23:17:14.506 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:14.507 INFO [lossy] skipping 1289
Sep 22 23:17:14.513 DEBG Read      :1289 deps:[JobId(1288)] res:true
Sep 22 23:17:14.540 DEBG Read      :1289 deps:[JobId(1288)] res:true
Sep 22 23:17:14.562 DEBG IO Read  1291 has deps [JobId(1290)]
Sep 22 23:17:14.562 DEBG IO Flush 1292 has deps [JobId(1291), JobId(1290)]
Sep 22 23:17:14.574 INFO [lossy] skipping 1291
Sep 22 23:17:14.581 DEBG Read      :1291 deps:[JobId(1290)] res:true
Sep 22 23:17:14.605 INFO [lossy] sleeping 1 second
Sep 22 23:17:14.605 INFO [lossy] skipping 1290
Sep 22 23:17:14.605 DEBG Flush     :1290 extent_limit None deps:[JobId(1289)] res:true f:123 g:1
Sep 22 23:17:14.606 INFO [lossy] sleeping 1 second
Sep 22 23:17:15.368 DEBG up_ds_listen was notified
Sep 22 23:17:15.368 DEBG up_ds_listen process 1290
Sep 22 23:17:15.368 DEBG [A] ack job 1290:291, : downstairs
Sep 22 23:17:15.368 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:15.370 INFO [lossy] sleeping 1 second
Sep 22 23:17:15.749 DEBG [1] Read AckReady 1291, : downstairs
Sep 22 23:17:15.750 DEBG up_ds_listen was notified
Sep 22 23:17:15.750 DEBG up_ds_listen process 1291
Sep 22 23:17:15.750 DEBG [A] ack job 1291:292, : downstairs
Sep 22 23:17:15.803 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:15.804 WARN returning error on flush!
Sep 22 23:17:15.804 DEBG Flush     :1290 extent_limit None deps:[JobId(1289)] res:false f:123 g:1
Sep 22 23:17:15.804 INFO [lossy] skipping 1291
Sep 22 23:17:15.804 INFO [lossy] skipping 1292
Sep 22 23:17:15.804 DEBG Flush     :1290 extent_limit None deps:[JobId(1289)] res:true f:123 g:1
Sep 22 23:17:15.810 DEBG Read      :1291 deps:[JobId(1290)] res:true
Sep 22 23:17:15.837 DEBG Read      :1291 deps:[JobId(1290)] res:true
Sep 22 23:17:15.859 DEBG IO Read  1293 has deps [JobId(1292)]
Sep 22 23:17:15.859 ERRO [2] job id 1290 saw error GenericError("test error")
Sep 22 23:17:15.859 DEBG [rc] retire 1290 clears [JobId(1289), JobId(1290)], : downstairs
Sep 22 23:17:15.872 DEBG IO Flush 1294 has deps [JobId(1293), JobId(1292)]
Sep 22 23:17:15.874 DEBG Flush     :1292 extent_limit None deps:[JobId(1291), JobId(1290)] res:true f:124 g:1
Sep 22 23:17:15.874 WARN returning error on read!
Sep 22 23:17:15.874 DEBG Read      :1293 deps:[JobId(1292)] res:false
Sep 22 23:17:15.880 DEBG Read      :1293 deps:[JobId(1292)] res:true
Sep 22 23:17:15.902 DEBG Flush     :1292 extent_limit None deps:[JobId(1291), JobId(1290)] res:true f:124 g:1
Sep 22 23:17:15.908 DEBG Read      :1293 deps:[JobId(1292)] res:true
Sep 22 23:17:16.693 ERRO [2] job id 1293 saw error GenericError("test error")
Sep 22 23:17:16.693 DEBG up_ds_listen was notified
Sep 22 23:17:16.693 DEBG up_ds_listen process 1292
Sep 22 23:17:16.693 DEBG [A] ack job 1292:293, : downstairs
Sep 22 23:17:16.693 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:16.693 DEBG Flush     :1292 extent_limit None deps:[JobId(1291), JobId(1290)] res:true f:124 g:1
Sep 22 23:17:16.699 DEBG Read      :1293 deps:[JobId(1292)] res:true
Sep 22 23:17:16.721 DEBG [rc] retire 1292 clears [JobId(1291), JobId(1292)], : downstairs
Sep 22 23:17:16.724 WARN returning error on flush!
Sep 22 23:17:16.724 DEBG Flush     :1294 extent_limit None deps:[JobId(1293), JobId(1292)] res:false f:125 g:1
Sep 22 23:17:16.724 DEBG Flush     :1294 extent_limit None deps:[JobId(1293), JobId(1292)] res:true f:125 g:1
Sep 22 23:17:16.724 INFO [lossy] sleeping 1 second
Sep 22 23:17:16.725 DEBG Flush     :1294 extent_limit None deps:[JobId(1293), JobId(1292)] res:true f:125 g:1
Sep 22 23:17:16.725 INFO [lossy] sleeping 1 second
Sep 22 23:17:17.154 DEBG [0] Read AckReady 1293, : downstairs
Sep 22 23:17:17.487 DEBG [2] Read already AckReady 1293, : downstairs
Sep 22 23:17:17.488 ERRO [2] job id 1294 saw error GenericError("test error")
Sep 22 23:17:17.488 DEBG up_ds_listen was notified
Sep 22 23:17:17.488 DEBG up_ds_listen process 1293
Sep 22 23:17:17.488 DEBG [A] ack job 1293:294, : downstairs
Sep 22 23:17:17.542 DEBG up_ds_listen process 1294
Sep 22 23:17:17.542 DEBG [A] ack job 1294:295, : downstairs
Sep 22 23:17:17.542 DEBG up_ds_listen checked 2 jobs, back to waiting
Sep 22 23:17:17.542 DEBG up_ds_listen was notified
Sep 22 23:17:17.542 DEBG up_ds_listen checked 0 jobs, back to waiting
Sep 22 23:17:17.544 DEBG Flush     :1294 extent_limit None deps:[JobId(1293), JobId(1292)] res:true f:125 g:1
Sep 22 23:17:17.544 INFO [lossy] sleeping 1 second
Sep 22 23:17:17.592 DEBG IO Read  1295 has deps [JobId(1294)]
Sep 22 23:17:17.926 DEBG [rc] retire 1294 clears [JobId(1293), JobId(1294)], : downstairs
Sep 22 23:17:17.938 INFO [lossy] sleeping 1 second
Sep 22 23:17:17.938 INFO [lossy] sleeping 1 second
Sep 22 23:17:18.045 DEBG IO Flush 1296 has deps [JobId(1295)]
Sep 22 23:17:18.552 DEBG Read      :1295 deps:[JobId(1294)] res:true
Sep 22 23:17:18.575 DEBG Flush     :1296 extent_limit None deps:[JobId(1295)] res:true f:126 g:1
Sep 22 23:17:18.575 INFO [lossy] sleeping 1 second
Sep 22 23:17:18.951 DEBG [1] Read AckReady 1295, : downstairs
Sep 22 23:17:18.952 DEBG up_ds_listen was notified
Sep 22 23:17:18.952 DEBG up_ds_listen process 1295
Sep 22 23:17:18.952 DEBG [A] ack job 1295:296, : downstairs
Sep 22 23:17:19.004 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:19.011 DEBG Read      :1295 deps:[JobId(1294)] res:true
Sep 22 23:17:19.039 DEBG Read      :1295 deps:[JobId(1294)] res:true
Sep 22 23:17:19.061 DEBG IO Read  1297 has deps [JobId(1296)]
Sep 22 23:17:19.074 DEBG IO Flush 1298 has deps [JobId(1297), JobId(1296)]
Sep 22 23:17:19.075 DEBG Flush     :1296 extent_limit None deps:[JobId(1295)] res:true f:126 g:1
Sep 22 23:17:19.081 DEBG Read      :1297 deps:[JobId(1296)] res:true
Sep 22 23:17:19.103 DEBG Flush     :1296 extent_limit None deps:[JobId(1295)] res:true f:126 g:1
Sep 22 23:17:19.109 DEBG Read      :1297 deps:[JobId(1296)] res:true
Sep 22 23:17:19.886 DEBG up_ds_listen was notified
Sep 22 23:17:19.886 DEBG up_ds_listen process 1296
Sep 22 23:17:19.887 DEBG [A] ack job 1296:297, : downstairs
Sep 22 23:17:19.887 DEBG [rc] retire 1296 clears [JobId(1295), JobId(1296)], : downstairs
Sep 22 23:17:19.887 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:19.887 INFO [lossy] skipping 1297
Sep 22 23:17:19.893 DEBG Read      :1297 deps:[JobId(1296)] res:true
Sep 22 23:17:19.917 DEBG Flush     :1298 extent_limit None deps:[JobId(1297), JobId(1296)] res:true f:127 g:1
Sep 22 23:17:19.917 INFO [lossy] sleeping 1 second
Sep 22 23:17:19.919 INFO [lossy] skipping 1298
Sep 22 23:17:19.919 INFO [lossy] skipping 1298
Sep 22 23:17:19.919 DEBG Flush     :1298 extent_limit None deps:[JobId(1297), JobId(1296)] res:true f:127 g:1
Sep 22 23:17:20.345 DEBG [0] Read AckReady 1297, : downstairs
Sep 22 23:17:20.674 DEBG [2] Read already AckReady 1297, : downstairs
Sep 22 23:17:20.676 DEBG up_ds_listen was notified
Sep 22 23:17:20.676 DEBG up_ds_listen process 1297
Sep 22 23:17:20.676 DEBG [A] ack job 1297:298, : downstairs
Sep 22 23:17:20.730 DEBG up_ds_listen process 1298
Sep 22 23:17:20.730 DEBG [A] ack job 1298:299, : downstairs
Sep 22 23:17:20.730 DEBG up_ds_listen checked 2 jobs, back to waiting
Sep 22 23:17:20.730 DEBG up_ds_listen was notified
Sep 22 23:17:20.730 DEBG up_ds_listen checked 0 jobs, back to waiting
Sep 22 23:17:20.732 INFO [lossy] skipping 1298
Sep 22 23:17:20.732 WARN returning error on flush!
Sep 22 23:17:20.732 DEBG Flush     :1298 extent_limit None deps:[JobId(1297), JobId(1296)] res:false f:127 g:1
Sep 22 23:17:20.732 DEBG Flush     :1298 extent_limit None deps:[JobId(1297), JobId(1296)] res:true f:127 g:1
Sep 22 23:17:20.780 DEBG IO Read  1299 has deps [JobId(1298)]
Sep 22 23:17:21.111 ERRO [1] job id 1298 saw error GenericError("test error")
Sep 22 23:17:21.111 DEBG [rc] retire 1298 clears [JobId(1297), JobId(1298)], : downstairs
Sep 22 23:17:21.123 INFO [lossy] sleeping 1 second
Sep 22 23:17:21.129 DEBG Read      :1299 deps:[JobId(1298)] res:true
Sep 22 23:17:21.156 DEBG Read      :1299 deps:[JobId(1298)] res:true
Sep 22 23:17:21.604 DEBG [0] Read AckReady 1299, : downstairs
Sep 22 23:17:21.934 DEBG [1] Read already AckReady 1299, : downstairs
Sep 22 23:17:21.936 DEBG up_ds_listen was notified
Sep 22 23:17:21.936 DEBG up_ds_listen process 1299
Sep 22 23:17:21.936 DEBG [A] ack job 1299:300, : downstairs
Sep 22 23:17:21.989 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:21.990 DEBG IO Flush 1300 has deps [JobId(1299)]
Sep 22 23:17:21.990 INFO [lossy] sleeping 1 second
Sep 22 23:17:21.991 DEBG IO Read  1301 has deps [JobId(1300)]
Sep 22 23:17:21.996 INFO [lossy] sleeping 1 second
Sep 22 23:17:22.130 DEBG Read      :1299 deps:[JobId(1298)] res:true
Sep 22 23:17:22.153 DEBG Flush     :1300 extent_limit None deps:[JobId(1299)] res:true f:128 g:1
Sep 22 23:17:22.158 DEBG Read      :1301 deps:[JobId(1300)] res:true
Sep 22 23:17:22.559 DEBG IO Flush 1302 has deps [JobId(1301), JobId(1300)]
Sep 22 23:17:22.561 DEBG Flush     :1302 extent_limit None deps:[JobId(1301), JobId(1300)] res:true f:129 g:1
Sep 22 23:17:22.562 INFO [lossy] sleeping 1 second
Sep 22 23:17:22.937 DEBG [2] Read AckReady 1301, : downstairs
Sep 22 23:17:22.938 DEBG up_ds_listen was notified
Sep 22 23:17:22.938 DEBG up_ds_listen process 1301
Sep 22 23:17:22.938 DEBG [A] ack job 1301:302, : downstairs
Sep 22 23:17:22.991 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:22.992 INFO [lossy] skipping 1300
Sep 22 23:17:22.992 DEBG Flush     :1300 extent_limit None deps:[JobId(1299)] res:true f:128 g:1
Sep 22 23:17:22.992 WARN returning error on read!
Sep 22 23:17:22.992 DEBG Read      :1301 deps:[JobId(1300)] res:false
Sep 22 23:17:22.992 INFO [lossy] skipping 1302
Sep 22 23:17:22.992 INFO [lossy] skipping 1301
Sep 22 23:17:22.992 WARN 1302 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:17:22.992 INFO [lossy] skipping 1301
Sep 22 23:17:22.992 WARN returning error on read!
Sep 22 23:17:22.992 DEBG Read      :1301 deps:[JobId(1300)] res:false
Sep 22 23:17:22.992 INFO [lossy] skipping 1301
Sep 22 23:17:22.992 INFO [lossy] skipping 1301
Sep 22 23:17:22.993 WARN returning error on read!
Sep 22 23:17:22.993 DEBG Read      :1301 deps:[JobId(1300)] res:false
Sep 22 23:17:22.999 DEBG Read      :1301 deps:[JobId(1300)] res:true
Sep 22 23:17:23.020 DEBG Flush     :1300 extent_limit None deps:[JobId(1299)] res:true f:128 g:1
Sep 22 23:17:23.020 WARN returning error on read!
Sep 22 23:17:23.020 DEBG Read      :1301 deps:[JobId(1300)] res:false
Sep 22 23:17:23.027 DEBG Read      :1301 deps:[JobId(1300)] res:true
Sep 22 23:17:23.049 DEBG IO Read  1303 has deps [JobId(1302)]
Sep 22 23:17:23.049 ERRO [1] job id 1301 saw error GenericError("test error")
Sep 22 23:17:23.049 ERRO [1] job id 1301 saw error GenericError("test error")
Sep 22 23:17:23.049 ERRO [1] job id 1301 saw error GenericError("test error")
Sep 22 23:17:23.054 DEBG up_ds_listen was notified
Sep 22 23:17:23.054 DEBG up_ds_listen process 1300
Sep 22 23:17:23.054 DEBG [A] ack job 1300:301, : downstairs
Sep 22 23:17:23.054 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:23.061 DEBG [rc] retire 1300 clears [JobId(1299), JobId(1300)], : downstairs
Sep 22 23:17:23.061 ERRO [0] job id 1301 saw error GenericError("test error")
Sep 22 23:17:23.061 DEBG IO Flush 1304 has deps [JobId(1303), JobId(1302)]
Sep 22 23:17:23.064 DEBG Flush     :1302 extent_limit None deps:[JobId(1301), JobId(1300)] res:true f:129 g:1
Sep 22 23:17:23.064 WARN returning error on read!
Sep 22 23:17:23.064 DEBG Read      :1303 deps:[JobId(1302)] res:false
Sep 22 23:17:23.064 WARN returning error on read!
Sep 22 23:17:23.064 DEBG Read      :1303 deps:[JobId(1302)] res:false
Sep 22 23:17:23.064 WARN returning error on read!
Sep 22 23:17:23.064 DEBG Read      :1303 deps:[JobId(1302)] res:false
Sep 22 23:17:23.070 DEBG Read      :1303 deps:[JobId(1302)] res:true
Sep 22 23:17:23.140 DEBG Flush     :1302 extent_limit None deps:[JobId(1301), JobId(1300)] res:true f:129 g:1
Sep 22 23:17:23.140 INFO [lossy] skipping 1303
Sep 22 23:17:23.140 WARN returning error on read!
Sep 22 23:17:23.140 DEBG Read      :1303 deps:[JobId(1302)] res:false
Sep 22 23:17:23.140 INFO [lossy] skipping 1303
Sep 22 23:17:23.146 DEBG Read      :1303 deps:[JobId(1302)] res:true
Sep 22 23:17:23.498 ERRO [1] job id 1303 saw error GenericError("test error")
Sep 22 23:17:23.499 ERRO [1] job id 1303 saw error GenericError("test error")
Sep 22 23:17:23.499 ERRO [1] job id 1303 saw error GenericError("test error")
Sep 22 23:17:23.499 DEBG up_ds_listen was notified
Sep 22 23:17:23.499 DEBG up_ds_listen process 1302
Sep 22 23:17:23.499 DEBG [A] ack job 1302:303, : downstairs
Sep 22 23:17:23.499 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:23.877 DEBG [rc] retire 1302 clears [JobId(1301), JobId(1302)], : downstairs
Sep 22 23:17:23.877 ERRO [0] job id 1303 saw error GenericError("test error")
Sep 22 23:17:23.883 DEBG Read      :1303 deps:[JobId(1302)] res:true
Sep 22 23:17:23.907 INFO [lossy] skipping 1304
Sep 22 23:17:23.907 DEBG Flush     :1304 extent_limit None deps:[JobId(1303), JobId(1302)] res:true f:130 g:1
Sep 22 23:17:23.955 INFO [lossy] skipping 1304
Sep 22 23:17:23.955 WARN returning error on flush!
Sep 22 23:17:23.955 DEBG Flush     :1304 extent_limit None deps:[JobId(1303), JobId(1302)] res:false f:130 g:1
Sep 22 23:17:23.955 DEBG Flush     :1304 extent_limit None deps:[JobId(1303), JobId(1302)] res:true f:130 g:1
Sep 22 23:17:23.956 INFO [lossy] sleeping 1 second
Sep 22 23:17:24.284 DEBG [1] Read AckReady 1303, : downstairs
Sep 22 23:17:24.285 DEBG up_ds_listen was notified
Sep 22 23:17:24.285 DEBG up_ds_listen process 1303
Sep 22 23:17:24.286 DEBG [A] ack job 1303:304, : downstairs
Sep 22 23:17:24.339 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:24.718 ERRO [0] job id 1304 saw error GenericError("test error")
Sep 22 23:17:24.719 DEBG up_ds_listen was notified
Sep 22 23:17:24.719 DEBG up_ds_listen process 1304
Sep 22 23:17:24.719 DEBG [A] ack job 1304:305, : downstairs
Sep 22 23:17:24.719 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:24.720 DEBG Flush     :1304 extent_limit None deps:[JobId(1303), JobId(1302)] res:true f:130 g:1
Sep 22 23:17:24.720 INFO [lossy] sleeping 1 second
Sep 22 23:17:24.720 DEBG IO Read  1305 has deps [JobId(1304)]
Sep 22 23:17:25.111 DEBG [rc] retire 1304 clears [JobId(1303), JobId(1304)], : downstairs
Sep 22 23:17:25.117 DEBG Read      :1305 deps:[JobId(1304)] res:true
Sep 22 23:17:25.139 INFO [lossy] skipping 1305
Sep 22 23:17:25.139 INFO [lossy] skipping 1305
Sep 22 23:17:25.144 DEBG Read      :1305 deps:[JobId(1304)] res:true
Sep 22 23:17:25.545 DEBG [1] Read AckReady 1305, : downstairs
Sep 22 23:17:25.545 DEBG up_ds_listen was notified
Sep 22 23:17:25.545 DEBG up_ds_listen process 1305
Sep 22 23:17:25.545 DEBG [A] ack job 1305:306, : downstairs
Sep 22 23:17:25.599 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:25.647 DEBG IO Flush 1306 has deps [JobId(1305)]
Sep 22 23:17:25.984 DEBG Read      :1305 deps:[JobId(1304)] res:true
Sep 22 23:17:26.006 DEBG IO Read  1307 has deps [JobId(1306)]
Sep 22 23:17:26.011 DEBG Flush     :1306 extent_limit None deps:[JobId(1305)] res:true f:131 g:1
Sep 22 23:17:26.011 DEBG Flush     :1306 extent_limit None deps:[JobId(1305)] res:true f:131 g:1
Sep 22 23:17:26.018 DEBG up_ds_listen was notified
Sep 22 23:17:26.018 DEBG up_ds_listen process 1306
Sep 22 23:17:26.018 DEBG [A] ack job 1306:307, : downstairs
Sep 22 23:17:26.018 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:26.018 INFO [lossy] sleeping 1 second
Sep 22 23:17:26.024 DEBG Read      :1307 deps:[JobId(1306)] res:true
Sep 22 23:17:26.048 WARN returning error on flush!
Sep 22 23:17:26.048 DEBG Flush     :1306 extent_limit None deps:[JobId(1305)] res:false f:131 g:1
Sep 22 23:17:26.048 INFO [lossy] skipping 1307
Sep 22 23:17:26.048 DEBG Flush     :1306 extent_limit None deps:[JobId(1305)] res:true f:131 g:1
Sep 22 23:17:26.054 DEBG Read      :1307 deps:[JobId(1306)] res:true
Sep 22 23:17:26.454 ERRO [2] job id 1306 saw error GenericError("test error")
Sep 22 23:17:26.454 DEBG [rc] retire 1306 clears [JobId(1305), JobId(1306)], : downstairs
Sep 22 23:17:26.501 DEBG IO Flush 1308 has deps [JobId(1307)]
Sep 22 23:17:26.829 DEBG [0] Read AckReady 1307, : downstairs
Sep 22 23:17:26.830 DEBG up_ds_listen was notified
Sep 22 23:17:26.830 DEBG up_ds_listen process 1307
Sep 22 23:17:26.831 DEBG [A] ack job 1307:308, : downstairs
Sep 22 23:17:26.883 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:26.884 DEBG Flush     :1308 extent_limit None deps:[JobId(1307)] res:true f:132 g:1
Sep 22 23:17:26.885 DEBG IO Read  1309 has deps [JobId(1308)]
Sep 22 23:17:26.903 DEBG Read      :1309 deps:[JobId(1308)] res:true
Sep 22 23:17:26.926 INFO [lossy] skipping 1308
Sep 22 23:17:26.926 INFO [lossy] skipping 1309
Sep 22 23:17:26.926 DEBG Flush     :1308 extent_limit None deps:[JobId(1307)] res:true f:132 g:1
Sep 22 23:17:26.932 DEBG Read      :1309 deps:[JobId(1308)] res:true
Sep 22 23:17:27.333 DEBG up_ds_listen was notified
Sep 22 23:17:27.333 DEBG up_ds_listen process 1308
Sep 22 23:17:27.333 DEBG [A] ack job 1308:309, : downstairs
Sep 22 23:17:27.333 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:27.333 DEBG IO Flush 1310 has deps [JobId(1309), JobId(1308)]
Sep 22 23:17:27.339 DEBG Read      :1307 deps:[JobId(1306)] res:true
Sep 22 23:17:27.362 DEBG Flush     :1310 extent_limit None deps:[JobId(1309), JobId(1308)] res:true f:133 g:1
Sep 22 23:17:27.738 DEBG [0] Read AckReady 1309, : downstairs
Sep 22 23:17:27.739 DEBG up_ds_listen was notified
Sep 22 23:17:27.739 DEBG up_ds_listen process 1309
Sep 22 23:17:27.739 DEBG [A] ack job 1309:310, : downstairs
Sep 22 23:17:27.791 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:27.793 INFO [lossy] sleeping 1 second
Sep 22 23:17:27.793 DEBG IO Read  1311 has deps [JobId(1310)]
Sep 22 23:17:27.807 INFO [lossy] skipping 1308
Sep 22 23:17:27.807 WARN returning error on flush!
Sep 22 23:17:27.807 DEBG Flush     :1308 extent_limit None deps:[JobId(1307)] res:false f:132 g:1
Sep 22 23:17:27.807 INFO [lossy] skipping 1308
Sep 22 23:17:27.807 DEBG Flush     :1308 extent_limit None deps:[JobId(1307)] res:true f:132 g:1
Sep 22 23:17:27.807 INFO [lossy] skipping 1309
Sep 22 23:17:27.807 INFO [lossy] skipping 1309
Sep 22 23:17:27.807 INFO [lossy] skipping 1309
Sep 22 23:17:27.814 DEBG Read      :1309 deps:[JobId(1308)] res:true
Sep 22 23:17:27.836 INFO [lossy] sleeping 1 second
Sep 22 23:17:27.931 DEBG IO Flush 1312 has deps [JobId(1311), JobId(1310)]
Sep 22 23:17:28.594 ERRO [1] job id 1308 saw error GenericError("test error")
Sep 22 23:17:28.594 DEBG [rc] retire 1308 clears [JobId(1307), JobId(1308)], : downstairs
Sep 22 23:17:28.597 INFO [lossy] skipping 1310
Sep 22 23:17:28.597 INFO [lossy] skipping 1312
Sep 22 23:17:28.597 DEBG Flush     :1310 extent_limit None deps:[JobId(1309), JobId(1308)] res:true f:133 g:1
Sep 22 23:17:28.597 INFO [lossy] skipping 1312
Sep 22 23:17:28.597 INFO [lossy] sleeping 1 second
Sep 22 23:17:28.975 DEBG up_ds_listen was notified
Sep 22 23:17:28.975 DEBG up_ds_listen process 1310
Sep 22 23:17:28.975 DEBG [A] ack job 1310:311, : downstairs
Sep 22 23:17:28.975 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:28.981 DEBG Read      :1311 deps:[JobId(1310)] res:true
Sep 22 23:17:29.002 DEBG Flush     :1310 extent_limit None deps:[JobId(1309), JobId(1308)] res:true f:133 g:1
Sep 22 23:17:29.008 DEBG Read      :1311 deps:[JobId(1310)] res:true
Sep 22 23:17:29.030 DEBG [rc] retire 1310 clears [JobId(1309), JobId(1310)], : downstairs
Sep 22 23:17:29.032 WARN returning error on flush!
Sep 22 23:17:29.032 DEBG Flush     :1312 extent_limit None deps:[JobId(1311), JobId(1310)] res:false f:134 g:1
Sep 22 23:17:29.032 INFO [lossy] skipping 1312
Sep 22 23:17:29.032 INFO [lossy] skipping 1312
Sep 22 23:17:29.032 WARN returning error on flush!
Sep 22 23:17:29.032 DEBG Flush     :1312 extent_limit None deps:[JobId(1311), JobId(1310)] res:false f:134 g:1
Sep 22 23:17:29.032 DEBG Flush     :1312 extent_limit None deps:[JobId(1311), JobId(1310)] res:true f:134 g:1
Sep 22 23:17:29.032 INFO [lossy] sleeping 1 second
Sep 22 23:17:29.033 DEBG Flush     :1312 extent_limit None deps:[JobId(1311), JobId(1310)] res:true f:134 g:1
Sep 22 23:17:29.456 DEBG [0] Read AckReady 1311, : downstairs
Sep 22 23:17:29.785 DEBG [2] Read already AckReady 1311, : downstairs
Sep 22 23:17:29.786 ERRO [2] job id 1312 saw error GenericError("test error")
Sep 22 23:17:29.786 ERRO [2] job id 1312 saw error GenericError("test error")
Sep 22 23:17:29.786 DEBG up_ds_listen was notified
Sep 22 23:17:29.786 DEBG up_ds_listen process 1311
Sep 22 23:17:29.786 DEBG [A] ack job 1311:312, : downstairs
Sep 22 23:17:29.839 DEBG up_ds_listen process 1312
Sep 22 23:17:29.839 DEBG [A] ack job 1312:313, : downstairs
Sep 22 23:17:29.839 DEBG up_ds_listen checked 2 jobs, back to waiting
Sep 22 23:17:29.839 DEBG up_ds_listen was notified
Sep 22 23:17:29.839 DEBG up_ds_listen checked 0 jobs, back to waiting
Sep 22 23:17:29.846 DEBG Read      :1311 deps:[JobId(1310)] res:true
Sep 22 23:17:29.868 DEBG IO Read  1313 has deps [JobId(1312)]
Sep 22 23:17:29.880 WARN returning error on read!
Sep 22 23:17:29.880 DEBG Read      :1313 deps:[JobId(1312)] res:false
Sep 22 23:17:29.880 WARN returning error on read!
Sep 22 23:17:29.880 DEBG Read      :1313 deps:[JobId(1312)] res:false
Sep 22 23:17:29.886 DEBG Read      :1313 deps:[JobId(1312)] res:true
Sep 22 23:17:29.907 ERRO [0] job id 1313 saw error GenericError("test error")
Sep 22 23:17:29.908 ERRO [0] job id 1313 saw error GenericError("test error")
Sep 22 23:17:29.909 DEBG Flush     :1312 extent_limit None deps:[JobId(1311), JobId(1310)] res:true f:134 g:1
Sep 22 23:17:29.915 DEBG Read      :1313 deps:[JobId(1312)] res:true
Sep 22 23:17:30.315 DEBG [rc] retire 1312 clears [JobId(1311), JobId(1312)], : downstairs
Sep 22 23:17:30.317 WARN returning error on read!
Sep 22 23:17:30.317 DEBG Read      :1313 deps:[JobId(1312)] res:false
Sep 22 23:17:30.317 INFO [lossy] skipping 1313
Sep 22 23:17:30.323 DEBG Read      :1313 deps:[JobId(1312)] res:true
Sep 22 23:17:30.391 DEBG IO Flush 1314 has deps [JobId(1313)]
Sep 22 23:17:30.720 DEBG [0] Read AckReady 1313, : downstairs
Sep 22 23:17:30.721 ERRO [2] job id 1313 saw error GenericError("test error")
Sep 22 23:17:30.721 DEBG up_ds_listen was notified
Sep 22 23:17:30.721 DEBG up_ds_listen process 1313
Sep 22 23:17:30.721 DEBG [A] ack job 1313:314, : downstairs
Sep 22 23:17:30.774 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:30.775 INFO [lossy] sleeping 1 second
Sep 22 23:17:30.776 DEBG IO Read  1315 has deps [JobId(1314)]
Sep 22 23:17:30.790 INFO [lossy] sleeping 1 second
Sep 22 23:17:31.170 DEBG Flush     :1314 extent_limit None deps:[JobId(1313)] res:true f:135 g:1
Sep 22 23:17:31.176 DEBG Read      :1315 deps:[JobId(1314)] res:true
Sep 22 23:17:31.198 DEBG IO Flush 1316 has deps [JobId(1315), JobId(1314)]
Sep 22 23:17:31.577 INFO [lossy] sleeping 1 second
Sep 22 23:17:31.952 DEBG [2] Read AckReady 1315, : downstairs
Sep 22 23:17:31.953 DEBG up_ds_listen was notified
Sep 22 23:17:31.953 DEBG up_ds_listen process 1315
Sep 22 23:17:31.953 DEBG [A] ack job 1315:316, : downstairs
Sep 22 23:17:32.005 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:32.007 DEBG Flush     :1314 extent_limit None deps:[JobId(1313)] res:true f:135 g:1
Sep 22 23:17:32.013 DEBG Read      :1315 deps:[JobId(1314)] res:true
Sep 22 23:17:32.034 WARN returning error on flush!
Sep 22 23:17:32.034 DEBG Flush     :1314 extent_limit None deps:[JobId(1313)] res:false f:135 g:1
Sep 22 23:17:32.034 INFO [lossy] skipping 1316
Sep 22 23:17:32.034 DEBG Flush     :1314 extent_limit None deps:[JobId(1313)] res:true f:135 g:1
Sep 22 23:17:32.034 INFO [lossy] skipping 1316
Sep 22 23:17:32.040 DEBG Read      :1315 deps:[JobId(1314)] res:true
Sep 22 23:17:32.062 DEBG IO Read  1317 has deps [JobId(1316)]
Sep 22 23:17:32.062 ERRO [1] job id 1314 saw error GenericError("test error")
Sep 22 23:17:32.067 DEBG up_ds_listen was notified
Sep 22 23:17:32.067 DEBG up_ds_listen process 1314
Sep 22 23:17:32.067 DEBG [A] ack job 1314:315, : downstairs
Sep 22 23:17:32.068 DEBG [rc] retire 1314 clears [JobId(1313), JobId(1314)], : downstairs
Sep 22 23:17:32.068 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:32.076 DEBG Flush     :1316 extent_limit None deps:[JobId(1315), JobId(1314)] res:true f:136 g:1
Sep 22 23:17:32.082 DEBG Read      :1317 deps:[JobId(1316)] res:true
Sep 22 23:17:32.103 DEBG Flush     :1316 extent_limit None deps:[JobId(1315), JobId(1314)] res:true f:136 g:1
Sep 22 23:17:32.104 WARN returning error on read!
Sep 22 23:17:32.104 DEBG Read      :1317 deps:[JobId(1316)] res:false
Sep 22 23:17:32.109 DEBG Read      :1317 deps:[JobId(1316)] res:true
Sep 22 23:17:32.557 ERRO [0] job id 1317 saw error GenericError("test error")
Sep 22 23:17:32.886 DEBG up_ds_listen was notified
Sep 22 23:17:32.886 DEBG up_ds_listen process 1316
Sep 22 23:17:32.886 DEBG [A] ack job 1316:317, : downstairs
Sep 22 23:17:32.886 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:32.886 DEBG IO Flush 1318 has deps [JobId(1317), JobId(1316)]
Sep 22 23:17:32.887 DEBG Flush     :1316 extent_limit None deps:[JobId(1315), JobId(1314)] res:true f:136 g:1
Sep 22 23:17:32.892 DEBG Read      :1317 deps:[JobId(1316)] res:true
Sep 22 23:17:32.914 DEBG [rc] retire 1316 clears [JobId(1315), JobId(1316)], : downstairs
Sep 22 23:17:32.917 INFO [lossy] skipping 1318
Sep 22 23:17:32.917 DEBG Flush     :1318 extent_limit None deps:[JobId(1317), JobId(1316)] res:true f:137 g:1
Sep 22 23:17:32.917 INFO [lossy] sleeping 1 second
Sep 22 23:17:32.918 WARN returning error on flush!
Sep 22 23:17:32.918 DEBG Flush     :1318 extent_limit None deps:[JobId(1317), JobId(1316)] res:false f:137 g:1
Sep 22 23:17:32.918 INFO [lossy] skipping 1318
Sep 22 23:17:32.918 DEBG Flush     :1318 extent_limit None deps:[JobId(1317), JobId(1316)] res:true f:137 g:1
Sep 22 23:17:32.918 INFO [lossy] sleeping 1 second
Sep 22 23:17:33.342 DEBG [0] Read AckReady 1317, : downstairs
Sep 22 23:17:33.342 ERRO [0] job id 1318 saw error GenericError("test error")
Sep 22 23:17:33.670 DEBG [1] Read already AckReady 1317, : downstairs
Sep 22 23:17:33.672 DEBG up_ds_listen was notified
Sep 22 23:17:33.672 DEBG up_ds_listen process 1317
Sep 22 23:17:33.672 DEBG [A] ack job 1317:318, : downstairs
Sep 22 23:17:33.725 DEBG up_ds_listen process 1318
Sep 22 23:17:33.725 DEBG [A] ack job 1318:319, : downstairs
Sep 22 23:17:33.725 DEBG up_ds_listen checked 2 jobs, back to waiting
Sep 22 23:17:33.725 DEBG up_ds_listen was notified
Sep 22 23:17:33.725 DEBG up_ds_listen checked 0 jobs, back to waiting
Sep 22 23:17:33.727 INFO [lossy] sleeping 1 second
Sep 22 23:17:33.775 DEBG IO Read  1319 has deps [JobId(1318)]
Sep 22 23:17:34.123 DEBG Read      :1319 deps:[JobId(1318)] res:true
Sep 22 23:17:34.144 INFO [lossy] sleeping 1 second
Sep 22 23:17:34.521 DEBG [1] Read AckReady 1319, : downstairs
Sep 22 23:17:34.521 DEBG up_ds_listen was notified
Sep 22 23:17:34.521 DEBG up_ds_listen process 1319
Sep 22 23:17:34.521 DEBG [A] ack job 1319:320, : downstairs
Sep 22 23:17:34.574 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:34.575 DEBG IO Flush 1320 has deps [JobId(1319), JobId(1318)]
Sep 22 23:17:34.575 DEBG IO Read  1321 has deps [JobId(1320)]
Sep 22 23:17:34.581 INFO [lossy] sleeping 1 second
Sep 22 23:17:34.728 DEBG Flush     :1318 extent_limit None deps:[JobId(1317), JobId(1316)] res:true f:137 g:1
Sep 22 23:17:34.729 WARN returning error on read!
Sep 22 23:17:34.729 DEBG Read      :1319 deps:[JobId(1318)] res:false
Sep 22 23:17:34.729 INFO [lossy] skipping 1319
Sep 22 23:17:34.734 DEBG Read      :1319 deps:[JobId(1318)] res:true
Sep 22 23:17:34.756 DEBG [rc] retire 1318 clears [JobId(1317), JobId(1318)], : downstairs
Sep 22 23:17:34.756 ERRO [2] job id 1319 saw error GenericError("test error")
Sep 22 23:17:34.757 DEBG Flush     :1320 extent_limit None deps:[JobId(1319), JobId(1318)] res:true f:138 g:1
Sep 22 23:17:34.763 DEBG Read      :1321 deps:[JobId(1320)] res:true
Sep 22 23:17:35.162 DEBG IO Flush 1322 has deps [JobId(1321), JobId(1320)]
Sep 22 23:17:35.168 DEBG Read      :1319 deps:[JobId(1318)] res:true
Sep 22 23:17:35.192 INFO [lossy] sleeping 1 second
Sep 22 23:17:35.568 DEBG [2] Read AckReady 1321, : downstairs
Sep 22 23:17:35.568 DEBG up_ds_listen was notified
Sep 22 23:17:35.568 DEBG up_ds_listen process 1321
Sep 22 23:17:35.568 DEBG [A] ack job 1321:322, : downstairs
Sep 22 23:17:35.621 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:35.623 DEBG Flush     :1320 extent_limit None deps:[JobId(1319), JobId(1318)] res:true f:138 g:1
Sep 22 23:17:35.623 INFO [lossy] skipping 1321
Sep 22 23:17:35.629 DEBG Read      :1321 deps:[JobId(1320)] res:true
Sep 22 23:17:35.650 DEBG Flush     :1320 extent_limit None deps:[JobId(1319), JobId(1318)] res:true f:138 g:1
Sep 22 23:17:35.650 INFO [lossy] skipping 1321
Sep 22 23:17:35.650 INFO [lossy] skipping 1322
Sep 22 23:17:35.650 INFO [lossy] skipping 1321
Sep 22 23:17:35.656 DEBG Read      :1321 deps:[JobId(1320)] res:true
Sep 22 23:17:35.725 DEBG IO Read  1323 has deps [JobId(1322)]
Sep 22 23:17:35.725 DEBG IO Flush 1324 has deps [JobId(1323), JobId(1322)]
Sep 22 23:17:36.061 DEBG up_ds_listen was notified
Sep 22 23:17:36.061 DEBG up_ds_listen process 1320
Sep 22 23:17:36.061 DEBG [A] ack job 1320:321, : downstairs
Sep 22 23:17:36.061 DEBG [rc] retire 1320 clears [JobId(1319), JobId(1320)], : downstairs
Sep 22 23:17:36.061 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:36.070 DEBG Flush     :1322 extent_limit None deps:[JobId(1321), JobId(1320)] res:true f:139 g:1
Sep 22 23:17:36.070 INFO [lossy] skipping 1323
Sep 22 23:17:36.071 WARN returning error on read!
Sep 22 23:17:36.071 DEBG Read      :1323 deps:[JobId(1322)] res:false
Sep 22 23:17:36.076 DEBG Read      :1323 deps:[JobId(1322)] res:true
Sep 22 23:17:36.099 DEBG Flush     :1322 extent_limit None deps:[JobId(1321), JobId(1320)] res:true f:139 g:1
Sep 22 23:17:36.099 WARN returning error on read!
Sep 22 23:17:36.099 DEBG Read      :1323 deps:[JobId(1322)] res:false
Sep 22 23:17:36.105 DEBG Read      :1323 deps:[JobId(1322)] res:true
Sep 22 23:17:36.552 ERRO [0] job id 1323 saw error GenericError("test error")
Sep 22 23:17:36.882 ERRO [1] job id 1323 saw error GenericError("test error")
Sep 22 23:17:36.882 DEBG up_ds_listen was notified
Sep 22 23:17:36.882 DEBG up_ds_listen process 1322
Sep 22 23:17:36.882 DEBG [A] ack job 1322:323, : downstairs
Sep 22 23:17:36.882 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:36.882 DEBG Flush     :1322 extent_limit None deps:[JobId(1321), JobId(1320)] res:true f:139 g:1
Sep 22 23:17:36.882 WARN returning error on read!
Sep 22 23:17:36.882 DEBG Read      :1323 deps:[JobId(1322)] res:false
Sep 22 23:17:36.882 INFO [lossy] skipping 1323
Sep 22 23:17:36.888 DEBG Read      :1323 deps:[JobId(1322)] res:true
Sep 22 23:17:36.910 DEBG [rc] retire 1322 clears [JobId(1321), JobId(1322)], : downstairs
Sep 22 23:17:36.910 ERRO [2] job id 1323 saw error GenericError("test error")
Sep 22 23:17:36.912 INFO [lossy] skipping 1324
Sep 22 23:17:36.912 WARN returning error on flush!
Sep 22 23:17:36.912 DEBG Flush     :1324 extent_limit None deps:[JobId(1323), JobId(1322)] res:false f:140 g:1
Sep 22 23:17:36.912 INFO [lossy] skipping 1324
Sep 22 23:17:36.912 WARN returning error on flush!
Sep 22 23:17:36.912 DEBG Flush     :1324 extent_limit None deps:[JobId(1323), JobId(1322)] res:false f:140 g:1
Sep 22 23:17:36.912 DEBG Flush     :1324 extent_limit None deps:[JobId(1323), JobId(1322)] res:true f:140 g:1
Sep 22 23:17:36.912 INFO [lossy] sleeping 1 second
Sep 22 23:17:37.288 DEBG [1] Read AckReady 1323, : downstairs
Sep 22 23:17:37.289 ERRO [1] job id 1324 saw error GenericError("test error")
Sep 22 23:17:37.289 ERRO [1] job id 1324 saw error GenericError("test error")
Sep 22 23:17:37.289 DEBG up_ds_listen was notified
Sep 22 23:17:37.289 DEBG up_ds_listen process 1323
Sep 22 23:17:37.289 DEBG [A] ack job 1323:324, : downstairs
Sep 22 23:17:37.342 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:37.344 INFO [lossy] sleeping 1 second
Sep 22 23:17:37.346 INFO [lossy] sleeping 1 second
Sep 22 23:17:37.441 DEBG IO Read  1325 has deps [JobId(1324)]
Sep 22 23:17:38.113 DEBG IO Flush 1326 has deps [JobId(1325), JobId(1324)]
Sep 22 23:17:38.113 INFO [lossy] sleeping 1 second
Sep 22 23:17:38.346 WARN returning error on flush!
Sep 22 23:17:38.346 DEBG Flush     :1324 extent_limit None deps:[JobId(1323), JobId(1322)] res:false f:140 g:1
Sep 22 23:17:38.346 DEBG Flush     :1324 extent_limit None deps:[JobId(1323), JobId(1322)] res:true f:140 g:1
Sep 22 23:17:38.352 DEBG Read      :1325 deps:[JobId(1324)] res:true
Sep 22 23:17:38.373 WARN returning error on flush!
Sep 22 23:17:38.373 DEBG Flush     :1324 extent_limit None deps:[JobId(1323), JobId(1322)] res:false f:140 g:1
Sep 22 23:17:38.373 INFO [lossy] skipping 1325
Sep 22 23:17:38.373 DEBG Flush     :1324 extent_limit None deps:[JobId(1323), JobId(1322)] res:true f:140 g:1
Sep 22 23:17:38.379 DEBG Read      :1325 deps:[JobId(1324)] res:true
Sep 22 23:17:38.401 ERRO [2] job id 1324 saw error GenericError("test error")
Sep 22 23:17:38.401 DEBG up_ds_listen was notified
Sep 22 23:17:38.401 DEBG up_ds_listen process 1324
Sep 22 23:17:38.401 DEBG [A] ack job 1324:325, : downstairs
Sep 22 23:17:38.401 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:38.401 ERRO [0] job id 1324 saw error GenericError("test error")
Sep 22 23:17:38.401 DEBG [rc] retire 1324 clears [JobId(1323), JobId(1324)], : downstairs
Sep 22 23:17:38.403 DEBG Flush     :1326 extent_limit None deps:[JobId(1325), JobId(1324)] res:true f:141 g:1
Sep 22 23:17:38.403 INFO [lossy] sleeping 1 second
Sep 22 23:17:38.451 DEBG Flush     :1326 extent_limit None deps:[JobId(1325), JobId(1324)] res:true f:141 g:1
Sep 22 23:17:38.779 DEBG [2] Read AckReady 1325, : downstairs
Sep 22 23:17:38.780 DEBG up_ds_listen was notified
Sep 22 23:17:38.780 DEBG up_ds_listen process 1325
Sep 22 23:17:38.780 DEBG [A] ack job 1325:326, : downstairs
Sep 22 23:17:38.832 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:39.210 DEBG up_ds_listen was notified
Sep 22 23:17:39.210 DEBG up_ds_listen process 1326
Sep 22 23:17:39.210 DEBG [A] ack job 1326:327, : downstairs
Sep 22 23:17:39.210 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:39.210 INFO [lossy] skipping 1325
Sep 22 23:17:39.210 WARN returning error on read!
Sep 22 23:17:39.211 DEBG Read      :1325 deps:[JobId(1324)] res:false
Sep 22 23:17:39.211 INFO [lossy] skipping 1325
Sep 22 23:17:39.211 WARN returning error on read!
Sep 22 23:17:39.211 DEBG Read      :1325 deps:[JobId(1324)] res:false
Sep 22 23:17:39.211 WARN returning error on read!
Sep 22 23:17:39.211 DEBG Read      :1325 deps:[JobId(1324)] res:false
Sep 22 23:17:39.217 DEBG Read      :1325 deps:[JobId(1324)] res:true
Sep 22 23:17:39.238 DEBG IO Read  1327 has deps [JobId(1326)]
Sep 22 23:17:39.250 ERRO [1] job id 1325 saw error GenericError("test error")
Sep 22 23:17:39.250 ERRO [1] job id 1325 saw error GenericError("test error")
Sep 22 23:17:39.250 ERRO [1] job id 1325 saw error GenericError("test error")
Sep 22 23:17:39.251 INFO [lossy] skipping 1327
Sep 22 23:17:39.251 INFO [lossy] skipping 1327
Sep 22 23:17:39.256 DEBG Read      :1327 deps:[JobId(1326)] res:true
Sep 22 23:17:39.280 INFO [lossy] skipping 1326
Sep 22 23:17:39.280 INFO [lossy] skipping 1326
Sep 22 23:17:39.280 DEBG Flush     :1326 extent_limit None deps:[JobId(1325), JobId(1324)] res:true f:141 g:1
Sep 22 23:17:39.280 INFO [lossy] sleeping 1 second
Sep 22 23:17:39.657 DEBG [rc] retire 1326 clears [JobId(1325), JobId(1326)], : downstairs
Sep 22 23:17:39.704 DEBG IO Flush 1328 has deps [JobId(1327)]
Sep 22 23:17:39.704 WARN returning error on read!
Sep 22 23:17:39.704 DEBG Read      :1327 deps:[JobId(1326)] res:false
Sep 22 23:17:39.704 WARN returning error on read!
Sep 22 23:17:39.704 DEBG Read      :1327 deps:[JobId(1326)] res:false
Sep 22 23:17:39.710 DEBG Read      :1327 deps:[JobId(1326)] res:true
Sep 22 23:17:40.061 DEBG [0] Read AckReady 1327, : downstairs
Sep 22 23:17:40.062 DEBG up_ds_listen was notified
Sep 22 23:17:40.062 DEBG up_ds_listen process 1327
Sep 22 23:17:40.062 DEBG [A] ack job 1327:328, : downstairs
Sep 22 23:17:40.115 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:40.116 ERRO [2] job id 1327 saw error GenericError("test error")
Sep 22 23:17:40.116 ERRO [2] job id 1327 saw error GenericError("test error")
Sep 22 23:17:40.116 WARN returning error on flush!
Sep 22 23:17:40.116 DEBG Flush     :1328 extent_limit None deps:[JobId(1327)] res:false f:142 g:1
Sep 22 23:17:40.116 INFO [lossy] skipping 1328
Sep 22 23:17:40.116 DEBG Flush     :1328 extent_limit None deps:[JobId(1327)] res:true f:142 g:1
Sep 22 23:17:40.117 DEBG IO Read  1329 has deps [JobId(1328)]
Sep 22 23:17:40.117 ERRO [0] job id 1328 saw error GenericError("test error")
Sep 22 23:17:40.134 DEBG Read      :1329 deps:[JobId(1328)] res:true
Sep 22 23:17:40.158 DEBG Flush     :1328 extent_limit None deps:[JobId(1327)] res:true f:142 g:1
Sep 22 23:17:40.163 DEBG Read      :1329 deps:[JobId(1328)] res:true
Sep 22 23:17:40.563 DEBG up_ds_listen was notified
Sep 22 23:17:40.563 DEBG up_ds_listen process 1328
Sep 22 23:17:40.563 DEBG [A] ack job 1328:329, : downstairs
Sep 22 23:17:40.563 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:40.563 DEBG IO Flush 1330 has deps [JobId(1329), JobId(1328)]
Sep 22 23:17:40.569 DEBG Read      :1327 deps:[JobId(1326)] res:true
Sep 22 23:17:40.592 INFO [lossy] sleeping 1 second
Sep 22 23:17:40.971 DEBG [0] Read AckReady 1329, : downstairs
Sep 22 23:17:40.971 DEBG up_ds_listen was notified
Sep 22 23:17:40.971 DEBG up_ds_listen process 1329
Sep 22 23:17:40.971 DEBG [A] ack job 1329:330, : downstairs
Sep 22 23:17:41.025 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:43.145 DEBG IO Write 1331 has deps [JobId(1330), JobId(1328)]
Sep 22 23:17:43.145 DEBG IO Flush 1332 has deps [JobId(1331), JobId(1330)]
Sep 22 23:17:43.145 DEBG Flush     :1330 extent_limit None deps:[JobId(1329), JobId(1328)] res:true f:143 g:1
Sep 22 23:17:43.146 DEBG up_ds_listen was notified
Sep 22 23:17:43.146 DEBG up_ds_listen process 1331
Sep 22 23:17:43.146 DEBG [A] ack job 1331:332, : downstairs
Sep 22 23:17:43.146 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:43.538 DEBG IO Write 1333 has deps [JobId(1332), JobId(1330), JobId(1328)]
Sep 22 23:17:43.538 DEBG up_ds_listen was notified
Sep 22 23:17:43.538 DEBG up_ds_listen process 1333
Sep 22 23:17:43.538 DEBG [A] ack job 1333:334, : downstairs
Sep 22 23:17:43.538 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:43.868 DEBG IO Write 1334 has deps [JobId(1332), JobId(1330), JobId(1328)]
Sep 22 23:17:43.868 DEBG up_ds_listen was notified
Sep 22 23:17:43.868 DEBG up_ds_listen process 1334
Sep 22 23:17:43.868 DEBG [A] ack job 1334:335, : downstairs
Sep 22 23:17:43.868 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:43.868 DEBG IO Flush 1335 has deps [JobId(1334), JobId(1333), JobId(1332)]
Sep 22 23:17:44.198 DEBG IO Write 1336 has deps [JobId(1335), JobId(1332), JobId(1330), JobId(1328)]
Sep 22 23:17:44.198 DEBG up_ds_listen was notified
Sep 22 23:17:44.198 DEBG up_ds_listen process 1336
Sep 22 23:17:44.198 DEBG [A] ack job 1336:337, : downstairs
Sep 22 23:17:44.198 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:44.528 DEBG IO Write 1337 has deps [JobId(1335), JobId(1332), JobId(1330), JobId(1328)]
Sep 22 23:17:44.529 DEBG up_ds_listen was notified
Sep 22 23:17:44.529 DEBG up_ds_listen process 1337
Sep 22 23:17:44.529 DEBG [A] ack job 1337:338, : downstairs
Sep 22 23:17:44.529 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:44.529 DEBG IO Flush 1338 has deps [JobId(1337), JobId(1336), JobId(1335)]
Sep 22 23:17:44.858 DEBG IO Write 1339 has deps [JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)]
Sep 22 23:17:44.859 DEBG up_ds_listen was notified
Sep 22 23:17:44.859 DEBG up_ds_listen process 1339
Sep 22 23:17:44.859 DEBG [A] ack job 1339:340, : downstairs
Sep 22 23:17:44.859 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:45.189 DEBG IO Write 1340 has deps [JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)]
Sep 22 23:17:45.189 DEBG up_ds_listen was notified
Sep 22 23:17:45.189 DEBG up_ds_listen process 1340
Sep 22 23:17:45.189 DEBG [A] ack job 1340:341, : downstairs
Sep 22 23:17:45.189 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:45.190 DEBG IO Flush 1341 has deps [JobId(1340), JobId(1339), JobId(1338)]
Sep 22 23:17:45.519 DEBG IO Write 1342 has deps [JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)]
Sep 22 23:17:45.520 DEBG up_ds_listen was notified
Sep 22 23:17:45.520 DEBG up_ds_listen process 1342
Sep 22 23:17:45.520 DEBG [A] ack job 1342:343, : downstairs
Sep 22 23:17:45.520 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:45.850 DEBG IO Write 1343 has deps [JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)]
Sep 22 23:17:45.850 DEBG up_ds_listen was notified
Sep 22 23:17:45.850 DEBG up_ds_listen process 1343
Sep 22 23:17:45.850 DEBG [A] ack job 1343:344, : downstairs
Sep 22 23:17:45.850 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:45.850 DEBG IO Flush 1344 has deps [JobId(1343), JobId(1342), JobId(1341)]
Sep 22 23:17:46.244 DEBG IO Write 1345 has deps [JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)]
Sep 22 23:17:46.244 DEBG up_ds_listen was notified
Sep 22 23:17:46.244 DEBG up_ds_listen process 1345
Sep 22 23:17:46.244 DEBG [A] ack job 1345:346, : downstairs
Sep 22 23:17:46.244 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:46.385 INFO [lossy] sleeping 1 second
Sep 22 23:17:46.715 DEBG IO Write 1346 has deps [JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)]
Sep 22 23:17:46.715 DEBG IO Flush 1347 has deps [JobId(1346), JobId(1345), JobId(1344)]
Sep 22 23:17:46.715 DEBG up_ds_listen was notified
Sep 22 23:17:46.715 DEBG up_ds_listen process 1346
Sep 22 23:17:46.715 DEBG [A] ack job 1346:347, : downstairs
Sep 22 23:17:46.715 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:47.045 DEBG IO Write 1348 has deps [JobId(1347), JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)]
Sep 22 23:17:47.045 DEBG up_ds_listen was notified
Sep 22 23:17:47.045 DEBG up_ds_listen process 1348
Sep 22 23:17:47.045 DEBG [A] ack job 1348:349, : downstairs
Sep 22 23:17:47.045 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:47.375 DEBG IO Write 1349 has deps [JobId(1347), JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)]
Sep 22 23:17:47.375 DEBG up_ds_listen was notified
Sep 22 23:17:47.375 DEBG up_ds_listen process 1349
Sep 22 23:17:47.375 DEBG [A] ack job 1349:350, : downstairs
Sep 22 23:17:47.375 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:47.375 DEBG IO Flush 1350 has deps [JobId(1349), JobId(1348), JobId(1347)]
Sep 22 23:17:47.704 DEBG IO Write 1351 has deps [JobId(1350), JobId(1347), JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)]
Sep 22 23:17:47.704 DEBG up_ds_listen was notified
Sep 22 23:17:47.704 DEBG up_ds_listen process 1351
Sep 22 23:17:47.704 DEBG [A] ack job 1351:352, : downstairs
Sep 22 23:17:47.705 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:47.735 DEBG Write     :1331 deps:[JobId(1330), JobId(1328)] res:true
Sep 22 23:17:47.740 DEBG Flush     :1332 extent_limit None deps:[JobId(1331), JobId(1330)] res:true f:144 g:1
Sep 22 23:17:48.069 DEBG IO Write 1352 has deps [JobId(1350), JobId(1347), JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)]
Sep 22 23:17:48.069 DEBG up_ds_listen was notified
Sep 22 23:17:48.069 DEBG up_ds_listen process 1352
Sep 22 23:17:48.069 DEBG [A] ack job 1352:353, : downstairs
Sep 22 23:17:48.069 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:48.070 DEBG IO Flush 1353 has deps [JobId(1352), JobId(1351), JobId(1350)]
Sep 22 23:17:48.398 DEBG IO Write 1354 has deps [JobId(1353), JobId(1350), JobId(1347), JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)]
Sep 22 23:17:48.398 DEBG up_ds_listen was notified
Sep 22 23:17:48.399 DEBG up_ds_listen process 1354
Sep 22 23:17:48.399 DEBG [A] ack job 1354:355, : downstairs
Sep 22 23:17:48.399 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:48.728 DEBG IO Write 1355 has deps [JobId(1353), JobId(1350), JobId(1347), JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)]
Sep 22 23:17:48.728 DEBG up_ds_listen was notified
Sep 22 23:17:48.728 DEBG up_ds_listen process 1355
Sep 22 23:17:48.728 DEBG [A] ack job 1355:356, : downstairs
Sep 22 23:17:48.728 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:48.729 DEBG IO Flush 1356 has deps [JobId(1355), JobId(1354), JobId(1353)]
Sep 22 23:17:49.058 DEBG IO Write 1357 has deps [JobId(1356), JobId(1353), JobId(1350), JobId(1347), JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)]
Sep 22 23:17:49.058 DEBG up_ds_listen was notified
Sep 22 23:17:49.058 DEBG up_ds_listen process 1357
Sep 22 23:17:49.058 DEBG [A] ack job 1357:358, : downstairs
Sep 22 23:17:49.058 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:49.451 DEBG IO Write 1358 has deps [JobId(1356), JobId(1353), JobId(1350), JobId(1347), JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)]
Sep 22 23:17:49.451 DEBG up_ds_listen was notified
Sep 22 23:17:49.451 DEBG up_ds_listen process 1358
Sep 22 23:17:49.451 DEBG [A] ack job 1358:359, : downstairs
Sep 22 23:17:49.451 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:49.594 DEBG IO Flush 1359 has deps [JobId(1358), JobId(1357), JobId(1356)]
Sep 22 23:17:49.625 DEBG Write     :1333 deps:[JobId(1332), JobId(1330), JobId(1328)] res:true
Sep 22 23:17:49.958 DEBG IO Write 1360 has deps [JobId(1359), JobId(1356), JobId(1353), JobId(1350), JobId(1347), JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)]
Sep 22 23:17:49.958 DEBG up_ds_listen was notified
Sep 22 23:17:49.959 DEBG up_ds_listen process 1360
Sep 22 23:17:49.959 DEBG [A] ack job 1360:361, : downstairs
Sep 22 23:17:49.959 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:50.289 DEBG IO Write 1361 has deps [JobId(1359), JobId(1356), JobId(1353), JobId(1350), JobId(1347), JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)]
Sep 22 23:17:50.289 DEBG up_ds_listen was notified
Sep 22 23:17:50.289 DEBG up_ds_listen process 1361
Sep 22 23:17:50.289 DEBG [A] ack job 1361:362, : downstairs
Sep 22 23:17:50.289 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:50.290 DEBG IO Flush 1362 has deps [JobId(1361), JobId(1360), JobId(1359)]
Sep 22 23:17:50.619 DEBG IO Write 1363 has deps [JobId(1362), JobId(1359), JobId(1356), JobId(1353), JobId(1350), JobId(1347), JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)]
Sep 22 23:17:50.619 DEBG up_ds_listen was notified
Sep 22 23:17:50.619 DEBG up_ds_listen process 1363
Sep 22 23:17:50.619 DEBG [A] ack job 1363:364, : downstairs
Sep 22 23:17:50.620 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:50.950 DEBG IO Write 1364 has deps [JobId(1362), JobId(1359), JobId(1356), JobId(1353), JobId(1350), JobId(1347), JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)]
Sep 22 23:17:50.950 DEBG up_ds_listen was notified
Sep 22 23:17:50.950 DEBG up_ds_listen process 1364
Sep 22 23:17:50.950 DEBG [A] ack job 1364:365, : downstairs
Sep 22 23:17:50.950 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:50.951 DEBG IO Flush 1365 has deps [JobId(1364), JobId(1363), JobId(1362)]
Sep 22 23:17:51.281 DEBG IO Write 1366 has deps [JobId(1365), JobId(1362), JobId(1359), JobId(1356), JobId(1353), JobId(1350), JobId(1347), JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)]
Sep 22 23:17:51.281 DEBG up_ds_listen was notified
Sep 22 23:17:51.281 DEBG up_ds_listen process 1366
Sep 22 23:17:51.281 DEBG [A] ack job 1366:367, : downstairs
Sep 22 23:17:51.281 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:51.611 DEBG IO Write 1367 has deps [JobId(1365), JobId(1362), JobId(1359), JobId(1356), JobId(1353), JobId(1350), JobId(1347), JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)]
Sep 22 23:17:51.611 DEBG up_ds_listen was notified
Sep 22 23:17:51.611 DEBG up_ds_listen process 1367
Sep 22 23:17:51.612 DEBG [A] ack job 1367:368, : downstairs
Sep 22 23:17:51.612 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:51.612 DEBG IO Flush 1368 has deps [JobId(1367), JobId(1366), JobId(1365)]
Sep 22 23:17:51.941 DEBG IO Write 1369 has deps [JobId(1368), JobId(1365), JobId(1362), JobId(1359), JobId(1356), JobId(1353), JobId(1350), JobId(1347), JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)]
Sep 22 23:17:51.941 DEBG up_ds_listen was notified
Sep 22 23:17:51.941 DEBG up_ds_listen process 1369
Sep 22 23:17:51.941 DEBG [A] ack job 1369:370, : downstairs
Sep 22 23:17:51.942 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:52.271 DEBG IO Write 1370 has deps [JobId(1368), JobId(1365), JobId(1362), JobId(1359), JobId(1356), JobId(1353), JobId(1350), JobId(1347), JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)]
Sep 22 23:17:52.272 DEBG up_ds_listen was notified
Sep 22 23:17:52.272 DEBG up_ds_listen process 1370
Sep 22 23:17:52.272 DEBG [A] ack job 1370:371, : downstairs
Sep 22 23:17:52.272 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:52.272 DEBG IO Flush 1371 has deps [JobId(1370), JobId(1369), JobId(1368)]
Sep 22 23:17:52.664 DEBG IO Write 1372 has deps [JobId(1371), JobId(1368), JobId(1365), JobId(1362), JobId(1359), JobId(1356), JobId(1353), JobId(1350), JobId(1347), JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)]
Sep 22 23:17:52.664 DEBG up_ds_listen was notified
Sep 22 23:17:52.664 DEBG up_ds_listen process 1372
Sep 22 23:17:52.665 DEBG [A] ack job 1372:373, : downstairs
Sep 22 23:17:52.665 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:52.805 INFO [lossy] sleeping 1 second
Sep 22 23:17:53.135 DEBG IO Write 1373 has deps [JobId(1371), JobId(1368), JobId(1365), JobId(1362), JobId(1359), JobId(1356), JobId(1353), JobId(1350), JobId(1347), JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)]
Sep 22 23:17:53.135 DEBG IO Flush 1374 has deps [JobId(1373), JobId(1372), JobId(1371)]
Sep 22 23:17:53.135 DEBG up_ds_listen was notified
Sep 22 23:17:53.135 DEBG up_ds_listen process 1373
Sep 22 23:17:53.135 DEBG [A] ack job 1373:374, : downstairs
Sep 22 23:17:53.135 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:53.465 DEBG IO Write 1375 has deps [JobId(1374), JobId(1371), JobId(1368), JobId(1365), JobId(1362), JobId(1359), JobId(1356), JobId(1353), JobId(1350), JobId(1347), JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)]
Sep 22 23:17:53.465 DEBG up_ds_listen was notified
Sep 22 23:17:53.465 DEBG up_ds_listen process 1375
Sep 22 23:17:53.465 DEBG [A] ack job 1375:376, : downstairs
Sep 22 23:17:53.465 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:53.795 DEBG IO Write 1376 has deps [JobId(1374), JobId(1371), JobId(1368), JobId(1365), JobId(1362), JobId(1359), JobId(1356), JobId(1353), JobId(1350), JobId(1347), JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)]
Sep 22 23:17:53.796 DEBG up_ds_listen was notified
Sep 22 23:17:53.796 DEBG up_ds_listen process 1376
Sep 22 23:17:53.796 DEBG [A] ack job 1376:377, : downstairs
Sep 22 23:17:53.796 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:53.796 DEBG IO Flush 1377 has deps [JobId(1376), JobId(1375), JobId(1374)]
Sep 22 23:17:54.126 DEBG IO Write 1378 has deps [JobId(1377), JobId(1374), JobId(1371), JobId(1368), JobId(1365), JobId(1362), JobId(1359), JobId(1356), JobId(1353), JobId(1350), JobId(1347), JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)]
Sep 22 23:17:54.126 DEBG up_ds_listen was notified
Sep 22 23:17:54.126 DEBG up_ds_listen process 1378
Sep 22 23:17:54.126 DEBG [A] ack job 1378:379, : downstairs
Sep 22 23:17:54.127 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:54.157 DEBG Write     :1334 deps:[JobId(1332), JobId(1330), JobId(1328)] res:true
Sep 22 23:17:54.166 DEBG Flush     :1335 extent_limit None deps:[JobId(1334), JobId(1333), JobId(1332)] res:true f:145 g:1
Sep 22 23:17:54.166 INFO [lossy] sleeping 1 second
Sep 22 23:17:54.495 DEBG IO Write 1379 has deps [JobId(1377), JobId(1374), JobId(1371), JobId(1368), JobId(1365), JobId(1362), JobId(1359), JobId(1356), JobId(1353), JobId(1350), JobId(1347), JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)]
Sep 22 23:17:54.495 DEBG up_ds_listen was notified
Sep 22 23:17:54.496 DEBG up_ds_listen process 1379
Sep 22 23:17:54.496 DEBG [A] ack job 1379:380, : downstairs
Sep 22 23:17:54.496 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:54.496 DEBG IO Flush 1380 has deps [JobId(1379), JobId(1378), JobId(1377)]
Sep 22 23:17:54.826 DEBG IO Write 1381 has deps [JobId(1380), JobId(1377), JobId(1374), JobId(1371), JobId(1368), JobId(1365), JobId(1362), JobId(1359), JobId(1356), JobId(1353), JobId(1350), JobId(1347), JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)]
Sep 22 23:17:54.826 DEBG up_ds_listen was notified
Sep 22 23:17:54.826 DEBG up_ds_listen process 1381
Sep 22 23:17:54.826 DEBG [A] ack job 1381:382, : downstairs
Sep 22 23:17:54.826 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:55.156 DEBG IO Write 1382 has deps [JobId(1380), JobId(1377), JobId(1374), JobId(1371), JobId(1368), JobId(1365), JobId(1362), JobId(1359), JobId(1356), JobId(1353), JobId(1350), JobId(1347), JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)]
Sep 22 23:17:55.156 DEBG up_ds_listen was notified
Sep 22 23:17:55.156 DEBG up_ds_listen process 1382
Sep 22 23:17:55.156 DEBG [A] ack job 1382:383, : downstairs
Sep 22 23:17:55.156 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:55.157 DEBG IO Flush 1383 has deps [JobId(1382), JobId(1381), JobId(1380)]
Sep 22 23:17:55.486 DEBG IO Write 1384 has deps [JobId(1383), JobId(1380), JobId(1377), JobId(1374), JobId(1371), JobId(1368), JobId(1365), JobId(1362), JobId(1359), JobId(1356), JobId(1353), JobId(1350), JobId(1347), JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)]
Sep 22 23:17:55.487 DEBG up_ds_listen was notified
Sep 22 23:17:55.487 DEBG up_ds_listen process 1384
Sep 22 23:17:55.487 DEBG [A] ack job 1384:385, : downstairs
Sep 22 23:17:55.487 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:55.817 DEBG IO Write 1385 has deps [JobId(1383), JobId(1380), JobId(1377), JobId(1374), JobId(1371), JobId(1368), JobId(1365), JobId(1362), JobId(1359), JobId(1356), JobId(1353), JobId(1350), JobId(1347), JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)]
Sep 22 23:17:55.817 DEBG up_ds_listen was notified
Sep 22 23:17:55.817 DEBG up_ds_listen process 1385
Sep 22 23:17:55.817 DEBG [A] ack job 1385:386, : downstairs
Sep 22 23:17:55.817 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:55.818 DEBG IO Flush 1386 has deps [JobId(1385), JobId(1384), JobId(1383)]
Sep 22 23:17:56.210 DEBG IO Write 1387 has deps [JobId(1386), JobId(1383), JobId(1380), JobId(1377), JobId(1374), JobId(1371), JobId(1368), JobId(1365), JobId(1362), JobId(1359), JobId(1356), JobId(1353), JobId(1350), JobId(1347), JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)]
Sep 22 23:17:56.210 DEBG up_ds_listen was notified
Sep 22 23:17:56.210 DEBG up_ds_listen process 1387
Sep 22 23:17:56.210 DEBG [A] ack job 1387:388, : downstairs
Sep 22 23:17:56.210 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:56.380 DEBG Write     :1336 deps:[JobId(1335), JobId(1332), JobId(1330), JobId(1328)] res:true
Sep 22 23:17:56.381 DEBG IO Flush 1388 has deps [JobId(1387), JobId(1386)]
Sep 22 23:17:56.710 DEBG IO Write 1389 has deps [JobId(1388), JobId(1386), JobId(1383), JobId(1380), JobId(1377), JobId(1374), JobId(1371), JobId(1368), JobId(1365), JobId(1362), JobId(1359), JobId(1356), JobId(1353), JobId(1350), JobId(1347), JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)]
Sep 22 23:17:56.710 DEBG up_ds_listen was notified
Sep 22 23:17:56.710 DEBG up_ds_listen process 1389
Sep 22 23:17:56.710 DEBG [A] ack job 1389:390, : downstairs
Sep 22 23:17:56.710 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:57.039 DEBG IO Write 1390 has deps [JobId(1388), JobId(1386), JobId(1383), JobId(1380), JobId(1377), JobId(1374), JobId(1371), JobId(1368), JobId(1365), JobId(1362), JobId(1359), JobId(1356), JobId(1353), JobId(1350), JobId(1347), JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)]
Sep 22 23:17:57.040 DEBG up_ds_listen was notified
Sep 22 23:17:57.040 DEBG up_ds_listen process 1390
Sep 22 23:17:57.040 DEBG [A] ack job 1390:391, : downstairs
Sep 22 23:17:57.040 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:57.040 DEBG IO Flush 1391 has deps [JobId(1390), JobId(1389), JobId(1388)]
Sep 22 23:17:57.369 DEBG IO Write 1392 has deps [JobId(1391), JobId(1388), JobId(1386), JobId(1383), JobId(1380), JobId(1377), JobId(1374), JobId(1371), JobId(1368), JobId(1365), JobId(1362), JobId(1359), JobId(1356), JobId(1353), JobId(1350), JobId(1347), JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)]
Sep 22 23:17:57.370 DEBG up_ds_listen was notified
Sep 22 23:17:57.370 DEBG up_ds_listen process 1392
Sep 22 23:17:57.370 DEBG [A] ack job 1392:393, : downstairs
Sep 22 23:17:57.370 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:57.699 DEBG IO Write 1393 has deps [JobId(1391), JobId(1388), JobId(1386), JobId(1383), JobId(1380), JobId(1377), JobId(1374), JobId(1371), JobId(1368), JobId(1365), JobId(1362), JobId(1359), JobId(1356), JobId(1353), JobId(1350), JobId(1347), JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)]
Sep 22 23:17:57.699 DEBG up_ds_listen was notified
Sep 22 23:17:57.699 DEBG up_ds_listen process 1393
Sep 22 23:17:57.699 DEBG [A] ack job 1393:394, : downstairs
Sep 22 23:17:57.700 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:57.700 DEBG IO Flush 1394 has deps [JobId(1393), JobId(1392), JobId(1391)]
Sep 22 23:17:58.029 DEBG IO Write 1395 has deps [JobId(1394), JobId(1391), JobId(1388), JobId(1386), JobId(1383), JobId(1380), JobId(1377), JobId(1374), JobId(1371), JobId(1368), JobId(1365), JobId(1362), JobId(1359), JobId(1356), JobId(1353), JobId(1350), JobId(1347), JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)]
Sep 22 23:17:58.030 DEBG up_ds_listen was notified
Sep 22 23:17:58.030 DEBG up_ds_listen process 1395
Sep 22 23:17:58.030 DEBG [A] ack job 1395:396, : downstairs
Sep 22 23:17:58.030 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:58.359 DEBG IO Write 1396 has deps [JobId(1394), JobId(1391), JobId(1388), JobId(1386), JobId(1383), JobId(1380), JobId(1377), JobId(1374), JobId(1371), JobId(1368), JobId(1365), JobId(1362), JobId(1359), JobId(1356), JobId(1353), JobId(1350), JobId(1347), JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)]
Sep 22 23:17:58.359 DEBG up_ds_listen was notified
Sep 22 23:17:58.359 DEBG up_ds_listen process 1396
Sep 22 23:17:58.360 DEBG [A] ack job 1396:397, : downstairs
Sep 22 23:17:58.360 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:58.360 DEBG IO Flush 1397 has deps [JobId(1396), JobId(1395), JobId(1394)]
Sep 22 23:17:58.689 DEBG IO Write 1398 has deps [JobId(1397), JobId(1394), JobId(1391), JobId(1388), JobId(1386), JobId(1383), JobId(1380), JobId(1377), JobId(1374), JobId(1371), JobId(1368), JobId(1365), JobId(1362), JobId(1359), JobId(1356), JobId(1353), JobId(1350), JobId(1347), JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)]
Sep 22 23:17:58.689 DEBG up_ds_listen was notified
Sep 22 23:17:58.689 DEBG up_ds_listen process 1398
Sep 22 23:17:58.690 DEBG [A] ack job 1398:399, : downstairs
Sep 22 23:17:58.690 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:59.020 DEBG IO Write 1399 has deps [JobId(1397), JobId(1394), JobId(1391), JobId(1388), JobId(1386), JobId(1383), JobId(1380), JobId(1377), JobId(1374), JobId(1371), JobId(1368), JobId(1365), JobId(1362), JobId(1359), JobId(1356), JobId(1353), JobId(1350), JobId(1347), JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328), JobId(1327)]
Sep 22 23:17:59.020 DEBG up_ds_listen was notified
Sep 22 23:17:59.020 DEBG up_ds_listen process 1399
Sep 22 23:17:59.020 DEBG [A] ack job 1399:400, : downstairs
Sep 22 23:17:59.020 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:59.021 DEBG IO Flush 1400 has deps [JobId(1399), JobId(1398), JobId(1397)]
Sep 22 23:17:59.412 DEBG IO Write 1401 has deps [JobId(1400), JobId(1397), JobId(1394), JobId(1391), JobId(1388), JobId(1386), JobId(1383), JobId(1380), JobId(1377), JobId(1374), JobId(1371), JobId(1368), JobId(1365), JobId(1362), JobId(1359), JobId(1356), JobId(1353), JobId(1350), JobId(1347), JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1329), JobId(1328)]
Sep 22 23:17:59.413 DEBG up_ds_listen was notified
Sep 22 23:17:59.413 DEBG up_ds_listen process 1401
Sep 22 23:17:59.413 DEBG [A] ack job 1401:402, : downstairs
Sep 22 23:17:59.413 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:17:59.584 DEBG Write     :1337 deps:[JobId(1335), JobId(1332), JobId(1330), JobId(1328)] res:true
Sep 22 23:17:59.585 WARN returning error on flush!
Sep 22 23:17:59.585 DEBG Flush     :1338 extent_limit None deps:[JobId(1337), JobId(1336), JobId(1335)] res:false f:146 g:1
Sep 22 23:17:59.593 DEBG Flush     :1338 extent_limit None deps:[JobId(1337), JobId(1336), JobId(1335)] res:true f:146 g:1
Sep 22 23:17:59.593 DEBG IO Read  1402 has deps [JobId(1400)]
Sep 22 23:17:59.593 DEBG IO Flush 1403 has deps [JobId(1402), JobId(1401), JobId(1400)]
Sep 22 23:17:59.798 INFO [lossy] skipping 1339
Sep 22 23:17:59.798 INFO [lossy] skipping 1339
Sep 22 23:17:59.828 DEBG Write     :1339 deps:[JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)] res:true
Sep 22 23:18:00.033 INFO [lossy] skipping 1340
Sep 22 23:18:00.034 INFO [lossy] skipping 1341
Sep 22 23:18:00.034 INFO [lossy] skipping 1340
Sep 22 23:18:00.034 INFO [lossy] skipping 1341
Sep 22 23:18:00.034 INFO [lossy] skipping 1340
Sep 22 23:18:00.064 DEBG Write     :1340 deps:[JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)] res:true
Sep 22 23:18:00.065 INFO [lossy] sleeping 1 second
Sep 22 23:18:00.335 ERRO [0] job id 1338 saw error GenericError("test error")
Sep 22 23:18:01.091 WARN returning error on flush!
Sep 22 23:18:01.091 DEBG Flush     :1341 extent_limit None deps:[JobId(1340), JobId(1339), JobId(1338)] res:false f:147 g:1
Sep 22 23:18:01.091 INFO [lossy] skipping 1357
Sep 22 23:18:01.100 DEBG Flush     :1341 extent_limit None deps:[JobId(1340), JobId(1339), JobId(1338)] res:true f:147 g:1
Sep 22 23:18:01.100 INFO [lossy] skipping 1357
Sep 22 23:18:01.100 INFO [lossy] skipping 1342
Sep 22 23:18:01.100 INFO [lossy] skipping 1343
Sep 22 23:18:01.100 WARN 1344 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:18:01.100 WARN 1345 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:18:01.100 INFO [lossy] skipping 1346
Sep 22 23:18:01.100 WARN 1348 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:18:01.100 WARN 1349 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:18:01.100 WARN 1351 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:18:01.100 WARN 1352 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:18:01.100 INFO [lossy] skipping 1353
Sep 22 23:18:01.100 WARN 1354 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:18:01.100 WARN 1355 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:18:01.100 INFO [lossy] skipping 1356
Sep 22 23:18:01.100 INFO [lossy] skipping 1342
Sep 22 23:18:01.129 DEBG Write     :1343 deps:[JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)] res:true
Sep 22 23:18:01.130 INFO [lossy] skipping 1346
Sep 22 23:18:01.159 DEBG Write     :1342 deps:[JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)] res:true
Sep 22 23:18:01.160 WARN 1346 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:18:01.168 DEBG Flush     :1344 extent_limit None deps:[JobId(1343), JobId(1342), JobId(1341)] res:true f:148 g:1
Sep 22 23:18:01.197 DEBG Write     :1345 deps:[JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)] res:true
Sep 22 23:18:01.228 DEBG Write     :1346 deps:[JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)] res:true
Sep 22 23:18:01.237 DEBG Flush     :1347 extent_limit None deps:[JobId(1346), JobId(1345), JobId(1344)] res:true f:149 g:1
Sep 22 23:18:01.267 DEBG Write     :1348 deps:[JobId(1347), JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)] res:true
Sep 22 23:18:01.297 DEBG Write     :1349 deps:[JobId(1347), JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)] res:true
Sep 22 23:18:01.307 DEBG Flush     :1350 extent_limit None deps:[JobId(1349), JobId(1348), JobId(1347)] res:true f:150 g:1
Sep 22 23:18:01.307 INFO [lossy] skipping 1351
Sep 22 23:18:01.336 DEBG Write     :1352 deps:[JobId(1350), JobId(1347), JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)] res:true
Sep 22 23:18:01.338 WARN 1353 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:18:01.338 WARN 1354 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:18:01.338 WARN 1355 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:18:01.338 WARN 1357 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:18:01.367 DEBG Write     :1351 deps:[JobId(1350), JobId(1347), JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)] res:true
Sep 22 23:18:01.376 DEBG Flush     :1353 extent_limit None deps:[JobId(1352), JobId(1351), JobId(1350)] res:true f:151 g:1
Sep 22 23:18:01.406 DEBG Write     :1354 deps:[JobId(1353), JobId(1350), JobId(1347), JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)] res:true
Sep 22 23:18:01.437 DEBG Write     :1355 deps:[JobId(1353), JobId(1350), JobId(1347), JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)] res:true
Sep 22 23:18:01.438 INFO [lossy] skipping 1356
Sep 22 23:18:01.438 WARN 1357 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:18:01.446 DEBG Flush     :1356 extent_limit None deps:[JobId(1355), JobId(1354), JobId(1353)] res:true f:152 g:1
Sep 22 23:18:01.476 DEBG Write     :1357 deps:[JobId(1356), JobId(1353), JobId(1350), JobId(1347), JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)] res:true
Sep 22 23:18:01.477 INFO [lossy] sleeping 1 second
Sep 22 23:18:02.511 INFO [lossy] skipping 1358
Sep 22 23:18:02.511 INFO [lossy] skipping 1362
Sep 22 23:18:02.511 INFO [lossy] skipping 1366
Sep 22 23:18:02.511 INFO [lossy] skipping 1373
Sep 22 23:18:02.511 INFO [lossy] skipping 1374
Sep 22 23:18:02.511 INFO [lossy] skipping 1376
Sep 22 23:18:02.511 INFO [lossy] skipping 1380
Sep 22 23:18:02.542 DEBG Write     :1358 deps:[JobId(1356), JobId(1353), JobId(1350), JobId(1347), JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)] res:true
Sep 22 23:18:02.543 INFO [lossy] skipping 1362
Sep 22 23:18:02.543 INFO [lossy] skipping 1366
Sep 22 23:18:02.543 INFO [lossy] skipping 1374
Sep 22 23:18:02.543 INFO [lossy] skipping 1366
Sep 22 23:18:02.543 INFO [lossy] skipping 1359
Sep 22 23:18:02.543 INFO [lossy] skipping 1367
Sep 22 23:18:02.544 INFO [lossy] skipping 1371
Sep 22 23:18:02.544 INFO [lossy] skipping 1372
Sep 22 23:18:02.544 INFO [lossy] skipping 1375
Sep 22 23:18:02.544 INFO [lossy] skipping 1376
Sep 22 23:18:02.544 INFO [lossy] skipping 1377
Sep 22 23:18:02.552 DEBG Flush     :1359 extent_limit None deps:[JobId(1358), JobId(1357), JobId(1356)] res:true f:153 g:1
Sep 22 23:18:02.552 INFO [lossy] skipping 1367
Sep 22 23:18:02.552 INFO [lossy] skipping 1372
Sep 22 23:18:02.552 WARN 1375 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 5 deps, role: work
Sep 22 23:18:02.552 INFO [lossy] skipping 1376
Sep 22 23:18:02.552 INFO [lossy] skipping 1377
Sep 22 23:18:02.552 WARN 1367 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:18:02.552 WARN 1372 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:18:02.552 WARN 1376 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 5 deps, role: work
Sep 22 23:18:02.552 INFO [lossy] skipping 1377
Sep 22 23:18:02.581 DEBG Write     :1360 deps:[JobId(1359), JobId(1356), JobId(1353), JobId(1350), JobId(1347), JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)] res:true
Sep 22 23:18:02.612 DEBG Write     :1361 deps:[JobId(1359), JobId(1356), JobId(1353), JobId(1350), JobId(1347), JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)] res:true
Sep 22 23:18:02.622 DEBG Flush     :1362 extent_limit None deps:[JobId(1361), JobId(1360), JobId(1359)] res:true f:154 g:1
Sep 22 23:18:02.623 INFO [lossy] skipping 1363
Sep 22 23:18:02.624 WARN returning error on write!
Sep 22 23:18:02.624 DEBG Write     :1364 deps:[JobId(1362), JobId(1359), JobId(1356), JobId(1353), JobId(1350), JobId(1347), JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)] res:false
Sep 22 23:18:02.624 WARN 1365 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:18:02.624 WARN 1366 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:18:02.624 WARN 1367 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:18:02.624 INFO [lossy] skipping 1368
Sep 22 23:18:02.624 WARN 1369 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:18:02.624 WARN 1370 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:18:02.625 INFO [lossy] skipping 1371
Sep 22 23:18:02.625 WARN 1372 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:18:02.625 INFO [lossy] skipping 1373
Sep 22 23:18:02.625 WARN 1375 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:18:02.625 WARN 1376 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:18:02.625 INFO [lossy] skipping 1377
Sep 22 23:18:02.625 WARN 1378 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 5 deps, role: work
Sep 22 23:18:02.625 WARN 1379 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 5 deps, role: work
Sep 22 23:18:02.655 DEBG Write     :1363 deps:[JobId(1362), JobId(1359), JobId(1356), JobId(1353), JobId(1350), JobId(1347), JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)] res:true
Sep 22 23:18:02.686 DEBG Write     :1364 deps:[JobId(1362), JobId(1359), JobId(1356), JobId(1353), JobId(1350), JobId(1347), JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)] res:true
Sep 22 23:18:02.688 INFO [lossy] skipping 1371
Sep 22 23:18:02.688 INFO [lossy] skipping 1373
Sep 22 23:18:02.688 INFO [lossy] skipping 1377
Sep 22 23:18:02.688 INFO [lossy] skipping 1373
Sep 22 23:18:02.688 WARN 1373 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:18:02.696 DEBG Flush     :1365 extent_limit None deps:[JobId(1364), JobId(1363), JobId(1362)] res:true f:155 g:1
Sep 22 23:18:02.697 WARN returning error on write!
Sep 22 23:18:02.697 DEBG Write     :1366 deps:[JobId(1365), JobId(1362), JobId(1359), JobId(1356), JobId(1353), JobId(1350), JobId(1347), JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)] res:false
Sep 22 23:18:02.698 INFO [lossy] skipping 1367
Sep 22 23:18:02.698 INFO [lossy] skipping 1368
Sep 22 23:18:02.698 WARN 1369 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:18:02.698 WARN 1370 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:18:02.698 INFO [lossy] skipping 1371
Sep 22 23:18:02.698 WARN 1372 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:18:02.698 INFO [lossy] skipping 1373
Sep 22 23:18:02.698 WARN 1375 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:18:02.698 WARN 1376 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:18:02.698 WARN 1378 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:18:02.698 WARN 1379 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:18:02.727 DEBG Write     :1366 deps:[JobId(1365), JobId(1362), JobId(1359), JobId(1356), JobId(1353), JobId(1350), JobId(1347), JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)] res:true
Sep 22 23:18:02.758 DEBG Write     :1367 deps:[JobId(1365), JobId(1362), JobId(1359), JobId(1356), JobId(1353), JobId(1350), JobId(1347), JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)] res:true
Sep 22 23:18:02.768 DEBG Flush     :1368 extent_limit None deps:[JobId(1367), JobId(1366), JobId(1365)] res:true f:156 g:1
Sep 22 23:18:02.768 WARN 1371 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:18:02.768 WARN 1373 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:18:02.769 WARN returning error on write!
Sep 22 23:18:02.769 DEBG Write     :1369 deps:[JobId(1368), JobId(1365), JobId(1362), JobId(1359), JobId(1356), JobId(1353), JobId(1350), JobId(1347), JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)] res:false
Sep 22 23:18:02.770 INFO [lossy] skipping 1370
Sep 22 23:18:02.770 WARN 1372 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:18:02.770 INFO [lossy] skipping 1375
Sep 22 23:18:02.770 WARN 1376 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:18:02.770 WARN 1378 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:18:02.770 WARN 1379 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:18:02.770 INFO [lossy] skipping 1369
Sep 22 23:18:02.800 DEBG Write     :1370 deps:[JobId(1368), JobId(1365), JobId(1362), JobId(1359), JobId(1356), JobId(1353), JobId(1350), JobId(1347), JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)] res:true
Sep 22 23:18:02.802 WARN 1375 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:18:02.802 INFO [lossy] skipping 1369
Sep 22 23:18:02.802 INFO [lossy] skipping 1369
Sep 22 23:18:02.802 INFO [lossy] skipping 1369
Sep 22 23:18:02.802 INFO [lossy] skipping 1369
Sep 22 23:18:02.831 DEBG Write     :1369 deps:[JobId(1368), JobId(1365), JobId(1362), JobId(1359), JobId(1356), JobId(1353), JobId(1350), JobId(1347), JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)] res:true
Sep 22 23:18:02.841 DEBG Flush     :1371 extent_limit None deps:[JobId(1370), JobId(1369), JobId(1368)] res:true f:157 g:1
Sep 22 23:18:02.871 DEBG Write     :1372 deps:[JobId(1371), JobId(1368), JobId(1365), JobId(1362), JobId(1359), JobId(1356), JobId(1353), JobId(1350), JobId(1347), JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)] res:true
Sep 22 23:18:02.902 DEBG Write     :1373 deps:[JobId(1371), JobId(1368), JobId(1365), JobId(1362), JobId(1359), JobId(1356), JobId(1353), JobId(1350), JobId(1347), JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)] res:true
Sep 22 23:18:02.912 DEBG Flush     :1374 extent_limit None deps:[JobId(1373), JobId(1372), JobId(1371)] res:true f:158 g:1
Sep 22 23:18:02.912 INFO [lossy] skipping 1375
Sep 22 23:18:02.912 INFO [lossy] skipping 1376
Sep 22 23:18:02.912 WARN 1377 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:18:02.912 WARN 1378 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:18:02.912 WARN 1379 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:18:02.912 INFO [lossy] skipping 1380
Sep 22 23:18:02.913 INFO [lossy] skipping 1375
Sep 22 23:18:02.942 DEBG Write     :1376 deps:[JobId(1374), JobId(1371), JobId(1368), JobId(1365), JobId(1362), JobId(1359), JobId(1356), JobId(1353), JobId(1350), JobId(1347), JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)] res:true
Sep 22 23:18:02.944 INFO [lossy] skipping 1375
Sep 22 23:18:02.944 INFO [lossy] skipping 1375
Sep 22 23:18:02.974 DEBG Write     :1375 deps:[JobId(1374), JobId(1371), JobId(1368), JobId(1365), JobId(1362), JobId(1359), JobId(1356), JobId(1353), JobId(1350), JobId(1347), JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)] res:true
Sep 22 23:18:02.975 INFO [lossy] sleeping 1 second
Sep 22 23:18:03.921 ERRO [0] job id 1341 saw error GenericError("test error")
Sep 22 23:18:03.921 ERRO [0] job id 1364 saw error GenericError("test error")
Sep 22 23:18:03.921 ERRO [0] job id 1366 saw error GenericError("test error")
Sep 22 23:18:03.921 ERRO [0] job id 1369 saw error GenericError("test error")
Sep 22 23:18:03.977 INFO [lossy] skipping 1377
Sep 22 23:18:03.977 INFO [lossy] skipping 1378
Sep 22 23:18:03.977 INFO [lossy] skipping 1380
Sep 22 23:18:03.977 INFO [lossy] skipping 1381
Sep 22 23:18:03.977 INFO [lossy] skipping 1384
Sep 22 23:18:03.977 INFO [lossy] skipping 1391
Sep 22 23:18:03.977 INFO [lossy] skipping 1399
Sep 22 23:18:03.977 INFO [lossy] skipping 1402
Sep 22 23:18:03.986 DEBG Flush     :1377 extent_limit None deps:[JobId(1376), JobId(1375), JobId(1374)] res:true f:159 g:1
Sep 22 23:18:03.986 INFO [lossy] skipping 1378
Sep 22 23:18:03.986 WARN 1380 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:18:03.986 INFO [lossy] skipping 1381
Sep 22 23:18:03.986 INFO [lossy] skipping 1391
Sep 22 23:18:04.016 DEBG Write     :1378 deps:[JobId(1377), JobId(1374), JobId(1371), JobId(1368), JobId(1365), JobId(1362), JobId(1359), JobId(1356), JobId(1353), JobId(1350), JobId(1347), JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)] res:true
Sep 22 23:18:04.046 DEBG Write     :1379 deps:[JobId(1377), JobId(1374), JobId(1371), JobId(1368), JobId(1365), JobId(1362), JobId(1359), JobId(1356), JobId(1353), JobId(1350), JobId(1347), JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)] res:true
Sep 22 23:18:04.047 WARN returning error on flush!
Sep 22 23:18:04.047 DEBG Flush     :1380 extent_limit None deps:[JobId(1379), JobId(1378), JobId(1377)] res:false f:160 g:1
Sep 22 23:18:04.047 INFO [lossy] skipping 1382
Sep 22 23:18:04.047 WARN 1385 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:18:04.047 WARN 1387 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:18:04.048 WARN 1389 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:18:04.048 WARN 1390 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:18:04.048 WARN 1392 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 5 deps, role: work
Sep 22 23:18:04.048 WARN 1393 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 5 deps, role: work
Sep 22 23:18:04.048 INFO [lossy] skipping 1394
Sep 22 23:18:04.048 INFO [lossy] skipping 1395
Sep 22 23:18:04.048 WARN 1396 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 6 deps, role: work
Sep 22 23:18:04.048 INFO [lossy] skipping 1397
Sep 22 23:18:04.048 WARN 1398 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 7 deps, role: work
Sep 22 23:18:04.048 INFO [lossy] skipping 1400
Sep 22 23:18:04.048 WARN 1401 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 8 deps, role: work
Sep 22 23:18:04.048 WARN returning error on flush!
Sep 22 23:18:04.048 DEBG Flush     :1380 extent_limit None deps:[JobId(1379), JobId(1378), JobId(1377)] res:false f:160 g:1
Sep 22 23:18:04.048 WARN 1382 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:18:04.048 WARN 1395 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 6 deps, role: work
Sep 22 23:18:04.048 INFO [lossy] skipping 1397
Sep 22 23:18:04.048 INFO [lossy] skipping 1380
Sep 22 23:18:04.048 INFO [lossy] skipping 1380
Sep 22 23:18:04.056 DEBG Flush     :1380 extent_limit None deps:[JobId(1379), JobId(1378), JobId(1377)] res:true f:160 g:1
Sep 22 23:18:04.085 DEBG Write     :1381 deps:[JobId(1380), JobId(1377), JobId(1374), JobId(1371), JobId(1368), JobId(1365), JobId(1362), JobId(1359), JobId(1356), JobId(1353), JobId(1350), JobId(1347), JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)] res:true
Sep 22 23:18:04.086 INFO [lossy] skipping 1382
Sep 22 23:18:04.086 WARN 1383 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:18:04.086 INFO [lossy] skipping 1384
Sep 22 23:18:04.086 WARN 1385 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:18:04.086 WARN 1387 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:18:04.086 INFO [lossy] skipping 1388
Sep 22 23:18:04.086 WARN 1389 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:18:04.086 WARN 1390 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:18:04.086 WARN 1392 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:18:04.086 WARN 1393 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:18:04.086 WARN 1395 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 5 deps, role: work
Sep 22 23:18:04.086 WARN 1396 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 5 deps, role: work
Sep 22 23:18:04.086 INFO [lossy] skipping 1397
Sep 22 23:18:04.086 WARN 1398 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 6 deps, role: work
Sep 22 23:18:04.086 INFO [lossy] skipping 1399
Sep 22 23:18:04.087 INFO [lossy] skipping 1400
Sep 22 23:18:04.087 WARN 1401 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 7 deps, role: work
Sep 22 23:18:04.087 INFO [lossy] skipping 1402
Sep 22 23:18:04.087 INFO [lossy] skipping 1403
Sep 22 23:18:04.087 WARN returning error on write!
Sep 22 23:18:04.087 DEBG Write     :1382 deps:[JobId(1380), JobId(1377), JobId(1374), JobId(1371), JobId(1368), JobId(1365), JobId(1362), JobId(1359), JobId(1356), JobId(1353), JobId(1350), JobId(1347), JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)] res:false
Sep 22 23:18:04.088 INFO [lossy] skipping 1384
Sep 22 23:18:04.088 WARN 1399 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 6 deps, role: work
Sep 22 23:18:04.088 INFO [lossy] skipping 1382
Sep 22 23:18:04.088 WARN 1384 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:18:04.088 WARN returning error on write!
Sep 22 23:18:04.088 DEBG Write     :1382 deps:[JobId(1380), JobId(1377), JobId(1374), JobId(1371), JobId(1368), JobId(1365), JobId(1362), JobId(1359), JobId(1356), JobId(1353), JobId(1350), JobId(1347), JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)] res:false
Sep 22 23:18:04.089 INFO [lossy] skipping 1382
Sep 22 23:18:04.118 DEBG Write     :1382 deps:[JobId(1380), JobId(1377), JobId(1374), JobId(1371), JobId(1368), JobId(1365), JobId(1362), JobId(1359), JobId(1356), JobId(1353), JobId(1350), JobId(1347), JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)] res:true
Sep 22 23:18:04.119 WARN returning error on flush!
Sep 22 23:18:04.119 DEBG Flush     :1383 extent_limit None deps:[JobId(1382), JobId(1381), JobId(1380)] res:false f:161 g:1
Sep 22 23:18:04.119 INFO [lossy] skipping 1400
Sep 22 23:18:04.119 INFO [lossy] skipping 1383
Sep 22 23:18:04.127 DEBG Flush     :1383 extent_limit None deps:[JobId(1382), JobId(1381), JobId(1380)] res:true f:161 g:1
Sep 22 23:18:04.157 DEBG Write     :1384 deps:[JobId(1383), JobId(1380), JobId(1377), JobId(1374), JobId(1371), JobId(1368), JobId(1365), JobId(1362), JobId(1359), JobId(1356), JobId(1353), JobId(1350), JobId(1347), JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)] res:true
Sep 22 23:18:04.188 DEBG Write     :1385 deps:[JobId(1383), JobId(1380), JobId(1377), JobId(1374), JobId(1371), JobId(1368), JobId(1365), JobId(1362), JobId(1359), JobId(1356), JobId(1353), JobId(1350), JobId(1347), JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)] res:true
Sep 22 23:18:04.198 DEBG Flush     :1386 extent_limit None deps:[JobId(1385), JobId(1384), JobId(1383)] res:true f:162 g:1
Sep 22 23:18:04.198 INFO [lossy] skipping 1387
Sep 22 23:18:04.198 WARN 1388 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:18:04.198 INFO [lossy] skipping 1389
Sep 22 23:18:04.198 INFO [lossy] skipping 1390
Sep 22 23:18:04.198 WARN 1392 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:18:04.198 WARN 1393 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:18:04.198 WARN 1395 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:18:04.198 WARN 1396 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:18:04.198 INFO [lossy] skipping 1398
Sep 22 23:18:04.198 WARN 1399 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:18:04.198 WARN 1401 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 5 deps, role: work
Sep 22 23:18:04.199 WARN returning error on write!
Sep 22 23:18:04.199 DEBG Write     :1387 deps:[JobId(1386), JobId(1383), JobId(1380), JobId(1377), JobId(1374), JobId(1371), JobId(1368), JobId(1365), JobId(1362), JobId(1359), JobId(1356), JobId(1353), JobId(1350), JobId(1347), JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)] res:false
Sep 22 23:18:04.199 INFO [lossy] skipping 1389
Sep 22 23:18:04.200 WARN 1390 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:18:04.200 WARN 1398 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:18:04.229 DEBG Write     :1387 deps:[JobId(1386), JobId(1383), JobId(1380), JobId(1377), JobId(1374), JobId(1371), JobId(1368), JobId(1365), JobId(1362), JobId(1359), JobId(1356), JobId(1353), JobId(1350), JobId(1347), JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)] res:true
Sep 22 23:18:04.230 WARN 1389 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:18:04.230 INFO [lossy] skipping 1388
Sep 22 23:18:04.230 INFO [lossy] skipping 1389
Sep 22 23:18:04.230 INFO [lossy] skipping 1391
Sep 22 23:18:04.230 INFO [lossy] skipping 1392
Sep 22 23:18:04.230 INFO [lossy] skipping 1393
Sep 22 23:18:04.230 INFO [lossy] skipping 1395
Sep 22 23:18:04.230 INFO [lossy] skipping 1388
Sep 22 23:18:04.234 DEBG Flush     :1388 extent_limit None deps:[JobId(1387), JobId(1386)] res:true f:163 g:1
Sep 22 23:18:04.264 DEBG Write     :1389 deps:[JobId(1388), JobId(1386), JobId(1383), JobId(1380), JobId(1377), JobId(1374), JobId(1371), JobId(1368), JobId(1365), JobId(1362), JobId(1359), JobId(1356), JobId(1353), JobId(1350), JobId(1347), JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)] res:true
Sep 22 23:18:04.265 INFO [lossy] skipping 1390
Sep 22 23:18:04.265 WARN 1391 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:18:04.265 WARN 1392 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:18:04.265 WARN 1393 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:18:04.265 WARN 1395 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:18:04.265 INFO [lossy] skipping 1396
Sep 22 23:18:04.265 WARN 1398 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:18:04.265 WARN 1399 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:18:04.265 INFO [lossy] skipping 1400
Sep 22 23:18:04.265 WARN 1401 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:18:04.265 INFO [lossy] skipping 1402
Sep 22 23:18:04.265 INFO [lossy] skipping 1390
Sep 22 23:18:04.265 WARN 1396 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:18:04.265 INFO [lossy] skipping 1402
Sep 22 23:18:04.265 INFO [lossy] skipping 1390
Sep 22 23:18:04.295 DEBG Write     :1390 deps:[JobId(1388), JobId(1386), JobId(1383), JobId(1380), JobId(1377), JobId(1374), JobId(1371), JobId(1368), JobId(1365), JobId(1362), JobId(1359), JobId(1356), JobId(1353), JobId(1350), JobId(1347), JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)] res:true
Sep 22 23:18:04.296 INFO [lossy] sleeping 1 second
Sep 22 23:18:04.296 ERRO [0] job id 1380 saw error GenericError("test error")
Sep 22 23:18:04.296 ERRO [0] job id 1380 saw error GenericError("test error")
Sep 22 23:18:04.296 ERRO [0] job id 1382 saw error GenericError("test error")
Sep 22 23:18:04.296 ERRO [0] job id 1382 saw error GenericError("test error")
Sep 22 23:18:04.296 ERRO [0] job id 1383 saw error GenericError("test error")
Sep 22 23:18:04.296 ERRO [0] job id 1387 saw error GenericError("test error")
Sep 22 23:18:05.296 WARN returning error on flush!
Sep 22 23:18:05.296 DEBG Flush     :1391 extent_limit None deps:[JobId(1390), JobId(1389), JobId(1388)] res:false f:164 g:1
Sep 22 23:18:05.296 INFO [lossy] skipping 1392
Sep 22 23:18:05.296 INFO [lossy] skipping 1394
Sep 22 23:18:05.297 INFO [lossy] skipping 1402
Sep 22 23:18:05.297 INFO [lossy] skipping 1403
Sep 22 23:18:05.305 DEBG Flush     :1391 extent_limit None deps:[JobId(1390), JobId(1389), JobId(1388)] res:true f:164 g:1
Sep 22 23:18:05.335 DEBG Write     :1392 deps:[JobId(1391), JobId(1388), JobId(1386), JobId(1383), JobId(1380), JobId(1377), JobId(1374), JobId(1371), JobId(1368), JobId(1365), JobId(1362), JobId(1359), JobId(1356), JobId(1353), JobId(1350), JobId(1347), JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)] res:true
Sep 22 23:18:05.336 WARN 1394 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:18:05.336 INFO [lossy] skipping 1402
Sep 22 23:18:05.336 INFO [lossy] skipping 1402
Sep 22 23:18:05.336 INFO [lossy] sleeping 1 second
Sep 22 23:18:05.336 ERRO [0] job id 1391 saw error GenericError("test error")
Sep 22 23:18:06.338 INFO [lossy] skipping 1393
Sep 22 23:18:06.338 INFO [lossy] skipping 1394
Sep 22 23:18:06.338 WARN 1395 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:18:06.338 WARN 1396 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:18:06.338 WARN 1398 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:18:06.338 WARN 1399 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:18:06.338 INFO [lossy] skipping 1401
Sep 22 23:18:06.338 INFO [lossy] skipping 1403
Sep 22 23:18:06.369 DEBG Write     :1393 deps:[JobId(1391), JobId(1388), JobId(1386), JobId(1383), JobId(1380), JobId(1377), JobId(1374), JobId(1371), JobId(1368), JobId(1365), JobId(1362), JobId(1359), JobId(1356), JobId(1353), JobId(1350), JobId(1347), JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)] res:true
Sep 22 23:18:06.379 DEBG Flush     :1394 extent_limit None deps:[JobId(1393), JobId(1392), JobId(1391)] res:true f:165 g:1
Sep 22 23:18:06.379 WARN 1401 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:18:06.379 INFO [lossy] skipping 1403
Sep 22 23:18:06.408 DEBG Write     :1395 deps:[JobId(1394), JobId(1391), JobId(1388), JobId(1386), JobId(1383), JobId(1380), JobId(1377), JobId(1374), JobId(1371), JobId(1368), JobId(1365), JobId(1362), JobId(1359), JobId(1356), JobId(1353), JobId(1350), JobId(1347), JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)] res:true
Sep 22 23:18:06.409 INFO [lossy] skipping 1396
Sep 22 23:18:06.409 INFO [lossy] skipping 1397
Sep 22 23:18:06.409 WARN 1398 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:18:06.409 WARN 1399 job Write for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:18:06.439 DEBG Write     :1396 deps:[JobId(1394), JobId(1391), JobId(1388), JobId(1386), JobId(1383), JobId(1380), JobId(1377), JobId(1374), JobId(1371), JobId(1368), JobId(1365), JobId(1362), JobId(1359), JobId(1356), JobId(1353), JobId(1350), JobId(1347), JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)] res:true
Sep 22 23:18:06.448 DEBG Flush     :1397 extent_limit None deps:[JobId(1396), JobId(1395), JobId(1394)] res:true f:166 g:1
Sep 22 23:18:06.478 DEBG Write     :1398 deps:[JobId(1397), JobId(1394), JobId(1391), JobId(1388), JobId(1386), JobId(1383), JobId(1380), JobId(1377), JobId(1374), JobId(1371), JobId(1368), JobId(1365), JobId(1362), JobId(1359), JobId(1356), JobId(1353), JobId(1350), JobId(1347), JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328)] res:true
Sep 22 23:18:06.509 DEBG Write     :1399 deps:[JobId(1397), JobId(1394), JobId(1391), JobId(1388), JobId(1386), JobId(1383), JobId(1380), JobId(1377), JobId(1374), JobId(1371), JobId(1368), JobId(1365), JobId(1362), JobId(1359), JobId(1356), JobId(1353), JobId(1350), JobId(1347), JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1328), JobId(1327)] res:true
Sep 22 23:18:06.518 DEBG Flush     :1400 extent_limit None deps:[JobId(1399), JobId(1398), JobId(1397)] res:true f:167 g:1
Sep 22 23:18:06.547 DEBG Write     :1401 deps:[JobId(1400), JobId(1397), JobId(1394), JobId(1391), JobId(1388), JobId(1386), JobId(1383), JobId(1380), JobId(1377), JobId(1374), JobId(1371), JobId(1368), JobId(1365), JobId(1362), JobId(1359), JobId(1356), JobId(1353), JobId(1350), JobId(1347), JobId(1344), JobId(1341), JobId(1338), JobId(1335), JobId(1332), JobId(1330), JobId(1329), JobId(1328)] res:true
Sep 22 23:18:06.555 DEBG Read      :1402 deps:[JobId(1400)] res:true
Sep 22 23:18:06.578 WARN returning error on flush!
Sep 22 23:18:06.578 DEBG Flush     :1403 extent_limit None deps:[JobId(1402), JobId(1401), JobId(1400)] res:false f:168 g:1
Sep 22 23:18:06.578 INFO [lossy] skipping 1403
Sep 22 23:18:06.578 INFO [lossy] skipping 1403
Sep 22 23:18:06.578 INFO [lossy] skipping 1403
Sep 22 23:18:06.582 DEBG Flush     :1403 extent_limit None deps:[JobId(1402), JobId(1401), JobId(1400)] res:true f:168 g:1
Sep 22 23:18:06.582 INFO [lossy] sleeping 1 second
Sep 22 23:18:06.958 DEBG [0] Read AckReady 1402, : downstairs
Sep 22 23:18:06.959 ERRO [0] job id 1403 saw error GenericError("test error")
Sep 22 23:18:06.959 DEBG up_ds_listen was notified
Sep 22 23:18:06.959 DEBG up_ds_listen process 1402
Sep 22 23:18:06.959 DEBG [A] ack job 1402:403, : downstairs
Sep 22 23:18:07.012 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:18:07.013 DEBG IO Read  1404 has deps [JobId(1403)]
Sep 22 23:18:07.515 DEBG IO Flush 1405 has deps [JobId(1404), JobId(1403)]
Sep 22 23:18:07.582 INFO [lossy] skipping 1404
Sep 22 23:18:07.589 DEBG Read      :1404 deps:[JobId(1403)] res:true
Sep 22 23:18:07.612 INFO [lossy] sleeping 1 second
Sep 22 23:18:07.989 DEBG [0] Read AckReady 1404, : downstairs
Sep 22 23:18:07.990 DEBG up_ds_listen was notified
Sep 22 23:18:07.990 DEBG up_ds_listen process 1404
Sep 22 23:18:07.990 DEBG [A] ack job 1404:405, : downstairs
Sep 22 23:18:08.044 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:18:08.045 DEBG IO Read  1406 has deps [JobId(1405)]
Sep 22 23:18:08.545 DEBG IO Flush 1407 has deps [JobId(1406), JobId(1405)]
Sep 22 23:18:08.612 DEBG Flush     :1405 extent_limit None deps:[JobId(1404), JobId(1403)] res:true f:169 g:1
Sep 22 23:18:08.612 INFO [lossy] skipping 1406
Sep 22 23:18:08.612 INFO [lossy] skipping 1406
Sep 22 23:18:08.619 DEBG Read      :1406 deps:[JobId(1405)] res:true
Sep 22 23:18:08.642 INFO [lossy] skipping 1407
Sep 22 23:18:08.642 DEBG Flush     :1407 extent_limit None deps:[JobId(1406), JobId(1405)] res:true f:170 g:1
Sep 22 23:18:08.642 INFO [lossy] sleeping 1 second
Sep 22 23:18:09.019 DEBG [0] Read AckReady 1406, : downstairs
Sep 22 23:18:09.020 DEBG up_ds_listen was notified
Sep 22 23:18:09.020 DEBG up_ds_listen process 1406
Sep 22 23:18:09.020 DEBG [A] ack job 1406:407, : downstairs
Sep 22 23:18:09.073 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:18:09.074 DEBG IO Read  1408 has deps [JobId(1407)]
Sep 22 23:18:09.575 DEBG IO Flush 1409 has deps [JobId(1408), JobId(1407)]
Sep 22 23:18:09.643 INFO [lossy] skipping 1408
Sep 22 23:18:09.643 INFO [lossy] skipping 1409
Sep 22 23:18:09.650 DEBG Read      :1408 deps:[JobId(1407)] res:true
Sep 22 23:18:09.672 DEBG Flush     :1409 extent_limit None deps:[JobId(1408), JobId(1407)] res:true f:171 g:1
Sep 22 23:18:09.672 INFO [lossy] sleeping 1 second
Sep 22 23:18:10.049 DEBG [0] Read AckReady 1408, : downstairs
Sep 22 23:18:10.050 DEBG up_ds_listen was notified
Sep 22 23:18:10.050 DEBG up_ds_listen process 1408
Sep 22 23:18:10.050 DEBG [A] ack job 1408:409, : downstairs
Sep 22 23:18:10.102 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:18:10.104 DEBG IO Read  1410 has deps [JobId(1409)]
Sep 22 23:18:10.604 DEBG IO Flush 1411 has deps [JobId(1410), JobId(1409)]
Sep 22 23:18:10.674 WARN returning error on read!
Sep 22 23:18:10.674 DEBG Read      :1410 deps:[JobId(1409)] res:false
Sep 22 23:18:10.674 WARN returning error on read!
Sep 22 23:18:10.674 DEBG Read      :1410 deps:[JobId(1409)] res:false
Sep 22 23:18:10.674 INFO [lossy] skipping 1410
Sep 22 23:18:10.674 INFO [lossy] skipping 1410
Sep 22 23:18:10.674 INFO [lossy] skipping 1410
Sep 22 23:18:10.674 INFO [lossy] skipping 1410
Sep 22 23:18:10.681 DEBG Read      :1410 deps:[JobId(1409)] res:true
Sep 22 23:18:10.702 ERRO [0] job id 1410 saw error GenericError("test error")
Sep 22 23:18:10.702 ERRO [0] job id 1410 saw error GenericError("test error")
Sep 22 23:18:10.704 INFO [lossy] skipping 1411
Sep 22 23:18:10.704 WARN returning error on flush!
Sep 22 23:18:10.704 DEBG Flush     :1411 extent_limit None deps:[JobId(1410), JobId(1409)] res:false f:172 g:1
Sep 22 23:18:10.704 DEBG Flush     :1411 extent_limit None deps:[JobId(1410), JobId(1409)] res:true f:172 g:1
Sep 22 23:18:10.704 INFO [lossy] sleeping 1 second
Sep 22 23:18:11.083 DEBG [0] Read AckReady 1410, : downstairs
Sep 22 23:18:11.084 ERRO [0] job id 1411 saw error GenericError("test error")
Sep 22 23:18:11.084 DEBG up_ds_listen was notified
Sep 22 23:18:11.084 DEBG up_ds_listen process 1410
Sep 22 23:18:11.084 DEBG [A] ack job 1410:411, : downstairs
Sep 22 23:18:11.136 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:18:11.138 DEBG IO Read  1412 has deps [JobId(1411)]
Sep 22 23:18:11.638 DEBG IO Flush 1413 has deps [JobId(1412), JobId(1411)]
Sep 22 23:18:11.712 DEBG Read      :1412 deps:[JobId(1411)] res:true
Sep 22 23:18:11.735 WARN returning error on flush!
Sep 22 23:18:11.735 DEBG Flush     :1413 extent_limit None deps:[JobId(1412), JobId(1411)] res:false f:173 g:1
Sep 22 23:18:11.735 DEBG Flush     :1413 extent_limit None deps:[JobId(1412), JobId(1411)] res:true f:173 g:1
Sep 22 23:18:11.735 INFO [lossy] sleeping 1 second
Sep 22 23:18:12.111 DEBG [0] Read AckReady 1412, : downstairs
Sep 22 23:18:12.112 ERRO [0] job id 1413 saw error GenericError("test error")
Sep 22 23:18:12.112 DEBG up_ds_listen was notified
Sep 22 23:18:12.112 DEBG up_ds_listen process 1412
Sep 22 23:18:12.112 DEBG [A] ack job 1412:413, : downstairs
Sep 22 23:18:12.164 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:18:12.166 DEBG IO Read  1414 has deps [JobId(1413)]
Sep 22 23:18:12.666 DEBG IO Flush 1415 has deps [JobId(1414), JobId(1413)]
Sep 22 23:18:12.743 DEBG Read      :1414 deps:[JobId(1413)] res:true
Sep 22 23:18:12.766 DEBG Flush     :1415 extent_limit None deps:[JobId(1414), JobId(1413)] res:true f:174 g:1
Sep 22 23:18:12.766 INFO [lossy] sleeping 1 second
Sep 22 23:18:13.143 DEBG [0] Read AckReady 1414, : downstairs
Sep 22 23:18:13.144 DEBG up_ds_listen was notified
Sep 22 23:18:13.144 DEBG up_ds_listen process 1414
Sep 22 23:18:13.144 DEBG [A] ack job 1414:415, : downstairs
Sep 22 23:18:13.197 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:18:13.198 DEBG IO Read  1416 has deps [JobId(1415)]
Sep 22 23:18:13.700 DEBG IO Flush 1417 has deps [JobId(1416), JobId(1415)]
Sep 22 23:18:13.773 DEBG Read      :1416 deps:[JobId(1415)] res:true
Sep 22 23:18:13.796 DEBG Flush     :1417 extent_limit None deps:[JobId(1416), JobId(1415)] res:true f:175 g:1
Sep 22 23:18:13.796 INFO [lossy] sleeping 1 second
Sep 22 23:18:14.172 DEBG [0] Read AckReady 1416, : downstairs
Sep 22 23:18:14.173 DEBG up_ds_listen was notified
Sep 22 23:18:14.173 DEBG up_ds_listen process 1416
Sep 22 23:18:14.173 DEBG [A] ack job 1416:417, : downstairs
Sep 22 23:18:14.226 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:18:14.227 DEBG IO Read  1418 has deps [JobId(1417)]
Sep 22 23:18:14.728 DEBG IO Flush 1419 has deps [JobId(1418), JobId(1417)]
Sep 22 23:18:14.798 WARN returning error on read!
Sep 22 23:18:14.798 DEBG Read      :1418 deps:[JobId(1417)] res:false
Sep 22 23:18:14.798 WARN returning error on read!
Sep 22 23:18:14.798 DEBG Read      :1418 deps:[JobId(1417)] res:false
Sep 22 23:18:14.804 DEBG Read      :1418 deps:[JobId(1417)] res:true
Sep 22 23:18:14.826 ERRO [0] job id 1418 saw error GenericError("test error")
Sep 22 23:18:14.826 ERRO [0] job id 1418 saw error GenericError("test error")
Sep 22 23:18:14.828 DEBG Flush     :1419 extent_limit None deps:[JobId(1418), JobId(1417)] res:true f:176 g:1
Sep 22 23:18:14.828 INFO [lossy] sleeping 1 second
Sep 22 23:18:15.207 DEBG [0] Read AckReady 1418, : downstairs
Sep 22 23:18:15.208 DEBG up_ds_listen was notified
Sep 22 23:18:15.208 DEBG up_ds_listen process 1418
Sep 22 23:18:15.208 DEBG [A] ack job 1418:419, : downstairs
Sep 22 23:18:15.260 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:18:15.262 DEBG IO Read  1420 has deps [JobId(1419)]
Sep 22 23:18:15.762 DEBG IO Flush 1421 has deps [JobId(1420), JobId(1419)]
Sep 22 23:18:15.829 INFO [lossy] skipping 1420
Sep 22 23:18:15.835 DEBG Read      :1420 deps:[JobId(1419)] res:true
Sep 22 23:18:15.858 INFO [lossy] sleeping 1 second
Sep 22 23:18:16.234 DEBG [0] Read AckReady 1420, : downstairs
Sep 22 23:18:16.235 DEBG up_ds_listen was notified
Sep 22 23:18:16.235 DEBG up_ds_listen process 1420
Sep 22 23:18:16.235 DEBG [A] ack job 1420:421, : downstairs
Sep 22 23:18:16.287 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:18:16.289 DEBG IO Read  1422 has deps [JobId(1421)]
Sep 22 23:18:16.789 DEBG IO Flush 1423 has deps [JobId(1422), JobId(1421)]
Sep 22 23:18:16.860 INFO [lossy] skipping 1421
Sep 22 23:18:16.860 DEBG Flush     :1421 extent_limit None deps:[JobId(1420), JobId(1419)] res:true f:177 g:1
Sep 22 23:18:16.866 DEBG Read      :1422 deps:[JobId(1421)] res:true
Sep 22 23:18:16.889 DEBG Flush     :1423 extent_limit None deps:[JobId(1422), JobId(1421)] res:true f:178 g:1
Sep 22 23:18:16.889 INFO [lossy] sleeping 1 second
Sep 22 23:18:17.265 DEBG [0] Read AckReady 1422, : downstairs
Sep 22 23:18:17.266 DEBG up_ds_listen was notified
Sep 22 23:18:17.266 DEBG up_ds_listen process 1422
Sep 22 23:18:17.266 DEBG [A] ack job 1422:423, : downstairs
Sep 22 23:18:17.319 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:18:17.320 DEBG IO Read  1424 has deps [JobId(1423)]
Sep 22 23:18:17.821 DEBG IO Flush 1425 has deps [JobId(1424), JobId(1423)]
Sep 22 23:18:17.896 DEBG Read      :1424 deps:[JobId(1423)] res:true
Sep 22 23:18:17.919 WARN returning error on flush!
Sep 22 23:18:17.919 DEBG Flush     :1425 extent_limit None deps:[JobId(1424), JobId(1423)] res:false f:179 g:1
Sep 22 23:18:17.919 WARN returning error on flush!
Sep 22 23:18:17.919 DEBG Flush     :1425 extent_limit None deps:[JobId(1424), JobId(1423)] res:false f:179 g:1
Sep 22 23:18:17.919 DEBG Flush     :1425 extent_limit None deps:[JobId(1424), JobId(1423)] res:true f:179 g:1
Sep 22 23:18:17.919 INFO [lossy] sleeping 1 second
Sep 22 23:18:18.298 DEBG [0] Read AckReady 1424, : downstairs
Sep 22 23:18:18.299 ERRO [0] job id 1425 saw error GenericError("test error")
Sep 22 23:18:18.299 ERRO [0] job id 1425 saw error GenericError("test error")
Sep 22 23:18:18.299 DEBG up_ds_listen was notified
Sep 22 23:18:18.299 DEBG up_ds_listen process 1424
Sep 22 23:18:18.300 DEBG [A] ack job 1424:425, : downstairs
Sep 22 23:18:18.353 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:18:18.354 DEBG IO Read  1426 has deps [JobId(1425)]
Sep 22 23:18:18.855 DEBG IO Flush 1427 has deps [JobId(1426), JobId(1425)]
Sep 22 23:18:18.926 DEBG Read      :1426 deps:[JobId(1425)] res:true
Sep 22 23:18:18.949 DEBG Flush     :1427 extent_limit None deps:[JobId(1426), JobId(1425)] res:true f:180 g:1
Sep 22 23:18:18.949 INFO [lossy] sleeping 1 second
Sep 22 23:18:19.325 DEBG [0] Read AckReady 1426, : downstairs
Sep 22 23:18:19.326 DEBG up_ds_listen was notified
Sep 22 23:18:19.326 DEBG up_ds_listen process 1426
Sep 22 23:18:19.326 DEBG [A] ack job 1426:427, : downstairs
Sep 22 23:18:19.379 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:18:19.380 DEBG IO Read  1428 has deps [JobId(1427)]
Sep 22 23:18:19.881 DEBG IO Flush 1429 has deps [JobId(1428), JobId(1427)]
Sep 22 23:18:19.950 INFO [lossy] skipping 1428
Sep 22 23:18:19.950 INFO [lossy] skipping 1429
Sep 22 23:18:19.950 INFO [lossy] skipping 1428
Sep 22 23:18:19.950 INFO [lossy] skipping 1428
Sep 22 23:18:19.957 DEBG Read      :1428 deps:[JobId(1427)] res:true
Sep 22 23:18:19.979 DEBG Flush     :1429 extent_limit None deps:[JobId(1428), JobId(1427)] res:true f:181 g:1
Sep 22 23:18:19.979 INFO [lossy] sleeping 1 second
Sep 22 23:18:20.356 DEBG [0] Read AckReady 1428, : downstairs
Sep 22 23:18:20.357 DEBG up_ds_listen was notified
Sep 22 23:18:20.357 DEBG up_ds_listen process 1428
Sep 22 23:18:20.357 DEBG [A] ack job 1428:429, : downstairs
Sep 22 23:18:20.409 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:18:20.410 DEBG IO Read  1430 has deps [JobId(1429)]
Sep 22 23:18:20.911 DEBG IO Flush 1431 has deps [JobId(1430), JobId(1429)]
Sep 22 23:18:20.981 INFO [lossy] skipping 1430
Sep 22 23:18:20.981 INFO [lossy] skipping 1430
Sep 22 23:18:20.981 INFO [lossy] skipping 1430
Sep 22 23:18:20.981 INFO [lossy] skipping 1430
Sep 22 23:18:20.988 DEBG Read      :1430 deps:[JobId(1429)] res:true
Sep 22 23:18:21.011 INFO [lossy] sleeping 1 second
Sep 22 23:18:21.387 DEBG [0] Read AckReady 1430, : downstairs
Sep 22 23:18:21.388 DEBG up_ds_listen was notified
Sep 22 23:18:21.388 DEBG up_ds_listen process 1430
Sep 22 23:18:21.388 DEBG [A] ack job 1430:431, : downstairs
Sep 22 23:18:21.441 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:18:21.442 DEBG IO Read  1432 has deps [JobId(1431)]
Sep 22 23:18:21.943 DEBG IO Flush 1433 has deps [JobId(1432), JobId(1431)]
Sep 22 23:18:22.012 DEBG Flush     :1431 extent_limit None deps:[JobId(1430), JobId(1429)] res:true f:182 g:1
Sep 22 23:18:22.018 DEBG Read      :1432 deps:[JobId(1431)] res:true
Sep 22 23:18:22.041 INFO [lossy] skipping 1433
Sep 22 23:18:22.041 INFO [lossy] skipping 1433
Sep 22 23:18:22.041 INFO [lossy] skipping 1433
Sep 22 23:18:22.041 DEBG Flush     :1433 extent_limit None deps:[JobId(1432), JobId(1431)] res:true f:183 g:1
Sep 22 23:18:22.041 INFO [lossy] sleeping 1 second
Sep 22 23:18:22.416 DEBG [0] Read AckReady 1432, : downstairs
Sep 22 23:18:22.417 DEBG up_ds_listen was notified
Sep 22 23:18:22.417 DEBG up_ds_listen process 1432
Sep 22 23:18:22.417 DEBG [A] ack job 1432:433, : downstairs
Sep 22 23:18:22.469 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:18:22.471 DEBG IO Read  1434 has deps [JobId(1433)]
Sep 22 23:18:22.972 DEBG IO Flush 1435 has deps [JobId(1434), JobId(1433)]
Sep 22 23:18:23.048 DEBG Read      :1434 deps:[JobId(1433)] res:true
Sep 22 23:18:23.070 DEBG Flush     :1435 extent_limit None deps:[JobId(1434), JobId(1433)] res:true f:184 g:1
Sep 22 23:18:23.071 INFO [lossy] sleeping 1 second
Sep 22 23:18:23.449 DEBG [0] Read AckReady 1434, : downstairs
Sep 22 23:18:23.450 DEBG up_ds_listen was notified
Sep 22 23:18:23.450 DEBG up_ds_listen process 1434
Sep 22 23:18:23.450 DEBG [A] ack job 1434:435, : downstairs
Sep 22 23:18:23.503 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:18:23.504 DEBG IO Read  1436 has deps [JobId(1435)]
Sep 22 23:18:24.006 DEBG IO Flush 1437 has deps [JobId(1436), JobId(1435)]
Sep 22 23:18:24.078 DEBG Read      :1436 deps:[JobId(1435)] res:true
Sep 22 23:18:24.101 DEBG Flush     :1437 extent_limit None deps:[JobId(1436), JobId(1435)] res:true f:185 g:1
Sep 22 23:18:24.102 INFO [lossy] sleeping 1 second
Sep 22 23:18:24.480 DEBG [0] Read AckReady 1436, : downstairs
Sep 22 23:18:24.481 DEBG up_ds_listen was notified
Sep 22 23:18:24.481 DEBG up_ds_listen process 1436
Sep 22 23:18:24.481 DEBG [A] ack job 1436:437, : downstairs
Sep 22 23:18:24.534 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:18:24.535 DEBG IO Read  1438 has deps [JobId(1437)]
Sep 22 23:18:25.036 DEBG IO Flush 1439 has deps [JobId(1438), JobId(1437)]
Sep 22 23:18:25.109 DEBG Read      :1438 deps:[JobId(1437)] res:true
Sep 22 23:18:25.132 DEBG Flush     :1439 extent_limit None deps:[JobId(1438), JobId(1437)] res:true f:186 g:1
Sep 22 23:18:25.509 DEBG [0] Read AckReady 1438, : downstairs
Sep 22 23:18:25.510 DEBG up_ds_listen was notified
Sep 22 23:18:25.510 DEBG up_ds_listen process 1438
Sep 22 23:18:25.510 DEBG [A] ack job 1438:439, : downstairs
Sep 22 23:18:25.563 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:18:25.565 DEBG IO Read  1440 has deps [JobId(1439)]
Sep 22 23:18:25.569 WARN returning error on read!
Sep 22 23:18:25.569 DEBG Read      :1440 deps:[JobId(1439)] res:false
Sep 22 23:18:25.569 INFO [lossy] skipping 1440
Sep 22 23:18:25.575 DEBG Read      :1440 deps:[JobId(1439)] res:true
Sep 22 23:18:25.597 ERRO [0] job id 1440 saw error GenericError("test error")
Sep 22 23:18:25.978 DEBG [0] Read AckReady 1440, : downstairs
Sep 22 23:18:25.979 DEBG up_ds_listen was notified
Sep 22 23:18:25.979 DEBG up_ds_listen process 1440
Sep 22 23:18:25.979 DEBG [A] ack job 1440:441, : downstairs
Sep 22 23:18:26.032 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:18:26.034 DEBG IO Read  1441 has deps [JobId(1439)]
Sep 22 23:18:26.044 DEBG Read      :1441 deps:[JobId(1439)] res:true
Sep 22 23:18:26.066 DEBG IO Flush 1442 has deps [JobId(1441), JobId(1440), JobId(1439)]
Sep 22 23:18:26.068 INFO [lossy] sleeping 1 second
Sep 22 23:18:26.445 DEBG [0] Read AckReady 1441, : downstairs
Sep 22 23:18:26.446 DEBG up_ds_listen was notified
Sep 22 23:18:26.446 DEBG up_ds_listen process 1441
Sep 22 23:18:26.447 DEBG [A] ack job 1441:442, : downstairs
Sep 22 23:18:26.499 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:18:26.500 DEBG IO Read  1443 has deps [JobId(1442)]
Sep 22 23:18:26.567 DEBG IO Flush 1444 has deps [JobId(1443), JobId(1442)]
Sep 22 23:18:27.069 DEBG Flush     :1442 extent_limit None deps:[JobId(1441), JobId(1440), JobId(1439)] res:true f:187 g:1
Sep 22 23:18:27.075 DEBG Read      :1443 deps:[JobId(1442)] res:true
Sep 22 23:18:27.098 INFO [lossy] skipping 1444
Sep 22 23:18:27.098 DEBG Flush     :1444 extent_limit None deps:[JobId(1443), JobId(1442)] res:true f:188 g:1
Sep 22 23:18:27.476 DEBG [0] Read AckReady 1443, : downstairs
Sep 22 23:18:27.477 DEBG up_ds_listen was notified
Sep 22 23:18:27.477 DEBG up_ds_listen process 1443
Sep 22 23:18:27.477 DEBG [A] ack job 1443:444, : downstairs
Sep 22 23:18:27.530 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:18:27.531 DEBG IO Read  1445 has deps [JobId(1444)]
Sep 22 23:18:27.542 DEBG Read      :1445 deps:[JobId(1444)] res:true
Sep 22 23:18:27.945 DEBG [0] Read AckReady 1445, : downstairs
Sep 22 23:18:27.946 DEBG up_ds_listen was notified
Sep 22 23:18:27.946 DEBG up_ds_listen process 1445
Sep 22 23:18:27.946 DEBG [A] ack job 1445:446, : downstairs
Sep 22 23:18:27.998 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:18:28.000 DEBG IO Flush 1446 has deps [JobId(1445), JobId(1444)]
Sep 22 23:18:28.000 DEBG IO Read  1447 has deps [JobId(1446)]
Sep 22 23:18:28.002 WARN returning error on flush!
Sep 22 23:18:28.002 DEBG Flush     :1446 extent_limit None deps:[JobId(1445), JobId(1444)] res:false f:189 g:1
Sep 22 23:18:28.002 INFO [lossy] skipping 1446
Sep 22 23:18:28.002 DEBG Flush     :1446 extent_limit None deps:[JobId(1445), JobId(1444)] res:true f:189 g:1
Sep 22 23:18:28.004 ERRO [0] job id 1446 saw error GenericError("test error")
Sep 22 23:18:28.005 INFO [lossy] skipping 1447
Sep 22 23:18:28.005 INFO [lossy] skipping 1447
Sep 22 23:18:28.011 DEBG Read      :1447 deps:[JobId(1446)] res:true
Sep 22 23:18:28.411 DEBG [0] Read AckReady 1447, : downstairs
Sep 22 23:18:28.411 DEBG up_ds_listen was notified
Sep 22 23:18:28.411 DEBG up_ds_listen process 1447
Sep 22 23:18:28.412 DEBG [A] ack job 1447:448, : downstairs
Sep 22 23:18:28.464 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:18:28.466 DEBG IO Read  1448 has deps [JobId(1446)]
Sep 22 23:18:28.476 DEBG Read      :1448 deps:[JobId(1446)] res:true
Sep 22 23:18:28.876 DEBG [0] Read AckReady 1448, : downstairs
Sep 22 23:18:28.877 DEBG up_ds_listen was notified
Sep 22 23:18:28.877 DEBG up_ds_listen process 1448
Sep 22 23:18:28.877 DEBG [A] ack job 1448:449, : downstairs
Sep 22 23:18:28.929 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:18:28.930 DEBG IO Flush 1449 has deps [JobId(1448), JobId(1447), JobId(1446)]
Sep 22 23:18:28.931 DEBG IO Read  1450 has deps [JobId(1449)]
Sep 22 23:18:28.933 DEBG Flush     :1449 extent_limit None deps:[JobId(1448), JobId(1447), JobId(1446)] res:true f:190 g:1
Sep 22 23:18:28.942 DEBG Read      :1450 deps:[JobId(1449)] res:true
Sep 22 23:18:29.341 DEBG [0] Read AckReady 1450, : downstairs
Sep 22 23:18:29.342 DEBG up_ds_listen was notified
Sep 22 23:18:29.342 DEBG up_ds_listen process 1450
Sep 22 23:18:29.343 DEBG [A] ack job 1450:451, : downstairs
Sep 22 23:18:29.395 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:18:29.396 DEBG IO Read  1451 has deps [JobId(1449)]
Sep 22 23:18:29.406 DEBG Read      :1451 deps:[JobId(1449)] res:true
Sep 22 23:18:29.804 DEBG [0] Read AckReady 1451, : downstairs
Sep 22 23:18:29.805 DEBG up_ds_listen was notified
Sep 22 23:18:29.805 DEBG up_ds_listen process 1451
Sep 22 23:18:29.805 DEBG [A] ack job 1451:452, : downstairs
Sep 22 23:18:29.857 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:18:29.858 DEBG IO Flush 1452 has deps [JobId(1451), JobId(1450), JobId(1449)]
Sep 22 23:18:29.858 INFO [lossy] sleeping 1 second
Sep 22 23:18:29.858 DEBG IO Read  1453 has deps [JobId(1452)]
Sep 22 23:18:30.360 DEBG IO Flush 1454 has deps [JobId(1453), JobId(1452)]
Sep 22 23:18:30.859 WARN returning error on flush!
Sep 22 23:18:30.859 DEBG Flush     :1452 extent_limit None deps:[JobId(1451), JobId(1450), JobId(1449)] res:false f:191 g:1
Sep 22 23:18:30.859 DEBG Flush     :1452 extent_limit None deps:[JobId(1451), JobId(1450), JobId(1449)] res:true f:191 g:1
Sep 22 23:18:30.860 WARN returning error on read!
Sep 22 23:18:30.860 DEBG Read      :1453 deps:[JobId(1452)] res:false
Sep 22 23:18:30.860 INFO [lossy] skipping 1454
Sep 22 23:18:30.860 INFO [lossy] skipping 1453
Sep 22 23:18:30.860 WARN 1454 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:18:30.860 WARN returning error on read!
Sep 22 23:18:30.860 DEBG Read      :1453 deps:[JobId(1452)] res:false
Sep 22 23:18:30.866 DEBG Read      :1453 deps:[JobId(1452)] res:true
Sep 22 23:18:30.887 ERRO [0] job id 1452 saw error GenericError("test error")
Sep 22 23:18:30.888 ERRO [0] job id 1453 saw error GenericError("test error")
Sep 22 23:18:30.888 ERRO [0] job id 1453 saw error GenericError("test error")
Sep 22 23:18:30.889 INFO [lossy] sleeping 1 second
Sep 22 23:18:31.266 DEBG [0] Read AckReady 1453, : downstairs
Sep 22 23:18:31.267 DEBG up_ds_listen was notified
Sep 22 23:18:31.267 DEBG up_ds_listen process 1453
Sep 22 23:18:31.267 DEBG [A] ack job 1453:454, : downstairs
Sep 22 23:18:31.319 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:18:31.320 DEBG IO Read  1455 has deps [JobId(1454)]
Sep 22 23:18:31.389 DEBG IO Flush 1456 has deps [JobId(1455), JobId(1454)]
Sep 22 23:18:31.890 INFO [lossy] skipping 1454
Sep 22 23:18:31.890 DEBG Flush     :1454 extent_limit None deps:[JobId(1453), JobId(1452)] res:true f:192 g:1
Sep 22 23:18:31.890 INFO [lossy] sleeping 1 second
Sep 22 23:18:32.891 INFO [lossy] skipping 1455
Sep 22 23:18:32.891 WARN 1456 job Flush for connection UpstairsConnection { upstairs_id: 4cf66ffa-a12c-4555-b256-f1d6116cf5e6, session_id: 61abb1c6-aab0-4918-867b-523054aca8f6, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:18:32.891 WARN returning error on read!
Sep 22 23:18:32.891 DEBG Read      :1455 deps:[JobId(1454)] res:false
Sep 22 23:18:32.898 DEBG Read      :1455 deps:[JobId(1454)] res:true
Sep 22 23:18:32.919 ERRO [0] job id 1455 saw error GenericError("test error")
Sep 22 23:18:32.920 INFO [lossy] sleeping 1 second
Sep 22 23:18:33.296 DEBG [0] Read AckReady 1455, : downstairs
Sep 22 23:18:33.297 DEBG up_ds_listen was notified
Sep 22 23:18:33.297 DEBG up_ds_listen process 1455
Sep 22 23:18:33.297 DEBG [A] ack job 1455:456, : downstairs
Sep 22 23:18:33.349 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:18:33.351 DEBG IO Read  1457 has deps [JobId(1456)]
Sep 22 23:18:33.420 DEBG IO Flush 1458 has deps [JobId(1457), JobId(1456)]
Sep 22 23:18:33.921 DEBG Flush     :1456 extent_limit None deps:[JobId(1455), JobId(1454)] res:true f:193 g:1
Sep 22 23:18:33.928 DEBG Read      :1457 deps:[JobId(1456)] res:true
Sep 22 23:18:33.950 WARN returning error on flush!
Sep 22 23:18:33.950 DEBG Flush     :1458 extent_limit None deps:[JobId(1457), JobId(1456)] res:false f:194 g:1
Sep 22 23:18:33.951 DEBG Flush     :1458 extent_limit None deps:[JobId(1457), JobId(1456)] res:true f:194 g:1
Sep 22 23:18:33.951 INFO [lossy] sleeping 1 second
Sep 22 23:18:34.326 DEBG [0] Read AckReady 1457, : downstairs
Sep 22 23:18:34.327 ERRO [0] job id 1458 saw error GenericError("test error")
Sep 22 23:18:34.327 DEBG up_ds_listen was notified
Sep 22 23:18:34.327 DEBG up_ds_listen process 1457
Sep 22 23:18:34.327 DEBG [A] ack job 1457:458, : downstairs
Sep 22 23:18:34.380 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:18:34.381 DEBG IO Read  1459 has deps [JobId(1458)]
Sep 22 23:18:34.450 DEBG IO Flush 1460 has deps [JobId(1459), JobId(1458)]
Sep 22 23:18:34.957 DEBG Read      :1459 deps:[JobId(1458)] res:true
Sep 22 23:18:34.980 DEBG Flush     :1460 extent_limit None deps:[JobId(1459), JobId(1458)] res:true f:195 g:1
Sep 22 23:18:34.980 INFO [lossy] sleeping 1 second
Sep 22 23:18:35.356 DEBG [0] Read AckReady 1459, : downstairs
Sep 22 23:18:35.357 DEBG up_ds_listen was notified
Sep 22 23:18:35.357 DEBG up_ds_listen process 1459
Sep 22 23:18:35.357 DEBG [A] ack job 1459:460, : downstairs
Sep 22 23:18:35.409 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:18:35.410 DEBG IO Read  1461 has deps [JobId(1460)]
Sep 22 23:18:35.481 DEBG IO Flush 1462 has deps [JobId(1461), JobId(1460)]
Sep 22 23:18:35.987 DEBG Read      :1461 deps:[JobId(1460)] res:true
Sep 22 23:18:36.010 DEBG Flush     :1462 extent_limit None deps:[JobId(1461), JobId(1460)] res:true f:196 g:1
Sep 22 23:18:36.010 INFO [lossy] sleeping 1 second
Sep 22 23:18:36.386 DEBG [0] Read AckReady 1461, : downstairs
Sep 22 23:18:36.386 DEBG up_ds_listen was notified
Sep 22 23:18:36.386 DEBG up_ds_listen process 1461
Sep 22 23:18:36.386 DEBG [A] ack job 1461:462, : downstairs
Sep 22 23:18:36.439 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:18:36.440 DEBG IO Read  1463 has deps [JobId(1462)]
Sep 22 23:18:36.509 DEBG IO Flush 1464 has deps [JobId(1463), JobId(1462)]
Sep 22 23:18:37.011 INFO [lossy] skipping 1463
Sep 22 23:18:37.011 INFO [lossy] skipping 1464
Sep 22 23:18:37.011 INFO [lossy] skipping 1463
Sep 22 23:18:37.011 INFO [lossy] skipping 1464
Sep 22 23:18:37.011 WARN returning error on read!
Sep 22 23:18:37.011 DEBG Read      :1463 deps:[JobId(1462)] res:false
Sep 22 23:18:37.011 INFO [lossy] skipping 1463
Sep 22 23:18:37.018 DEBG Read      :1463 deps:[JobId(1462)] res:true
Sep 22 23:18:37.039 ERRO [0] job id 1463 saw error GenericError("test error")
Sep 22 23:18:37.040 INFO [lossy] sleeping 1 second
Sep 22 23:18:37.416 DEBG [0] Read AckReady 1463, : downstairs
Sep 22 23:18:37.417 DEBG up_ds_listen was notified
Sep 22 23:18:37.417 DEBG up_ds_listen process 1463
Sep 22 23:18:37.417 DEBG [A] ack job 1463:464, : downstairs
Sep 22 23:18:37.469 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:18:37.470 DEBG IO Read  1465 has deps [JobId(1464)]
Sep 22 23:18:37.511 DEBG IO Flush 1466 has deps [JobId(1465), JobId(1464)]
Sep 22 23:18:38.041 INFO [lossy] skipping 1464
Sep 22 23:18:38.041 INFO [lossy] skipping 1465
Sep 22 23:18:38.041 DEBG Flush     :1464 extent_limit None deps:[JobId(1463), JobId(1462)] res:true f:197 g:1
Sep 22 23:18:38.048 DEBG Read      :1465 deps:[JobId(1464)] res:true
Sep 22 23:18:38.071 INFO [lossy] sleeping 1 second
Sep 22 23:18:38.446 DEBG [0] Read AckReady 1465, : downstairs
Sep 22 23:18:38.447 DEBG up_ds_listen was notified
Sep 22 23:18:38.447 DEBG up_ds_listen process 1465
Sep 22 23:18:38.447 DEBG [A] ack job 1465:466, : downstairs
Sep 22 23:18:38.499 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:18:38.500 DEBG IO Read  1467 has deps [JobId(1466)]
Sep 22 23:18:38.514 DEBG IO Flush 1468 has deps [JobId(1467), JobId(1466)]
Sep 22 23:18:39.072 DEBG Flush     :1466 extent_limit None deps:[JobId(1465), JobId(1464)] res:true f:198 g:1
Sep 22 23:18:39.079 DEBG Read      :1467 deps:[JobId(1466)] res:true
Sep 22 23:18:39.102 DEBG Flush     :1468 extent_limit None deps:[JobId(1467), JobId(1466)] res:true f:199 g:1
Sep 22 23:18:39.477 DEBG [0] Read AckReady 1467, : downstairs
Sep 22 23:18:39.478 DEBG up_ds_listen was notified
Sep 22 23:18:39.478 DEBG up_ds_listen process 1467
Sep 22 23:18:39.478 DEBG [A] ack job 1467:468, : downstairs
Sep 22 23:18:39.530 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:18:39.531 DEBG IO Read  1469 has deps [JobId(1468)]
Sep 22 23:18:39.541 DEBG Read      :1469 deps:[JobId(1468)] res:true
Sep 22 23:18:39.938 DEBG [0] Read AckReady 1469, : downstairs
Sep 22 23:18:39.939 DEBG up_ds_listen was notified
Sep 22 23:18:39.939 DEBG up_ds_listen process 1469
Sep 22 23:18:39.939 DEBG [A] ack job 1469:470, : downstairs
Sep 22 23:18:39.991 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:18:39.992 DEBG IO Read  1470 has deps [JobId(1468)]
Sep 22 23:18:40.002 DEBG Read      :1470 deps:[JobId(1468)] res:true
Sep 22 23:18:40.400 DEBG [0] Read AckReady 1470, : downstairs
Sep 22 23:18:40.400 DEBG up_ds_listen was notified
Sep 22 23:18:40.400 DEBG up_ds_listen process 1470
Sep 22 23:18:40.401 DEBG [A] ack job 1470:471, : downstairs
Sep 22 23:18:40.453 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:18:40.454 DEBG IO Flush 1471 has deps [JobId(1470), JobId(1469), JobId(1468)]
Sep 22 23:18:40.454 DEBG IO Read  1472 has deps [JobId(1471)]
Sep 22 23:18:40.456 INFO [lossy] sleeping 1 second
Sep 22 23:18:40.955 DEBG IO Flush 1473 has deps [JobId(1472), JobId(1471)]
Sep 22 23:18:41.456 DEBG Flush     :1471 extent_limit None deps:[JobId(1470), JobId(1469), JobId(1468)] res:true f:200 g:1
Sep 22 23:18:41.463 DEBG Read      :1472 deps:[JobId(1471)] res:true
Sep 22 23:18:41.486 WARN returning error on flush!
Sep 22 23:18:41.486 DEBG Flush     :1473 extent_limit None deps:[JobId(1472), JobId(1471)] res:false f:201 g:1
Sep 22 23:18:41.486 DEBG Flush     :1473 extent_limit None deps:[JobId(1472), JobId(1471)] res:true f:201 g:1
Sep 22 23:18:41.862 DEBG [0] Read AckReady 1472, : downstairs
Sep 22 23:18:41.862 ERRO [0] job id 1473 saw error GenericError("test error")
Sep 22 23:18:41.862 DEBG up_ds_listen was notified
Sep 22 23:18:41.862 DEBG up_ds_listen process 1472
Sep 22 23:18:41.862 DEBG [A] ack job 1472:473, : downstairs
Sep 22 23:18:41.915 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:18:41.916 DEBG IO Read  1474 has deps [JobId(1473)]
Sep 22 23:18:41.925 DEBG Read      :1474 deps:[JobId(1473)] res:true
Sep 22 23:18:42.324 DEBG [0] Read AckReady 1474, : downstairs
Sep 22 23:18:42.324 DEBG up_ds_listen was notified
Sep 22 23:18:42.324 DEBG up_ds_listen process 1474
Sep 22 23:18:42.324 DEBG [A] ack job 1474:475, : downstairs
Sep 22 23:18:42.377 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:18:42.378 DEBG IO Flush 1475 has deps [JobId(1474), JobId(1473)]
Sep 22 23:18:42.378 DEBG IO Read  1476 has deps [JobId(1475)]
Sep 22 23:18:42.380 DEBG Flush     :1475 extent_limit None deps:[JobId(1474), JobId(1473)] res:true f:202 g:1
Sep 22 23:18:42.383 WARN returning error on read!
Sep 22 23:18:42.383 DEBG Read      :1476 deps:[JobId(1475)] res:false
Sep 22 23:18:42.383 INFO [lossy] skipping 1476
Sep 22 23:18:42.383 INFO [lossy] skipping 1476
Sep 22 23:18:42.383 WARN returning error on read!
Sep 22 23:18:42.383 DEBG Read      :1476 deps:[JobId(1475)] res:false
Sep 22 23:18:42.383 WARN returning error on read!
Sep 22 23:18:42.383 DEBG Read      :1476 deps:[JobId(1475)] res:false
Sep 22 23:18:42.383 INFO [lossy] skipping 1476
Sep 22 23:18:42.383 INFO [lossy] skipping 1476
Sep 22 23:18:42.389 DEBG Read      :1476 deps:[JobId(1475)] res:true
Sep 22 23:18:42.410 ERRO [0] job id 1476 saw error GenericError("test error")
Sep 22 23:18:42.410 ERRO [0] job id 1476 saw error GenericError("test error")
Sep 22 23:18:42.410 ERRO [0] job id 1476 saw error GenericError("test error")
Sep 22 23:18:42.787 DEBG [0] Read AckReady 1476, : downstairs
Sep 22 23:18:42.787 DEBG up_ds_listen was notified
Sep 22 23:18:42.787 DEBG up_ds_listen process 1476
Sep 22 23:18:42.787 DEBG [A] ack job 1476:477, : downstairs
Sep 22 23:18:42.839 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:18:42.840 DEBG IO Read  1477 has deps [JobId(1475)]
Sep 22 23:18:42.845 INFO [lossy] skipping 1477
Sep 22 23:18:42.850 DEBG Read      :1477 deps:[JobId(1475)] res:true
Sep 22 23:18:43.248 DEBG [0] Read AckReady 1477, : downstairs
Sep 22 23:18:43.249 DEBG up_ds_listen was notified
Sep 22 23:18:43.249 DEBG up_ds_listen process 1477
Sep 22 23:18:43.249 DEBG [A] ack job 1477:478, : downstairs
Sep 22 23:18:43.301 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:18:43.302 DEBG IO Flush 1478 has deps [JobId(1477), JobId(1476), JobId(1475)]
Sep 22 23:18:43.303 DEBG IO Read  1479 has deps [JobId(1478)]
Sep 22 23:18:43.305 INFO [lossy] sleeping 1 second
Sep 22 23:18:43.804 DEBG IO Flush 1480 has deps [JobId(1479), JobId(1478)]
Sep 22 23:18:44.306 INFO [lossy] skipping 1478
Sep 22 23:18:44.306 DEBG Flush     :1478 extent_limit None deps:[JobId(1477), JobId(1476), JobId(1475)] res:true f:203 g:1
Sep 22 23:18:44.306 INFO [lossy] sleeping 1 second
Sep 22 23:18:45.313 DEBG Read      :1479 deps:[JobId(1478)] res:true
Sep 22 23:18:45.336 WARN returning error on flush!
Sep 22 23:18:45.336 DEBG Flush     :1480 extent_limit None deps:[JobId(1479), JobId(1478)] res:false f:204 g:1
Sep 22 23:18:45.336 INFO [lossy] skipping 1480
Sep 22 23:18:45.336 WARN returning error on flush!
Sep 22 23:18:45.336 DEBG Flush     :1480 extent_limit None deps:[JobId(1479), JobId(1478)] res:false f:204 g:1
Sep 22 23:18:45.336 WARN returning error on flush!
Sep 22 23:18:45.336 DEBG Flush     :1480 extent_limit None deps:[JobId(1479), JobId(1478)] res:false f:204 g:1
Sep 22 23:18:45.337 DEBG Flush     :1480 extent_limit None deps:[JobId(1479), JobId(1478)] res:true f:204 g:1
Sep 22 23:18:45.712 DEBG [0] Read AckReady 1479, : downstairs
Sep 22 23:18:45.713 ERRO [0] job id 1480 saw error GenericError("test error")
Sep 22 23:18:45.713 ERRO [0] job id 1480 saw error GenericError("test error")
Sep 22 23:18:45.713 ERRO [0] job id 1480 saw error GenericError("test error")
Sep 22 23:18:45.713 DEBG up_ds_listen was notified
Sep 22 23:18:45.713 DEBG up_ds_listen process 1479
Sep 22 23:18:45.714 DEBG [A] ack job 1479:480, : downstairs
Sep 22 23:18:45.766 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:18:45.767 DEBG IO Read  1481 has deps [JobId(1480)]
Sep 22 23:18:45.771 INFO [lossy] skipping 1481
Sep 22 23:18:45.777 DEBG Read      :1481 deps:[JobId(1480)] res:true
Sep 22 23:18:46.176 DEBG [0] Read AckReady 1481, : downstairs
Sep 22 23:18:46.177 DEBG up_ds_listen was notified
Sep 22 23:18:46.177 DEBG up_ds_listen process 1481
Sep 22 23:18:46.177 DEBG [A] ack job 1481:482, : downstairs
Sep 22 23:18:46.230 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:18:46.231 DEBG IO Flush 1482 has deps [JobId(1481), JobId(1480)]
Sep 22 23:18:46.231 DEBG IO Read  1483 has deps [JobId(1482)]
Sep 22 23:18:46.233 WARN returning error on flush!
Sep 22 23:18:46.233 DEBG Flush     :1482 extent_limit None deps:[JobId(1481), JobId(1480)] res:false f:205 g:1
Sep 22 23:18:46.233 DEBG Flush     :1482 extent_limit None deps:[JobId(1481), JobId(1480)] res:true f:205 g:1
Sep 22 23:18:46.236 ERRO [0] job id 1482 saw error GenericError("test error")
Sep 22 23:18:46.241 DEBG Read      :1483 deps:[JobId(1482)] res:true
Sep 22 23:18:46.640 DEBG [0] Read AckReady 1483, : downstairs
Sep 22 23:18:46.641 DEBG up_ds_listen was notified
Sep 22 23:18:46.641 DEBG up_ds_listen process 1483
Sep 22 23:18:46.641 DEBG [A] ack job 1483:484, : downstairs
Sep 22 23:18:46.693 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:18:46.695 DEBG IO Read  1484 has deps [JobId(1482)]
Sep 22 23:18:46.705 DEBG Read      :1484 deps:[JobId(1482)] res:true
Sep 22 23:18:47.103 DEBG [0] Read AckReady 1484, : downstairs
Sep 22 23:18:47.104 DEBG up_ds_listen was notified
Sep 22 23:18:47.104 DEBG up_ds_listen process 1484
Sep 22 23:18:47.104 DEBG [A] ack job 1484:485, : downstairs
Sep 22 23:18:47.156 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:18:47.157 DEBG IO Flush 1485 has deps [JobId(1484), JobId(1483), JobId(1482)]
Sep 22 23:18:47.158 DEBG IO Read  1486 has deps [JobId(1485)]
Sep 22 23:18:47.160 INFO [lossy] skipping 1485
Sep 22 23:18:47.160 DEBG Flush     :1485 extent_limit None deps:[JobId(1484), JobId(1483), JobId(1482)] res:true f:206 g:1
Sep 22 23:18:47.162 INFO [lossy] sleeping 1 second
Sep 22 23:18:47.658 DEBG IO Flush 1487 has deps [JobId(1486), JobId(1485)]
Sep 22 23:18:48.170 DEBG Read      :1486 deps:[JobId(1485)] res:true
Sep 22 23:18:48.192 DEBG Flush     :1487 extent_limit None deps:[JobId(1486), JobId(1485)] res:true f:207 g:1
Sep 22 23:18:48.568 DEBG [0] Read AckReady 1486, : downstairs
Sep 22 23:18:48.569 DEBG up_ds_listen was notified
Sep 22 23:18:48.569 DEBG up_ds_listen process 1486
Sep 22 23:18:48.569 DEBG [A] ack job 1486:487, : downstairs
Sep 22 23:18:48.622 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:18:48.623 DEBG IO Read  1488 has deps [JobId(1487)]
Sep 22 23:18:48.627 INFO [lossy] skipping 1488
Sep 22 23:18:48.627 WARN returning error on read!
Sep 22 23:18:48.627 DEBG Read      :1488 deps:[JobId(1487)] res:false
Sep 22 23:18:48.633 DEBG Read      :1488 deps:[JobId(1487)] res:true
Sep 22 23:18:48.655 ERRO [0] job id 1488 saw error GenericError("test error")
Sep 22 23:18:49.032 DEBG [0] Read AckReady 1488, : downstairs
Sep 22 23:18:49.033 DEBG up_ds_listen was notified
Sep 22 23:18:49.033 DEBG up_ds_listen process 1488
Sep 22 23:18:49.033 DEBG [A] ack job 1488:489, : downstairs
Sep 22 23:18:49.085 DEBG up_ds_listen checked 1 jobs, back to waiting
test test::integration_test_problematic_downstairs ... ok

test result: ok. 57 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 261.76s

     Running `/work/oxidecomputer/crucible/target/debug/deps/crucible_nbd_server-8c1612631a1669fd --nocapture`

running 0 tests

test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s

     Running `/work/oxidecomputer/crucible/target/debug/deps/crucible_package-e8ff0170d25e0da5 --nocapture`

running 0 tests

test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s

     Running `/work/oxidecomputer/crucible/target/debug/deps/crucible_pantry-8e1bf492bfe90e8c --nocapture`

running 0 tests

test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s

     Running `/work/oxidecomputer/crucible/target/debug/deps/crucible_pantry-b51bc30f7a0cbfa5 --nocapture`

running 1 test
test tests::test_crucible_pantry_openapi ... ok

test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.02s

     Running `/work/oxidecomputer/crucible/target/debug/deps/crucible_pantry_client-8a27f01eb086219e --nocapture`

running 0 tests

test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s

     Running `/work/oxidecomputer/crucible/target/debug/deps/crucible_protocol-c776b78ce4b42bf6 --nocapture`

running 9 tests
test tests::latest_message_version ... ok
test tests::correctly_detect_truncated_message ... ok
test tests::rt_evp ... ok
test tests::rt_ev_0 ... ok
test tests::rt_ev_7 ... ok
test tests::rt_here_i_am ... ok
test tests::rt_imok ... ok
test tests::rt_ruok ... ok
test tests::rt_yes_its_me ... ok

test result: ok. 9 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s

     Running `/work/oxidecomputer/crucible/target/debug/deps/crucible_protocol-d81e6562be2ffe77 --nocapture`

running 0 tests

test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s

     Running `/work/oxidecomputer/crucible/target/debug/deps/crucible_smf-6114df38a9482a0c --nocapture`

running 10 tests
SCF_LIMIT_MAX_NAME_LENGTH = 119
SCF_LIMIT_MAX_VALUE_LENGTH = 4095
SCF_LIMIT_MAX_PG_TYPE_LENGTH = 119
SCF_LIMIT_MAX_FMRI_LENGTH = 628
test scf_sys::tests::limits ... ok
name = Ok("localhost")
test scf_sys::tests::handle ... ok
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
milestone/multi-user
test tests::scope_local ... ok
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
test tests::scope_iter ... ok
test tests::basic ... ok
test tests::scope_not_set ... ok
    default
test tests::iter ... ok
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
milestone/multi-user
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
milestone/multi-user
milestone/name-services
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
milestone/name-services
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
milestone/single-user
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
network/datalink-management
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
system/install-discovery
milestone/single-user
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        previous
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
system/device/local
        running
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    default
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
network/physical
        start
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
network/initial
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
network/datalink-management
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
network/ip-interface-management
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
milestone/name-services
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
network/loopback
    default
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
    default
network/iptun
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
network/netcfg
        initial
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
system/install-discovery
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
        last-import
network/rpc/bind
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        previous
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
system/boot-archive
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
system/device/local
milestone/devices
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        start
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
system/filesystem/local
    default
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
system/filesystem/minimal
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
milestone/single-user
system/filesystem/root
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
network/physical
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
system/filesystem/usr
    default
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
system/identity
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
    default
        initial
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
system/manifest-import
    nwam
        last-import
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        previous
system/svc/global
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
        running
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
system/svc/restarter
network/initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        start
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
milestone/multi-user-server
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
    default
network/inetd-upgrade
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
system/console-login
network/datalink-management
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
network/ip-interface-management
system/utmp
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
application/management/net-snmp
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
smf/manifest
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
application/pkg/repositories-setup
        previous
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
network/loopback
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
application/pkg/dynamic-mirror
        running
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
        start
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
application/pkg/mirror
    default
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
application/pkg/server
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
application/security/tcsd
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
network/iptun
system/install-discovery
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
milestone/sysconfig
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
    default
milestone/network
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
system/device/local
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
network/ntp
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
network/netcfg
    default
network/ipmp
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
        initial
network/routing/rdisc
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
    default
        last-import
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
network/routing/route
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        previous
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
network/routing/ndp
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
network/rpc/bind
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
network/routing/legacy-routing
        start
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
network/routing/ripng
    default
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
network/inetd
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
network/physical
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
network/bridge
system/boot-archive
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
network/ipv4-forwarding
    default
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
    default
network/ipv6-forwarding
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
network/ipqos
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
        previous
milestone/devices
network/ipsec/ipsecalgs
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
network/ipsec/policy
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
    default
        start
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
network/ipsec/manual-key
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
network/ipsec/ike
    nwam
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
system/filesystem/local
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
network/install
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
    default
network/shares/group
        last-import
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
system/filesystem/reparse
        previous
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
        running
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
network/rpc/smserver
system/filesystem/minimal
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
network/rpc/keyserv
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
    default
network/rpc/gss
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
network/initial
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
network/ipfilter
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
    default
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
system/filesystem/root
network/ldap/client
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
        initial
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
network/smb/client
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
network/smb/server
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        previous
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
network/routing-setup
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
        running
system/filesystem/usr
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
network/npiv_config
        start
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
    default
system/device/fc-fabric
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
network/ssh
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
network/ip-interface-management
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
network/varpd
system/identity
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
    default
network/security/kadmin
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    domain
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
network/security/krb5kdc
        initial
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    node
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
        last-import
network/security/ktkt_warn
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        previous
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
network/device-discovery/printers
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
system/manifest-import
        running
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
network/service
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        start
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
    default
network/nis/client
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
network/location
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
network/dns/client
network/loopback
system/svc/global
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
system/name-service-cache
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
    default
network/nfs/mapid
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
network/chrony
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
system/svc/restarter
network/ibd-post-upgrade
        previous
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
        running
network/tftp/udp6
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
    default
        start
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
network/netmask
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
network/dns/multicast
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
milestone/multi-user-server
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
network/dns/install
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
network/iptun
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
network/nfs/log
    default
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
network/nfs/rquota
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
network/nfs/client
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
network/inetd-upgrade
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
network/nfs/server
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        previous
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
    default
network/nfs/cbd
        running
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
        start
network/nfs/status
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
system/console-login
network/nfs/nlockmgr
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
platform/i86pc/acpihpd
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
    default
network/netcfg
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
system/sac
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    vt2
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
system/fcoe_initiator
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    vt3
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
        initial
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
system/fmd
    vt4
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
system/sysevent
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
    vt5
        previous
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
system/boot-config
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
        running
    vt6
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
system/picl
        start
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
system/coreadm
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
system/utmp
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
system/hal
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
network/rpc/bind
    default
system/resource-mgmt
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
    default
system/rcap
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
system/system-log
        initial
application/management/net-snmp
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
        last-import
system/dumpadm
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
    default
        previous
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
system/dbus
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
system/pools
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        start
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
smf/manifest
system/power
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
system/keymap
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
system/auditset
system/boot-archive
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
application/pkg/repositories-setup
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
system/stmf
    default
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
    default
system/hotplug
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
system/rbac
        last-import
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
        previous
system/logadm-upgrade
application/pkg/dynamic-mirror
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
        running
system/hostid
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
    default
        start
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
system/filesystem/autofs
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
system/cron
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
application/pkg/mirror
system/illumos/userscript
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
milestone/devices
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
system/vtdaemon
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    default
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
system/boot-archive-update
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
system/cryptosvc
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
application/pkg/server
        last-import
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
system/intrd
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        previous
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
    default
system/rmtmpfiles
        running
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
        start
system/t6init
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
system/auditd
application/security/tcsd
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
system/idmap
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
system/filesystem/local
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
    default
system/zones-monitoring
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
system/zones
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
        initial
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
system/ipcc
milestone/sysconfig
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
system/update-man-index
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
        previous
    default
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
system/process-security
        running
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        start
system/fm/notify-params
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
milestone/network
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
system/pkgserv
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
system/extended-accounting
    default
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
system/filesystem/minimal
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
system/consadm
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
    default
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
system/scheduler
network/ntp
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
system/pfexec
        initial
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
    default
        last-import
system/illumos/metadata
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        previous
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
system/sar
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
network/ipmp
system/early-manifest-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        start
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
system/fcoe_target
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    default
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
system/device/mpxio-upgrade
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
system/filesystem/root
system/device/audio
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
network/routing/rdisc
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
system/device/allocate
    default
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
    default
smf/legacy_run
        initial
Ok(Service { scf: Scf { handle: 0x6bc550 }, service: 0x6bd790 })
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
site/buildomat/agent
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        previous
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
network/routing/route
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        start
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    default
test tests::service_iter ... ok
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
network/routing/ndp
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
system/filesystem/usr
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    default
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
        initial
network/routing/legacy-routing
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
    ipv4
        previous
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
    ipv6
        running
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        start
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
network/routing/ripng
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    default
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
system/identity
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
network/inetd
    domain
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
    default
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
network/bridge
        previous
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        start
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
network/ipv4-forwarding
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    node
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
network/ipv6-forwarding
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
        previous
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        start
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
network/ipqos
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    default
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
system/manifest-import
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
network/ipsec/ipsecalgs
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
network/ipsec/policy
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        previous
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
    default
        running
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        start
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
network/ipsec/manual-key
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    default
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
system/svc/global
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
network/ipsec/ike
    default
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
    default
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
        previous
network/install
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    default
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
network/shares/group
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
system/svc/restarter
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    default
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    zfs
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
        last-import
system/filesystem/reparse
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        previous
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
network/rpc/smserver
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
    default
milestone/multi-user-server
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
network/rpc/keyserv
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        previous
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
network/rpc/gss
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
        start
    default
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
network/ipfilter
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
network/inetd-upgrade
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    default
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
network/ldap/client
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        previous
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
network/smb/client
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        start
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    default
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
network/smb/server
system/console-login
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
network/routing-setup
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
        previous
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        start
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
network/npiv_config
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    default
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    vt2
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
        initial
system/device/fc-fabric
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
    default
        previous
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
network/ssh
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    vt3
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
network/varpd
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        previous
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
network/security/kadmin
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    vt4
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
network/security/krb5kdc
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        previous
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
    default
        running
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
network/security/ktkt_warn
    vt5
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
    default
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
        previous
network/device-discovery/printers
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    snmp
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    vt6
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
network/service
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
    default
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        previous
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
network/nis/client
        running
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    default
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
system/utmp
network/location
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    default
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
network/dns/client
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        previous
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        start
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
system/name-service-cache
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    default
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
application/management/net-snmp
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
network/nfs/mapid
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
network/chrony
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    default
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
network/ibd-post-upgrade
smf/manifest
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    default
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
application/pkg/repositories-setup
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
network/tftp/udp6
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
network/netmask
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    default
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
network/dns/multicast
application/pkg/dynamic-mirror
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
network/dns/install
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
        running
    default
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
network/nfs/log
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
application/pkg/mirror
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    default
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
network/nfs/rquota
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
        last-import
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
network/nfs/client
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    default
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
application/pkg/server
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
    default
network/nfs/server
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
        initial
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
network/nfs/cbd
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    default
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
application/security/tcsd
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
network/nfs/status
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
network/nfs/nlockmgr
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    default
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
platform/i86pc/acpihpd
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
milestone/sysconfig
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    default
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
system/sac
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
    default
        running
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        start
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
system/fcoe_initiator
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    default
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
milestone/network
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
system/fmd
    default
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
    default
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
system/sysevent
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        start
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    default
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
system/boot-config
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
network/ntp
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    default
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
        initial
system/picl
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        start
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
system/coreadm
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    default
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
network/ipmp
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
system/hal
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
system/resource-mgmt
        running
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
        start
    default
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
system/rcap
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
network/routing/rdisc
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
system/system-log
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    rsyslog
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
system/dumpadm
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    default
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
network/routing/route
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
    default
system/dbus
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
        initial
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
system/pools
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    default
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
network/routing/ndp
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
system/power
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
system/keymap
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
        start
    default
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
system/auditset
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
network/routing/legacy-routing
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    default
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    ipv4
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
system/stmf
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
        last-import
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
system/hotplug
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    ipv6
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
system/rbac
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    default
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
system/logadm-upgrade
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
network/routing/ripng
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    default
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
system/hostid
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
    default
        running
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
system/filesystem/autofs
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
network/inetd
    default
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
system/cron
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
    default
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
        start
system/illumos/userscript
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    default
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
network/bridge
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
system/vtdaemon
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    default
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
network/ipv4-forwarding
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
    default
system/boot-archive-update
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
        initial
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
system/cryptosvc
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    default
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
network/ipv6-forwarding
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
system/intrd
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
        last-import
system/rmtmpfiles
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    default
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
system/t6init
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
network/ipqos
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    default
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
system/auditd
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
    default
        running
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
system/idmap
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
network/ipsec/ipsecalgs
    default
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
system/zones-monitoring
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
    default
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
        start
system/zones
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    default
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
network/ipsec/policy
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
system/ipcc
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
system/update-man-index
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
    default
        start
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
system/process-security
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
network/ipsec/manual-key
    default
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
system/fm/notify-params
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
    default
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
system/pkgserv
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    default
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
network/ipsec/ike
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
    default
system/extended-accounting
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
        initial
    flow
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
        last-import
    net
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
        running
    process
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    task
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
system/consadm
network/install
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    default
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
system/scheduler
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
        running
    default
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
system/pfexec
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
network/shares/group
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    default
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
system/illumos/metadata
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
        last-import
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        start
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
system/sar
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    zfs
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
system/early-manifest-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        start
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    default
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
system/fcoe_target
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
system/filesystem/reparse
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    default
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
system/device/mpxio-upgrade
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
    default
        running
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
system/device/audio
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
network/rpc/smserver
    default
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
system/device/allocate
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
    default
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
smf/legacy_run
        start
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
network/rpc/keyserv
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
network/rpc/gss
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
Ok(Service { scf: Scf { handle: 0x6b1ad0 }, service: 0x6bddd0 })
site/buildomat/agent
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
    Ok(Instance { scf: Scf { handle: 0x6b1ad0 }, instance: 0x6bd8d0 })
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
    default
        start
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
network/ipfilter
test tests::instance_iter ... ok
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
network/ldap/client
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
network/smb/client
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
network/smb/server
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
network/routing-setup
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        start
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
network/npiv_config
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        start
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
system/device/fc-fabric
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        previous
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        start
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
network/ssh
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        start
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
network/varpd
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
network/security/kadmin
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
network/security/krb5kdc
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
network/security/ktkt_warn
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        start
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
network/device-discovery/printers
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    snmp
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
network/service
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        start
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
network/nis/client
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
network/location
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
network/dns/client
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        previous
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        start
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
system/name-service-cache
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        previous
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        start
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
network/nfs/mapid
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        previous
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
network/chrony
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
network/ibd-post-upgrade
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        start
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
network/tftp/udp6
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
network/netmask
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        start
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
network/dns/multicast
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
network/dns/install
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
network/nfs/log
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
network/nfs/rquota
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
network/nfs/client
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
network/nfs/server
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
network/nfs/cbd
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
network/nfs/status
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
network/nfs/nlockmgr
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
platform/i86pc/acpihpd
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        start
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
system/sac
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        start
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
system/fcoe_initiator
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        start
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
system/fmd
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        start
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
system/sysevent
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        start
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
system/boot-config
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        start
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
system/picl
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        start
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
system/coreadm
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        start
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
system/hal
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        start
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
system/resource-mgmt
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        start
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
system/rcap
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
system/system-log
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        previous
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        start
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    rsyslog
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        previous
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
system/dumpadm
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        start
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
system/dbus
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        start
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
system/pools
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
system/power
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        start
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
system/keymap
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        start
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
system/auditset
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        start
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
system/stmf
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
system/hotplug
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
system/rbac
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        start
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
system/logadm-upgrade
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        start
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
system/hostid
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        start
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
system/filesystem/autofs
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        start
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
system/cron
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        start
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
system/illumos/userscript
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        start
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
system/vtdaemon
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
system/boot-archive-update
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        start
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
system/cryptosvc
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        start
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
system/intrd
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        start
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
system/rmtmpfiles
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        start
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
system/t6init
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
system/auditd
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
system/idmap
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
system/zones-monitoring
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        start
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
system/zones
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        previous
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        start
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
system/ipcc
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        start
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
system/update-man-index
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        start
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
system/process-security
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
system/fm/notify-params
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
system/pkgserv
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        start
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
system/extended-accounting
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    flow
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    net
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    process
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    task
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
system/consadm
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
system/scheduler
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        start
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
system/pfexec
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        start
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
system/illumos/metadata
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        start
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
system/sar
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
system/early-manifest-import
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        start
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
system/fcoe_target
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
system/device/mpxio-upgrade
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
system/device/audio
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        start
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
system/device/allocate
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
smf/legacy_run
Ok(Service { scf: Scf { handle: 0x6bc010 }, service: 0x6bd750 })
site/buildomat/agent
    Ok(Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 })
    default
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        last-import
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        running
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        initial
    Instance { scf: Scf { handle: 0x6bc010 }, instance: 0x6bd6d0 }
        start
test tests::snapshot_iter ... ok

test result: ok. 10 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.11s

     Running `/work/oxidecomputer/crucible/target/debug/deps/crudd-3e9b00990c25260e --nocapture`

running 0 tests

test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s

     Running `/work/oxidecomputer/crucible/target/debug/deps/crutest-af78e92d646e2d06 --nocapture`

running 64 tests
test protocol::tests::correctly_detect_truncated_message ... ok
test protocol::tests::rt_commit ... ok
test protocol::tests::rt_activate ... ok
test protocol::tests::rt_deactivate ... ok
test protocol::tests::rt_done_ok ... ok
test protocol::tests::rt_generic ... ok
test protocol::tests::rt_info ... ok
test protocol::tests::rt_info_please ... ok
test protocol::tests::rt_is_active ... ok
test protocol::tests::rt_is_show ... ok
test protocol::tests::rt_perf ... ok
test protocol::tests::rt_read ... ok
test protocol::tests::rt_my_uuid ... ok
test test::test_95_10 ... ok
test protocol::tests::rt_write ... ok
test protocol::tests::rt_write_unwritten ... ok
test protocol::tests::rt_replace ... ok
test protocol::tests::rt_uuid ... ok
test test::test_95_2 ... ok
test test::test_95_21 ... ok
test test::test_95_small ... ok
test test::test_perc_bad_big_perc ... ok
test test::test_perc_bad_perc ... ok
test test::test_95_20 ... ok
test test::test_perc_mixed ... ok
test test::test_read_compare_empty_data ... ok
test test::test_read_compare_1 ... ok
test test::test_read_compare_commit ... ok
test test::test_read_compare_empty ... Mismatch     Block::0 bo:1 Volume offset:1  Expected:3 Got:2
Mismatch     Block::0 bo:2 Volume offset:2  Expected:3 Got:2
Mismatch     Block::0 bo:1 Volume offset:1  Expected:1 Got:2
Mismatch     Block::0 bo:2 Volume offset:2  Expected:1 Got:2
Mismatch     Block::0 bo:3 Volume offset:3  Expected:1 Got:2
Mismatch     Block::0 bo:4 Volume offset:4  Expected:1 Got:2
Mismatch     Block::0 bo:5 Volume offset:5  Expected:1 Got:2
Mismatch     Block::0 bo:6 Volume offset:6  Expected:1 Got:2
Mismatch     Block::0 bo:7 Volume offset:7  Expected:1 Got:2
Mismatch     Block::0 bo:8 Volume offset:8  Expected:1 Got:2
Mismatch     Block::0 bo:9 Volume offset:9  Expected:1 Got:2
Mismatch     Block::0 bo:3 Volume offset:3  Expected:3 Got:2
Mismatch     Block::0 bo:10 Volume offset:10  Expected:1 Got:2
Mismatch     Block::0 bo:4 Volume offset:4  Expected:3 Got:2
Mismatch     Block::0 bo:11 Volume offset:11  Expected:1 Got:2
Mismatch     Block::0 bo:5 Volume offset:5  Expected:3 Got:2
Mismatch     Block::0 bo:12 Volume offset:12  Expected:1 Got:2
Mismatch     Block::0 bo:6 Volume offset:6  Expected:3 Got:2
Mismatch     Block::0 bo:13 Volume offset:13  Expected:1 Got:2
Mismatch     Block::0 bo:7 Volume offset:7  Expected:3 Got:2
Mismatch     Block::0 bo:14 Volume offset:14  Expected:1 Got:2
Mismatch     Block::0 bo:8 Volume offset:8  Expected:3 Got:2
Mismatch     Block::0 bo:15 Volume offset:15  Expected:1 Got:2
Mismatch     Block::0 bo:9 Volume offset:9  Expected:3 Got:2
Mismatch     Block::0 bo:16 Volume offset:16  Expected:1 Got:2
Mismatch     Block::0 bo:10 Volume offset:10  Expected:3 Got:2
Mismatch     Block::0 bo:17 Volume offset:17  Expected:1 Got:2
Mismatch     Block::0 bo:11 Volume offset:11  Expected:3 Got:2
Mismatch     Block::0 bo:18 Volume offset:18  Expected:1 Got:2
Mismatch     Block::0 bo:12 Volume offset:12  Expected:3 Got:2
Mismatch     Block::0 bo:19 Volume offset:19  Expected:1 Got:2
Mismatch     Block::0 bo:13 Volume offset:13  Expected:3 Got:2
Mismatch     Block::0 bo:20 Volume offset:20  Expected:1 Got:2
Mismatch     Block::0 bo:14 Volume offset:14  Expected:3 Got:2
Mismatch     Block::0 bo:21 Volume offset:21  Expected:1 Got:2
Mismatch     Block::0 bo:15 Volume offset:15  Expected:3 Got:2
Mismatch     Block::0 bo:22 Volume offset:22  Expected:1 Got:2
Mismatch     Block::0 bo:16 Volume offset:16  Expected:3 Got:2
Mismatch     Block::0 bo:23 Volume offset:23  Expected:1 Got:2
Mismatch     Block::0 bo:17 Volume offset:17  Expected:3 Got:2
Mismatch     Block::2 bo:1 Volume offset:1025  Expected:2 Got:9
Mismatch     Block::0 bo:18 Volume offset:18  Expected:3 Got:2
Mismatch     Block::0 bo:24 Volume offset:24  Expected:1 Got:2
Mismatch     Block::0 bo:19 Volume offset:19  Expected:3 Got:2
Mismatch     Block::0 bo:25 Volume offset:25  Expected:1 Got:2
Mismatch     Block::0 bo:20 Volume offset:20  Expected:3 Got:2
Mismatch     Block::0 bo:26 Volume offset:26  Expected:1 Got:2
Mismatch     Block::0 bo:21 Volume offset:21  Expected:3 Got:2
Mismatch     Block::0 bo:27 Volume offset:27  Expected:1 Got:2
Mismatch     Block::0 bo:22 Volume offset:22  Expected:3 Got:2
Mismatch     Block::0 bo:28 Volume offset:28  Expected:1 Got:2
Mismatch     Block::0 bo:23 Volume offset:23  Expected:3 Got:2
Mismatch     Block::0 bo:29 Volume offset:29  Expected:1 Got:2
Mismatch     Block::0 bo:24 Volume offset:24  Expected:3 Got:2
Mismatch     Block::0 bo:30 Volume offset:30  Expected:1 Got:2
Mismatch     Block::0 bo:25 Volume offset:25  Expected:3 Got:2
Mismatch     Block::0 bo:31 Volume offset:31  Expected:1 Got:2
Mismatch     Block::0 bo:26 Volume offset:26  Expected:3 Got:2
Mismatch     Block::0 bo:32 Volume offset:32  Expected:1 Got:2
Mismatch     Block::0 bo:27 Volume offset:27  Expected:3 Got:2
Mismatch     Block::0 bo:33 Volume offset:33  Expected:1 Got:2
Mismatch     Block::0 bo:28 Volume offset:28  Expected:3 Got:2
Mismatch     Block::0 bo:34 Volume offset:34  Expected:1 Got:2
Mismatch     Block::0 bo:29 Volume offset:29  Expected:3 Got:2
Mismatch     Block::0 bo:35 Volume offset:35  Expected:1 Got:2
Mismatch     Block::0 bo:30 Volume offset:30  Expected:3 Got:2
Mismatch     Block::0 bo:36 Volume offset:36  Expected:1 Got:2
Mismatch     Block::0 bo:31 Volume offset:31  Expected:3 Got:2
Mismatch     Block::0 bo:37 Volume offset:37  Expected:1 Got:2
Mismatch     Block::0 bo:32 Volume offset:32  Expected:3 Got:2
Mismatch     Block::0 bo:38 Volume offset:38  Expected:1 Got:2
Mismatch     Block::0 bo:33 Volume offset:33  Expected:3 Got:2
Mismatch     Block::0 bo:39 Volume offset:39  Expected:1 Got:2
Mismatch     Block::0 bo:34 Volume offset:34  Expected:3 Got:2
Mismatch     Block::0 bo:40 Volume offset:40  Expected:1 Got:2
Mismatch     Block::0 bo:35 Volume offset:35  Expected:3 Got:2
Mismatch     Block::0 bo:41 Volume offset:41  Expected:1 Got:2
Mismatch     Block::0 bo:36 Volume offset:36  Expected:3 Got:2
Mismatch     Block::0 bo:42 Volume offset:42  Expected:1 Got:2
Mismatch     Block::0 bo:37 Volume offset:37  Expected:3 Got:2
Mismatch     Block::0 bo:43 Volume offset:43  Expected:1 Got:2
Mismatch     Block::0 bo:38 Volume offset:38  Expected:3 Got:2
Mismatch     Block::0 bo:44 Volume offset:44  Expected:1 Got:2
Mismatch     Block::0 bo:39 Volume offset:39  Expected:3 Got:2
Mismatch     Block::0 bo:45 Volume offset:45  Expected:1 Got:2
Mismatch     Block::0 bo:40 Volume offset:40  Expected:3 Got:2
Mismatch     Block::0 bo:46 Volume offset:46  Expected:1 Got:2
Mismatch     Block::0 bo:41 Volume offset:41  Expected:3 Got:2
Mismatch     Block::0 bo:47 Volume offset:47  Expected:1 Got:2
Mismatch     Block::0 bo:42 Volume offset:42  Expected:3 Got:2
Mismatch     Block::0 bo:48 Volume offset:48  Expected:1 Got:2
Mismatch     Block::0 bo:43 Volume offset:43  Expected:3 Got:2
Mismatch     Block::0 bo:49 Volume offset:49  Expected:1 Got:2
Mismatch     Block::0 bo:44 Volume offset:44  Expected:3 Got:2
Mismatch     Block::0 bo:50 Volume offset:50  Expected:1 Got:2
Mismatch     Block::0 bo:45 Volume offset:45  Expected:3 Got:2
Mismatch     Block::0 bo:51 Volume offset:51  Expected:1 Got:2
Mismatch     Block::0 bo:46 Volume offset:46  Expected:3 Got:2
Mismatch     Block::0 bo:52 Volume offset:52  Expected:1 Got:2
Mismatch     Block::0 bo:47 Volume offset:47  Expected:3 Got:2
Mismatch     Block::0 bo:53 Volume offset:53  Expected:1 Got:2
Mismatch     Block::0 bo:48 Volume offset:48  Expected:3 Got:2
Mismatch     Block::0 bo:54 Volume offset:54  Expected:1 Got:2
Mismatch     Block::0 bo:49 Volume offset:49  Expected:3 Got:2
Mismatch     Block::0 bo:55 Volume offset:55  Expected:1 Got:2
Mismatch     Block::0 bo:50 Volume offset:50  Expected:3 Got:2
Mismatch     Block::0 bo:56 Volume offset:56  Expected:1 Got:2
Mismatch     Block::0 bo:51 Volume offset:51  Expected:3 Got:2
Mismatch     Block::0 bo:57 Volume offset:57  Expected:1 Got:2
Mismatch     Block::0 bo:52 Volume offset:52  Expected:3 Got:2
Mismatch     Block::0 bo:58 Volume offset:58  Expected:1 Got:2
Mismatch     Block::0 bo:53 Volume offset:53  Expected:3 Got:2
Mismatch     Block::0 bo:59 Volume offset:59  Expected:1 Got:2
Mismatch     Block::0 bo:54 Volume offset:54  Expected:3 Got:2
Mismatch     Block::0 bo:60 Volume offset:60  Expected:1 Got:2
Mismatch     Block::0 bo:55 Volume offset:55  Expected:3 Got:2
Mismatch     Block::0 bo:61 Volume offset:61  Expected:1 Got:2
Mismatch     Block::0 bo:56 Volume offset:56  Expected:3 Got:2
Mismatch     Block::0 bo:62 Volume offset:62  Expected:1 Got:2
Mismatch     Block::0 bo:57 Volume offset:57  Expected:3 Got:2
Mismatch     Block::0 bo:63 Volume offset:63  Expected:1 Got:2
Mismatch     Block::0 bo:58 Volume offset:58  Expected:3 Got:2
Mismatch     Block::0 bo:64 Volume offset:64  Expected:1 Got:2
Mismatch     Block::0 bo:59 Volume offset:59  Expected:3 Got:2
Mismatch     Block::0 bo:65 Volume offset:65  Expected:1 Got:2
Mismatch     Block::0 bo:60 Volume offset:60  Expected:3 Got:2
Mismatch     Block::0 bo:66 Volume offset:66  Expected:1 Got:2
Mismatch     Block::0 bo:61 Volume offset:61  Expected:3 Got:2
Mismatch     Block::0 bo:67 Volume offset:67  Expected:1 Got:2
Mismatch     Block::0 bo:62 Volume offset:62  Expected:3 Got:2
Mismatch     Block::0 bo:68 Volume offset:68  Expected:1 Got:2
Mismatch     Block::0 bo:63 Volume offset:63  Expected:3 Got:2
Mismatch     Block::0 bo:69 Volume offset:69  Expected:1 Got:2
Mismatch     Block::0 bo:64 Volume offset:64  Expected:3 Got:2
Mismatch     Block::0 bo:70 Volume offset:70  Expected:1 Got:2
Mismatch     Block::0 bo:65 Volume offset:65  Expected:3 Got:2
Mismatch     Block::0 bo:71 Volume offset:71  Expected:1 Got:2
Mismatch     Block::0 bo:66 Volume offset:66  Expected:3 Got:2
Mismatch     Block::0 bo:72 Volume offset:72  Expected:1 Got:2
Mismatch     Block::0 bo:67 Volume offset:67  Expected:3 Got:2
Mismatch     Block::0 bo:73 Volume offset:73  Expected:1 Got:2
Mismatch     Block::0 bo:68 Volume offset:68  Expected:3 Got:2
Mismatch     Block::0 bo:74 Volume offset:74  Expected:1 Got:2
Mismatch     Block::0 bo:69 Volume offset:69  Expected:3 Got:2
Mismatch     Block::0 bo:75 Volume offset:75  Expected:1 Got:2
Mismatch     Block::0 bo:70 Volume offset:70  Expected:3 Got:2
Mismatch     Block::0 bo:76 Volume offset:76  Expected:1 Got:2
Mismatch     Block::0 bo:71 Volume offset:71  Expected:3 Got:2
Mismatch     Block::0 bo:77 Volume offset:77  Expected:1 Got:2
Mismatch     Block::0 bo:72 Volume offset:72  Expected:3 Got:2
Mismatch     Block::0 bo:78 Volume offset:78  Expected:1 Got:2
Mismatch     Block::0 bo:73 Volume offset:73  Expected:3 Got:2
Mismatch     Block::0 bo:79 Volume offset:79  Expected:1 Got:2
Mismatch     Block::0 bo:74 Volume offset:74  Expected:3 Got:2
Mismatch     Block::0 bo:80 Volume offset:80  Expected:1 Got:2
Mismatch     Block::0 bo:75 Volume offset:75  Expected:3 Got:2
Mismatch     Block::0 bo:81 Volume offset:81  Expected:1 Got:2
Mismatch     Block::0 bo:76 Volume offset:76  Expected:3 Got:2
Mismatch     Block::0 bo:82 Volume offset:82  Expected:1 Got:2
Mismatch     Block::0 bo:77 Volume offset:77  Expected:3 Got:2
Mismatch     Block::0 bo:83 Volume offset:83  Expected:1 Got:2
Mismatch     Block::0 bo:78 Volume offset:78  Expected:3 Got:2
Mismatch     Block::0 bo:84 Volume offset:84  Expected:1 Got:2
Mismatch     Block::0 bo:79 Volume offset:79  Expected:3 Got:2
Mismatch     Block::0 bo:85 Volume offset:85  Expected:1 Got:2
Mismatch     Block::0 bo:80 Volume offset:80  Expected:3 Got:2
Mismatch     Block::0 bo:86 Volume offset:86  Expected:1 Got:2
Mismatch     Block::0 bo:81 Volume offset:81  Expected:3 Got:2
Mismatch     Block::0 bo:87 Volume offset:87  Expected:1 Got:2
Mismatch     Block::0 bo:82 Volume offset:82  Expected:3 Got:2
Mismatch     Block::0 bo:88 Volume offset:88  Expected:1 Got:2
Mismatch     Block::0 bo:83 Volume offset:83  Expected:3 Got:2
Mismatch     Block::0 bo:89 Volume offset:89  Expected:1 Got:2
Mismatch     Block::0 bo:84 Volume offset:84  Expected:3 Got:2
Mismatch     Block::0 bo:90 Volume offset:90  Expected:1 Got:2
Mismatch     Block::0 bo:85 Volume offset:85  Expected:3 Got:2
Mismatch     Block::0 bo:91 Volume offset:91  Expected:1 Got:2
Mismatch     Block::0 bo:86 Volume offset:86  Expected:3 Got:2
Mismatch     Block::0 bo:92 Volume offset:92  Expected:1 Got:2
Mismatch     Block::0 bo:87 Volume offset:87  Expected:3 Got:2
Mismatch     Block::0 bo:93 Volume offset:93  Expected:1 Got:2
Mismatch     Block::0 bo:88 Volume offset:88  Expected:3 Got:2
Mismatch     Block::0 bo:94 Volume offset:94  Expected:1 Got:2
Mismatch     Block::0 bo:89 Volume offset:89  Expected:3 Got:2
Mismatch     Block::0 bo:95 Volume offset:95  Expected:1 Got:2
Mismatch     Block::0 bo:90 Volume offset:90  Expected:3 Got:2
Mismatch     Block::0 bo:96 Volume offset:96  Expected:1 Got:2
Mismatch     Block::0 bo:91 Volume offset:91  Expected:3 Got:2
Mismatch     Block::0 bo:97 Volume offset:97  Expected:1 Got:2
Mismatch     Block::0 bo:92 Volume offset:92  Expected:3 Got:2
Mismatch     Block::0 bo:98 Volume offset:98  Expected:1 Got:2
Mismatch     Block::0 bo:93 Volume offset:93  Expected:3 Got:2
Mismatch     Block::0 bo:99 Volume offset:99  Expected:1 Got:2
Mismatch     Block::0 bo:94 Volume offset:94  Expected:3 Got:2
Mismatch     Block::0 bo:100 Volume offset:100  Expected:1 Got:2
Mismatch     Block::0 bo:95 Volume offset:95  Expected:3 Got:2
Mismatch     Block::0 bo:101 Volume offset:101  Expected:1 Got:2
Mismatch     Block::0 bo:96 Volume offset:96  Expected:3 Got:2
Mismatch     Block::0 bo:102 Volume offset:102  Expected:1 Got:2
Mismatch     Block::0 bo:97 Volume offset:97  Expected:3 Got:2
Mismatch     Block::0 bo:103 Volume offset:103  Expected:1 Got:2
Mismatch     Block::0 bo:98 Volume offset:98  Expected:3 Got:2
Mismatch     Block::0 bo:104 Volume offset:104  Expected:1 Got:2
Mismatch     Block::0 bo:99 Volume offset:99  Expected:3 Got:2
Mismatch     Block::0 bo:105 Volume offset:105  Expected:1 Got:2
Mismatch     Block::0 bo:100 Volume offset:100  Expected:3 Got:2
Mismatch     Block::0 bo:106 Volume offset:106  Expected:1 Got:2
Mismatch     Block::0 bo:101 Volume offset:101  Expected:3 Got:2
Mismatch     Block::0 bo:107 Volume offset:107  Expected:1 Got:2
Mismatch     Block::0 bo:102 Volume offset:102  Expected:3 Got:2
Mismatch     Block::0 bo:108 Volume offset:108  Expected:1 Got:2
Mismatch     Block::0 bo:103 Volume offset:103  Expected:3 Got:2
Mismatch     Block::0 bo:109 Volume offset:109  Expected:1 Got:2
Mismatch     Block::0 bo:104 Volume offset:104  Expected:3 Got:2
Mismatch     Block::0 bo:110 Volume offset:110  Expected:1 Got:2
Mismatch     Block::0 bo:105 Volume offset:105  Expected:3 Got:2
Mismatch     Block::0 bo:111 Volume offset:111  Expected:1 Got:2
Mismatch     Block::0 bo:106 Volume offset:106  Expected:3 Got:2
Mismatch     Block::0 bo:112 Volume offset:112  Expected:1 Got:2
Mismatch     Block::0 bo:107 Volume offset:107  Expected:3 Got:2
Mismatch     Block::0 bo:113 Volume offset:113  Expected:1 Got:2
Mismatch     Block::0 bo:108 Volume offset:108  Expected:3 Got:2
Mismatch     Block::0 bo:114 Volume offset:114  Expected:1 Got:2
Mismatch     Block::0 bo:109 Volume offset:109  Expected:3 Got:2
Mismatch     Block::0 bo:115 Volume offset:115  Expected:1 Got:2
Mismatch     Block::0 bo:110 Volume offset:110  Expected:3 Got:2
Mismatch     Block::0 bo:116 Volume offset:116  Expected:1 Got:2
Mismatch     Block::0 bo:111 Volume offset:111  Expected:3 Got:2
Mismatch     Block::0 bo:117 Volume offset:117  Expected:1 Got:2
Mismatch     Block::0 bo:112 Volume offset:112  Expected:3 Got:2
Mismatch     Block::0 bo:118 Volume offset:118  Expected:1 Got:2
Mismatch     Block::0 bo:113 Volume offset:113  Expected:3 Got:2
Mismatch     Block::0 bo:119 Volume offset:119  Expected:1 Got:2
Mismatch     Block::0 bo:114 Volume offset:114  Expected:3 Got:2
Mismatch     Block::0 bo:120 Volume offset:120  Expected:1 Got:2
Mismatch     Block::0 bo:115 Volume offset:115  Expected:3 Got:2
Mismatch     Block::0 bo:121 Volume offset:121  Expected:1 Got:2
Mismatch     Block::0 bo:116 Volume offset:116  Expected:3 Got:2
Mismatch     Block::0 bo:122 Volume offset:122  Expected:1 Got:2
Mismatch     Block::0 bo:117 Volume offset:117  Expected:3 Got:2
Mismatch     Block::0 bo:123 Volume offset:123  Expected:1 Got:2
Mismatch     Block::0 bo:118 Volume offset:118  Expected:3 Got:2
Mismatch     Block::0 bo:124 Volume offset:124  Expected:1 Got:2
Mismatch     Block::0 bo:119 Volume offset:119  Expected:3 Got:2
Mismatch     Block::0 bo:125 Volume offset:125  Expected:1 Got:2
Mismatch     Block::0 bo:120 Volume offset:120  Expected:3 Got:2
Mismatch     Block::0 bo:126 Volume offset:126  Expected:1 Got:2
Mismatch     Block::0 bo:121 Volume offset:121  Expected:3 Got:2
Mismatch     Block::0 bo:127 Volume offset:127  Expected:1 Got:2
Mismatch     Block::0 bo:122 Volume offset:122  Expected:3 Got:2
Mismatch     Block::0 bo:128 Volume offset:128  Expected:1 Got:2
Mismatch     Block::0 bo:123 Volume offset:123  Expected:3 Got:2
Mismatch     Block::0 bo:129 Volume offset:129  Expected:1 Got:2
Mismatch     Block::0 bo:124 Volume offset:124  Expected:3 Got:2
Mismatch     Block::0 bo:130 Volume offset:130  Expected:1 Got:2
Mismatch     Block::0 bo:125 Volume offset:125  Expected:3 Got:2
Mismatch     Block::0 bo:131 Volume offset:131  Expected:1 Got:2
Mismatch     Block::0 bo:126 Volume offset:126  Expected:3 Got:2
Mismatch     Block::0 bo:132 Volume offset:132  Expected:1 Got:2
Mismatch     Block::0 bo:127 Volume offset:127  Expected:3 Got:2
Mismatch     Block::0 bo:133 Volume offset:133  Expected:1 Got:2
Mismatch     Block::0 bo:128 Volume offset:128  Expected:3 Got:2
Mismatch     Block::0 bo:134 Volume offset:134  Expected:1 Got:2
Mismatch     Block::0 bo:129 Volume offset:129  Expected:3 Got:2
Mismatch     Block::0 bo:135 Volume offset:135  Expected:1 Got:2
Mismatch     Block::0 bo:130 Volume offset:130  Expected:3 Got:2
Mismatch     Block::0 bo:136 Volume offset:136  Expected:1 Got:2
Mismatch     Block::0 bo:131 Volume offset:131  Expected:3 Got:2
Mismatch     Block::0 bo:137 Volume offset:137  Expected:1 Got:2
Mismatch     Block::0 bo:132 Volume offset:132  Expected:3 Got:2
Mismatch     Block::0 bo:138 Volume offset:138  Expected:1 Got:2
Mismatch     Block::0 bo:133 Volume offset:133  Expected:3 Got:2
Mismatch     Block::0 bo:139 Volume offset:139  Expected:1 Got:2
Mismatch     Block::0 bo:134 Volume offset:134  Expected:3 Got:2
Mismatch     Block::0 bo:140 Volume offset:140  Expected:1 Got:2
Mismatch     Block::0 bo:135 Volume offset:135  Expected:3 Got:2
Mismatch     Block::0 bo:141 Volume offset:141  Expected:1 Got:2
Mismatch     Block::0 bo:136 Volume offset:136  Expected:3 Got:2
Mismatch     Block::0 bo:142 Volume offset:142  Expected:1 Got:2
Mismatch     Block::0 bo:137 Volume offset:137  Expected:3 Got:2
Mismatch     Block::0 bo:143 Volume offset:143  Expected:1 Got:2
Mismatch     Block::0 bo:138 Volume offset:138  Expected:3 Got:2
Mismatch     Block::0 bo:144 Volume offset:144  Expected:1 Got:2
Mismatch     Block::0 bo:139 Volume offset:139  Expected:3 Got:2
Mismatch     Block::0 bo:145 Volume offset:145  Expected:1 Got:2
Mismatch     Block::0 bo:140 Volume offset:140  Expected:3 Got:2
Mismatch     Block::0 bo:146 Volume offset:146  Expected:1 Got:2
Mismatch     Block::0 bo:141 Volume offset:141  Expected:3 Got:2
Mismatch     Block::0 bo:147 Volume offset:147  Expected:1 Got:2
Mismatch     Block::0 bo:142 Volume offset:142  Expected:3 Got:2
Mismatch     Block::0 bo:148 Volume offset:148  Expected:1 Got:2
Mismatch     Block::0 bo:143 Volume offset:143  Expected:3 Got:2
Mismatch     Block::0 bo:149 Volume offset:149  Expected:1 Got:2
Mismatch     Block::0 bo:144 Volume offset:144  Expected:3 Got:2
Mismatch     Block::0 bo:150 Volume offset:150  Expected:1 Got:2
Mismatch     Block::0 bo:145 Volume offset:145  Expected:3 Got:2
Mismatch     Block::0 bo:151 Volume offset:151  Expected:1 Got:2
Mismatch     Block::0 bo:146 Volume offset:146  Expected:3 Got:2
Mismatch     Block::0 bo:152 Volume offset:152  Expected:1 Got:2
Mismatch     Block::0 bo:147 Volume offset:147  Expected:3 Got:2
Mismatch     Block::0 bo:153 Volume offset:153  Expected:1 Got:2
Mismatch     Block::0 bo:148 Volume offset:148  Expected:3 Got:2
Mismatch     Block::0 bo:154 Volume offset:154  Expected:1 Got:2
Mismatch     Block::0 bo:149 Volume offset:149  Expected:3 Got:2
Mismatch     Block::0 bo:155 Volume offset:155  Expected:1 Got:2
Mismatch     Block::0 bo:150 Volume offset:150  Expected:3 Got:2
Mismatch     Block::0 bo:156 Volume offset:156  Expected:1 Got:2
Mismatch     Block::0 bo:151 Volume offset:151  Expected:3 Got:2
Mismatch     Block::0 bo:157 Volume offset:157  Expected:1 Got:2
Mismatch     Block::0 bo:152 Volume offset:152  Expected:3 Got:2
Mismatch     Block::0 bo:158 Volume offset:158  Expected:1 Got:2
Mismatch     Block::0 bo:153 Volume offset:153  Expected:3 Got:2
Mismatch     Block::0 bo:159 Volume offset:159  Expected:1 Got:2
Mismatch     Block::0 bo:154 Volume offset:154  Expected:3 Got:2
Mismatch     Block::0 bo:160 Volume offset:160  Expected:1 Got:2
Mismatch     Block::0 bo:155 Volume offset:155  Expected:3 Got:2
Mismatch     Block::0 bo:161 Volume offset:161  Expected:1 Got:2
Mismatch     Block::0 bo:156 Volume offset:156  Expected:3 Got:2
Mismatch     Block::0 bo:162 Volume offset:162  Expected:1 Got:2
Mismatch     Block::0 bo:157 Volume offset:157  Expected:3 Got:2
Mismatch     Block::0 bo:163 Volume offset:163  Expected:1 Got:2
Mismatch     Block::0 bo:158 Volume offset:158  Expected:3 Got:2
Mismatch     Block::0 bo:164 Volume offset:164  Expected:1 Got:2
Mismatch     Block::0 bo:159 Volume offset:159  Expected:3 Got:2
Mismatch     Block::0 bo:165 Volume offset:165  Expected:1 Got:2
Mismatch     Block::0 bo:160 Volume offset:160  Expected:3 Got:2
Mismatch     Block::0 bo:166 Volume offset:166  Expected:1 Got:2
Mismatch     Block::0 bo:161 Volume offset:161  Expected:3 Got:2
Mismatch     Block::0 bo:167 Volume offset:167  Expected:1 Got:2
Mismatch     Block::0 bo:162 Volume offset:162  Expected:3 Got:2
Mismatch     Block::0 bo:168 Volume offset:168  Expected:1 Got:2
Mismatch     Block::0 bo:163 Volume offset:163  Expected:3 Got:2
Mismatch     Block::0 bo:169 Volume offset:169  Expected:1 Got:2
Mismatch     Block::0 bo:164 Volume offset:164  Expected:3 Got:2
Mismatch     Block::0 bo:170 Volume offset:170  Expected:1 Got:2
Mismatch     Block::0 bo:165 Volume offset:165  Expected:3 Got:2
Mismatch     Block::0 bo:171 Volume offset:171  Expected:1 Got:2
Mismatch     Block::0 bo:166 Volume offset:166  Expected:3 Got:2
Mismatch     Block::0 bo:172 Volume offset:172  Expected:1 Got:2
Mismatch     Block::0 bo:167 Volume offset:167  Expected:3 Got:2
Mismatch     Block::0 bo:173 Volume offset:173  Expected:1 Got:2
Mismatch     Block::0 bo:168 Volume offset:168  Expected:3 Got:2
Mismatch     Block::0 bo:174 Volume offset:174  Expected:1 Got:2
Mismatch     Block::0 bo:169 Volume offset:169  Expected:3 Got:2
Mismatch     Block::0 bo:175 Volume offset:175  Expected:1 Got:2
Mismatch     Block::0 bo:170 Volume offset:170  Expected:3 Got:2
Mismatch     Block::0 bo:176 Volume offset:176  Expected:1 Got:2
Mismatch     Block::0 bo:171 Volume offset:171  Expected:3 Got:2
Mismatch     Block::0 bo:177 Volume offset:177  Expected:1 Got:2
Mismatch     Block::0 bo:172 Volume offset:172  Expected:3 Got:2
Mismatch     Block::0 bo:178 Volume offset:178  Expected:1 Got:2
Mismatch     Block::0 bo:173 Volume offset:173  Expected:3 Got:2
Mismatch     Block::0 bo:179 Volume offset:179  Expected:1 Got:2
Mismatch     Block::0 bo:174 Volume offset:174  Expected:3 Got:2
Mismatch     Block::0 bo:180 Volume offset:180  Expected:1 Got:2
Mismatch     Block::0 bo:175 Volume offset:175  Expected:3 Got:2
Mismatch     Block::0 bo:181 Volume offset:181  Expected:1 Got:2
Mismatch     Block::0 bo:176 Volume offset:176  Expected:3 Got:2
Mismatch     Block::0 bo:182 Volume offset:182  Expected:1 Got:2
Mismatch     Block::0 bo:177 Volume offset:177  Expected:3 Got:2
Mismatch     Block::0 bo:183 Volume offset:183  Expected:1 Got:2
Mismatch     Block::0 bo:178 Volume offset:178  Expected:3 Got:2
Mismatch     Block::0 bo:184 Volume offset:184  Expected:1 Got:2
Mismatch     Block::0 bo:179 Volume offset:179  Expected:3 Got:2
Mismatch     Block::0 bo:185 Volume offset:185  Expected:1 Got:2
Mismatch     Block::0 bo:180 Volume offset:180  Expected:3 Got:2
Mismatch     Block::0 bo:186 Volume offset:186  Expected:1 Got:2
Mismatch     Block::0 bo:181 Volume offset:181  Expected:3 Got:2
Mismatch     Block::0 bo:187 Volume offset:187  Expected:1 Got:2
Mismatch     Block::0 bo:182 Volume offset:182  Expected:3 Got:2
Mismatch     Block::0 bo:188 Volume offset:188  Expected:1 Got:2
Mismatch     Block::0 bo:183 Volume offset:183  Expected:3 Got:2
Mismatch     Block::0 bo:189 Volume offset:189  Expected:1 Got:2
Mismatch     Block::0 bo:184 Volume offset:184  Expected:3 Got:2
Mismatch     Block::0 bo:190 Volume offset:190  Expected:1 Got:2
Mismatch     Block::0 bo:185 Volume offset:185  Expected:3 Got:2
Mismatch     Block::0 bo:191 Volume offset:191  Expected:1 Got:2
Mismatch     Block::0 bo:186 Volume offset:186  Expected:3 Got:2
Mismatch     Block::0 bo:192 Volume offset:192  Expected:1 Got:2
Mismatch     Block::0 bo:187 Volume offset:187  Expected:3 Got:2
Mismatch     Block::0 bo:193 Volume offset:193  Expected:1 Got:2
Mismatch     Block::0 bo:188 Volume offset:188  Expected:3 Got:2
Mismatch     Block::0 bo:194 Volume offset:194  Expected:1 Got:2
Mismatch     Block::0 bo:189 Volume offset:189  Expected:3 Got:2
Mismatch     Block::0 bo:195 Volume offset:195  Expected:1 Got:2
Mismatch     Block::0 bo:190 Volume offset:190  Expected:3 Got:2
Mismatch     Block::0 bo:196 Volume offset:196  Expected:1 Got:2
Mismatch     Block::0 bo:191 Volume offset:191  Expected:3 Got:2
Mismatch     Block::0 bo:197 Volume offset:197  Expected:1 Got:2
Mismatch     Block::0 bo:192 Volume offset:192  Expected:3 Got:2
Mismatch     Block::0 bo:198 Volume offset:198  Expected:1 Got:2
Mismatch     Block::0 bo:193 Volume offset:193  Expected:3 Got:2
Mismatch     Block::0 bo:199 Volume offset:199  Expected:1 Got:2
Mismatch     Block::0 bo:194 Volume offset:194  Expected:3 Got:2
Mismatch     Block::0 bo:200 Volume offset:200  Expected:1 Got:2
Mismatch     Block::0 bo:195 Volume offset:195  Expected:3 Got:2
Mismatch     Block::0 bo:201 Volume offset:201  Expected:1 Got:2
Mismatch     Block::0 bo:196 Volume offset:196  Expected:3 Got:2
Mismatch     Block::0 bo:202 Volume offset:202  Expected:1 Got:2
Mismatch     Block::0 bo:197 Volume offset:197  Expected:3 Got:2
Mismatch     Block::0 bo:203 Volume offset:203  Expected:1 Got:2
Mismatch     Block::0 bo:198 Volume offset:198  Expected:3 Got:2
Mismatch     Block::0 bo:204 Volume offset:204  Expected:1 Got:2
Mismatch     Block::0 bo:199 Volume offset:199  Expected:3 Got:2
Mismatch     Block::0 bo:205 Volume offset:205  Expected:1 Got:2
Mismatch     Block::0 bo:200 Volume offset:200  Expected:3 Got:2
Mismatch     Block::0 bo:206 Volume offset:206  Expected:1 Got:2
Mismatch     Block::0 bo:201 Volume offset:201  Expected:3 Got:2
Mismatch     Block::0 bo:207 Volume offset:207  Expected:1 Got:2
Mismatch     Block::0 bo:202 Volume offset:202  Expected:3 Got:2
Mismatch     Block::0 bo:208 Volume offset:208  Expected:1 Got:2
Mismatch     Block::0 bo:203 Volume offset:203  Expected:3 Got:2
Mismatch     Block::0 bo:209 Volume offset:209  Expected:1 Got:2
Mismatch     Block::0 bo:204 Volume offset:204  Expected:3 Got:2
Mismatch     Block::0 bo:210 Volume offset:210  Expected:1 Got:2
Mismatch     Block::0 bo:205 Volume offset:205  Expected:3 Got:2
Mismatch     Block::0 bo:211 Volume offset:211  Expected:1 Got:2
Mismatch     Block::0 bo:206 Volume offset:206  Expected:3 Got:2
Mismatch     Block::0 bo:212 Volume offset:212  Expected:1 Got:2
Mismatch     Block::0 bo:207 Volume offset:207  Expected:3 Got:2
Mismatch     Block::0 bo:213 Volume offset:213  Expected:1 Got:2
Mismatch     Block::0 bo:208 Volume offset:208  Expected:3 Got:2
Mismatch     Block::0 bo:214 Volume offset:214  Expected:1 Got:2
Mismatch     Block::0 bo:209 Volume offset:209  Expected:3 Got:2
Mismatch     Block::0 bo:215 Volume offset:215  Expected:1 Got:2
Mismatch     Block::0 bo:210 Volume offset:210  Expected:3 Got:2
Mismatch     Block::0 bo:216 Volume offset:216  Expected:1 Got:2
Mismatch     Block::0 bo:211 Volume offset:211  Expected:3 Got:2
Mismatch     Block::0 bo:217 Volume offset:217  Expected:1 Got:2
Mismatch     Block::0 bo:212 Volume offset:212  Expected:3 Got:2
Mismatch     Block::0 bo:218 Volume offset:218  Expected:1 Got:2
Mismatch     Block::0 bo:213 Volume offset:213  Expected:3 Got:2
Mismatch     Block::0 bo:219 Volume offset:219  Expected:1 Got:2
Mismatch     Block::0 bo:214 Volume offset:214  Expected:3 Got:2
Mismatch     Block::0 bo:220 Volume offset:220  Expected:1 Got:2
Mismatch     Block::0 bo:215 Volume offset:215  Expected:3 Got:2
Mismatch     Block::0 bo:221 Volume offset:221  Expected:1 Got:2
Mismatch     Block::0 bo:216 Volume offset:216  Expected:3 Got:2
Mismatch     Block::0 bo:222 Volume offset:222  Expected:1 Got:2
Mismatch     Block::0 bo:217 Volume offset:217  Expected:3 Got:2
Mismatch     Block::0 bo:223 Volume offset:223  Expected:1 Got:2
Mismatch     Block::0 bo:218 Volume offset:218  Expected:3 Got:2
Mismatch     Block::0 bo:224 Volume offset:224  Expected:1 Got:2
Mismatch     Block::0 bo:219 Volume offset:219  Expected:3 Got:2
Mismatch     Block::0 bo:225 Volume offset:225  Expected:1 Got:2
Mismatch     Block::0 bo:220 Volume offset:220  Expected:3 Got:2
Mismatch     Block::0 bo:226 Volume offset:226  Expected:1 Got:2
Mismatch     Block::0 bo:221 Volume offset:221  Expected:3 Got:2
Mismatch     Block::0 bo:227 Volume offset:227  Expected:1 Got:2
Mismatch     Block::0 bo:222 Volume offset:222  Expected:3 Got:2
Mismatch     Block::0 bo:228 Volume offset:228  Expected:1 Got:2
Mismatch     Block::0 bo:223 Volume offset:223  Expected:3 Got:2
Mismatch     Block::0 bo:229 Volume offset:229  Expected:1 Got:2
Mismatch     Block::0 bo:224 Volume offset:224  Expected:3 Got:2
Mismatch     Block::0 bo:230 Volume offset:230  Expected:1 Got:2
Mismatch     Block::0 bo:225 Volume offset:225  Expected:3 Got:2
Mismatch     Block::0 bo:231 Volume offset:231  Expected:1 Got:2
Mismatch     Block::0 bo:226 Volume offset:226  Expected:3 Got:2
Mismatch     Block::0 bo:232 Volume offset:232  Expected:1 Got:2
Mismatch     Block::0 bo:227 Volume offset:227  Expected:3 Got:2
Mismatch     Block::0 bo:233 Volume offset:233  Expected:1 Got:2
Mismatch     Block::0 bo:228 Volume offset:228  Expected:3 Got:2
Mismatch     Block::0 bo:234 Volume offset:234  Expected:1 Got:2
Mismatch     Block::0 bo:229 Volume offset:229  Expected:3 Got:2
Mismatch     Block::0 bo:235 Volume offset:235  Expected:1 Got:2
Mismatch     Block::0 bo:230 Volume offset:230  Expected:3 Got:2
Mismatch     Block::0 bo:236 Volume offset:236  Expected:1 Got:2
Mismatch     Block::0 bo:231 Volume offset:231  Expected:3 Got:2
Mismatch     Block::0 bo:237 Volume offset:237  Expected:1 Got:2
Mismatch     Block::0 bo:232 Volume offset:232  Expected:3 Got:2
Mismatch     Block::0 bo:238 Volume offset:238  Expected:1 Got:2
Mismatch     Block::0 bo:233 Volume offset:233  Expected:3 Got:2
Mismatch     Block::0 bo:239 Volume offset:239  Expected:1 Got:2
Mismatch     Block::0 bo:234 Volume offset:234  Expected:3 Got:2
Mismatch     Block::0 bo:240 Volume offset:240  Expected:1 Got:2
Mismatch     Block::0 bo:235 Volume offset:235  Expected:3 Got:2
Mismatch     Block::0 bo:241 Volume offset:241  Expected:1 Got:2
Mismatch     Block::0 bo:236 Volume offset:236  Expected:3 Got:2
Mismatch     Block::0 bo:242 Volume offset:242  Expected:1 Got:2
Mismatch     Block::0 bo:237 Volume offset:237  Expected:3 Got:2
Mismatch     Block::0 bo:243 Volume offset:243  Expected:1 Got:2
Mismatch     Block::0 bo:238 Volume offset:238  Expected:3 Got:2
Mismatch     Block::0 bo:244 Volume offset:244  Expected:1 Got:2
Mismatch     Block::0 bo:239 Volume offset:239  Expected:3 Got:2
Mismatch     Block::0 bo:245 Volume offset:245  Expected:1 Got:2
Mismatch     Block::0 bo:240 Volume offset:240  Expected:3 Got:2
Mismatch     Block::0 bo:246 Volume offset:246  Expected:1 Got:2
Mismatch     Block::0 bo:241 Volume offset:241  Expected:3 Got:2
Mismatch     Block::0 bo:247 Volume offset:247  Expected:1 Got:2
Mismatch     Block::0 bo:242 Volume offset:242  Expected:3 Got:2
Mismatch     Block::0 bo:248 Volume offset:248  Expected:1 Got:2
Mismatch     Block::0 bo:243 Volume offset:243  Expected:3 Got:2
Mismatch     Block::0 bo:249 Volume offset:249  Expected:1 Got:2
Mismatch     Block::0 bo:244 Volume offset:244  Expected:3 Got:2
Mismatch     Block::0 bo:250 Volume offset:250  Expected:1 Got:2
Mismatch     Block::0 bo:245 Volume offset:245  Expected:3 Got:2
Mismatch     Block::0 bo:251 Volume offset:251  Expected:1 Got:2
Mismatch     Block::0 bo:246 Volume offset:246  Expected:3 Got:2
Mismatch     Block::0 bo:252 Volume offset:252  Expected:1 Got:2
Mismatch     Block::0 bo:247 Volume offset:247  Expected:3 Got:2
Mismatch     Block::0 bo:253 Volume offset:253  Expected:1 Got:2
Mismatch     Block::0 bo:248 Volume offset:248  Expected:3 Got:2
Mismatch     Block::0 bo:254 Volume offset:254  Expected:1 Got:2
Mismatch     Block::0 bo:249 Volume offset:249  Expected:3 Got:2
Mismatch     Block::0 bo:255 Volume offset:255  Expected:1 Got:2
Mismatch     Block::0 bo:250 Volume offset:250  Expected:3 Got:2
Mismatch     Block::0 bo:256 Volume offset:256  Expected:1 Got:2
Mismatch     Block::0 bo:251 Volume offset:251  Expected:3 Got:2
Mismatch     Block::0 bo:257 Volume offset:257  Expected:1 Got:2
Mismatch     Block::0 bo:252 Volume offset:252  Expected:3 Got:2
Mismatch     Block::0 bo:258 Volume offset:258  Expected:1 Got:2
Mismatch     Block::0 bo:253 Volume offset:253  Expected:3 Got:2
Mismatch     Block::0 bo:259 Volume offset:259  Expected:1 Got:2
Mismatch     Block::0 bo:254 Volume offset:254  Expected:3 Got:2
Mismatch     Block::0 bo:260 Volume offset:260  Expected:1 Got:2
Mismatch     Block::0 bo:255 Volume offset:255  Expected:3 Got:2
Mismatch     Block::0 bo:261 Volume offset:261  Expected:1 Got:2
Mismatch     Block::0 bo:256 Volume offset:256  Expected:3 Got:2
Mismatch     Block::0 bo:262 Volume offset:262  Expected:1 Got:2
Mismatch     Block::0 bo:257 Volume offset:257  Expected:3 Got:2
Mismatch     Block::0 bo:263 Volume offset:263  Expected:1 Got:2
Mismatch     Block::0 bo:258 Volume offset:258  Expected:3 Got:2
Mismatch     Block::0 bo:264 Volume offset:264  Expected:1 Got:2
Mismatch     Block::0 bo:259 Volume offset:259  Expected:3 Got:2
Mismatch     Block::0 bo:265 Volume offset:265  Expected:1 Got:2
Mismatch     Block::0 bo:260 Volume offset:260  Expected:3 Got:2
Mismatch     Block::0 bo:266 Volume offset:266  Expected:1 Got:2
Mismatch     Block::0 bo:261 Volume offset:261  Expected:3 Got:2
Mismatch     Block::0 bo:267 Volume offset:267  Expected:1 Got:2
Mismatch     Block::0 bo:262 Volume offset:262  Expected:3 Got:2
Mismatch     Block::0 bo:268 Volume offset:268  Expected:1 Got:2
Mismatch     Block::0 bo:263 Volume offset:263  Expected:3 Got:2
Mismatch     Block::0 bo:269 Volume offset:269  Expected:1 Got:2
Mismatch     Block::0 bo:264 Volume offset:264  Expected:3 Got:2
Mismatch     Block::0 bo:270 Volume offset:270  Expected:1 Got:2
Mismatch     Block::0 bo:265 Volume offset:265  Expected:3 Got:2
Mismatch     Block::0 bo:271 Volume offset:271  Expected:1 Got:2
Mismatch     Block::0 bo:266 Volume offset:266  Expected:3 Got:2
Mismatch     Block::0 bo:272 Volume offset:272  Expected:1 Got:2
Mismatch     Block::0 bo:267 Volume offset:267  Expected:3 Got:2
Mismatch     Block::0 bo:273 Volume offset:273  Expected:1 Got:2
Mismatch     Block::0 bo:268 Volume offset:268  Expected:3 Got:2
Mismatch     Block::0 bo:274 Volume offset:274  Expected:1 Got:2
Mismatch     Block::0 bo:269 Volume offset:269  Expected:3 Got:2
Mismatch     Block::0 bo:275 Volume offset:275  Expected:1 Got:2
Mismatch     Block::0 bo:270 Volume offset:270  Expected:3 Got:2
Mismatch     Block::0 bo:276 Volume offset:276  Expected:1 Got:2
Mismatch     Block::0 bo:271 Volume offset:271  Expected:3 Got:2
Mismatch     Block::0 bo:277 Volume offset:277  Expected:1 Got:2
Mismatch     Block::0 bo:272 Volume offset:272  Expected:3 Got:2
Mismatch     Block::0 bo:278 Volume offset:278  Expected:1 Got:2
Mismatch     Block::0 bo:273 Volume offset:273  Expected:3 Got:2
Mismatch     Block::0 bo:279 Volume offset:279  Expected:1 Got:2
Mismatch     Block::0 bo:274 Volume offset:274  Expected:3 Got:2
Mismatch     Block::0 bo:280 Volume offset:280  Expected:1 Got:2
Mismatch     Block::0 bo:275 Volume offset:275  Expected:3 Got:2
Mismatch     Block::0 bo:281 Volume offset:281  Expected:1 Got:2
Mismatch     Block::0 bo:276 Volume offset:276  Expected:3 Got:2
Mismatch     Block::0 bo:282 Volume offset:282  Expected:1 Got:2
Mismatch     Block::0 bo:277 Volume offset:277  Expected:3 Got:2
Mismatch     Block::0 bo:283 Volume offset:283  Expected:1 Got:2
Mismatch     Block::0 bo:278 Volume offset:278  Expected:3 Got:2
Mismatch     Block::0 bo:284 Volume offset:284  Expected:1 Got:2
Mismatch     Block::0 bo:279 Volume offset:279  Expected:3 Got:2
Mismatch     Block::0 bo:285 Volume offset:285  Expected:1 Got:2
Mismatch     Block::0 bo:280 Volume offset:280  Expected:3 Got:2
Mismatch     Block::0 bo:286 Volume offset:286  Expected:1 Got:2
Mismatch     Block::0 bo:281 Volume offset:281  Expected:3 Got:2
Mismatch     Block::0 bo:287 Volume offset:287  Expected:1 Got:2
Mismatch     Block::0 bo:282 Volume offset:282  Expected:3 Got:2
Mismatch     Block::0 bo:288 Volume offset:288  Expected:1 Got:2
Mismatch     Block::0 bo:283 Volume offset:283  Expected:3 Got:2
Mismatch     Block::0 bo:289 Volume offset:289  Expected:1 Got:2
Mismatch     Block::0 bo:284 Volume offset:284  Expected:3 Got:2
Mismatch     Block::0 bo:290 Volume offset:290  Expected:1 Got:2
Mismatch     Block::0 bo:285 Volume offset:285  Expected:3 Got:2
Mismatch     Block::0 bo:291 Volume offset:291  Expected:1 Got:2
Mismatch     Block::0 bo:286 Volume offset:286  Expected:3 Got:2
Mismatch     Block::0 bo:292 Volume offset:292  Expected:1 Got:2
Mismatch     Block::0 bo:287 Volume offset:287  Expected:3 Got:2
Mismatch     Block::0 bo:293 Volume offset:293  Expected:1 Got:2
Mismatch     Block::0 bo:288 Volume offset:288  Expected:3 Got:2
Mismatch     Block::0 bo:294 Volume offset:294  Expected:1 Got:2
Mismatch     Block::0 bo:289 Volume offset:289  Expected:3 Got:2
Mismatch     Block::0 bo:295 Volume offset:295  Expected:1 Got:2
Mismatch     Block::0 bo:290 Volume offset:290  Expected:3 Got:2
Mismatch     Block::0 bo:296 Volume offset:296  Expected:1 Got:2
Mismatch     Block::0 bo:291 Volume offset:291  Expected:3 Got:2
Mismatch     Block::0 bo:297 Volume offset:297  Expected:1 Got:2
Mismatch     Block::0 bo:292 Volume offset:292  Expected:3 Got:2
Mismatch     Block::0 bo:298 Volume offset:298  Expected:1 Got:2
Mismatch     Block::0 bo:293 Volume offset:293  Expected:3 Got:2
Mismatch     Block::0 bo:299 Volume offset:299  Expected:1 Got:2
Mismatch     Block::0 bo:294 Volume offset:294  Expected:3 Got:2
Mismatch     Block::0 bo:300 Volume offset:300  Expected:1 Got:2
Mismatch     Block::0 bo:295 Volume offset:295  Expected:3 Got:2
Mismatch     Block::0 bo:301 Volume offset:301  Expected:1 Got:2
Mismatch     Block::0 bo:296 Volume offset:296  Expected:3 Got:2
Mismatch     Block::0 bo:302 Volume offset:302  Expected:1 Got:2
Mismatch     Block::0 bo:297 Volume offset:297  Expected:3 Got:2
Mismatch     Block::0 bo:303 Volume offset:303  Expected:1 Got:2
Mismatch     Block::0 bo:298 Volume offset:298  Expected:3 Got:2
Mismatch     Block::0 bo:304 Volume offset:304  Expected:1 Got:2
Mismatch     Block::0 bo:299 Volume offset:299  Expected:3 Got:2
Mismatch     Block::0 bo:305 Volume offset:305  Expected:1 Got:2
Mismatch     Block::0 bo:300 Volume offset:300  Expected:3 Got:2
Mismatch     Block::0 bo:306 Volume offset:306  Expected:1 Got:2
Mismatch     Block::0 bo:301 Volume offset:301  Expected:3 Got:2
Mismatch     Block::0 bo:307 Volume offset:307  Expected:1 Got:2
Mismatch     Block::0 bo:302 Volume offset:302  Expected:3 Got:2
Mismatch     Block::0 bo:308 Volume offset:308  Expected:1 Got:2
Mismatch     Block::0 bo:303 Volume offset:303  Expected:3 Got:2
Mismatch     Block::0 bo:309 Volume offset:309  Expected:1 Got:2
Mismatch     Block::0 bo:304 Volume offset:304  Expected:3 Got:2
Mismatch     Block::0 bo:310 Volume offset:310  Expected:1 Got:2
Mismatch     Block::0 bo:305 Volume offset:305  Expected:3 Got:2
Mismatch     Block::0 bo:311 Volume offset:311  Expected:1 Got:2
Mismatch     Block::0 bo:306 Volume offset:306  Expected:3 Got:2
Mismatch     Block::0 bo:312 Volume offset:312  Expected:1 Got:2
Mismatch     Block::0 bo:307 Volume offset:307  Expected:3 Got:2
Mismatch     Block::0 bo:313 Volume offset:313  Expected:1 Got:2
Mismatch     Block::0 bo:308 Volume offset:308  Expected:3 Got:2
Mismatch     Block::0 bo:314 Volume offset:314  Expected:1 Got:2
Mismatch     Block::0 bo:309 Volume offset:309  Expected:3 Got:2
Mismatch     Block::0 bo:315 Volume offset:315  Expected:1 Got:2
Mismatch     Block::0 bo:310 Volume offset:310  Expected:3 Got:2
Mismatch     Block::0 bo:316 Volume offset:316  Expected:1 Got:2
Mismatch     Block::0 bo:311 Volume offset:311  Expected:3 Got:2
Mismatch     Block::0 bo:317 Volume offset:317  Expected:1 Got:2
Mismatch     Block::0 bo:312 Volume offset:312  Expected:3 Got:2
Mismatch     Block::0 bo:318 Volume offset:318  Expected:1 Got:2
Mismatch     Block::0 bo:313 Volume offset:313  Expected:3 Got:2
Mismatch     Block::0 bo:319 Volume offset:319  Expected:1 Got:2
Mismatch     Block::0 bo:314 Volume offset:314  Expected:3 Got:2
Mismatch     Block::0 bo:320 Volume offset:320  Expected:1 Got:2
Mismatch     Block::0 bo:315 Volume offset:315  Expected:3 Got:2
Mismatch     Block::0 bo:321 Volume offset:321  Expected:1 Got:2
Mismatch     Block::0 bo:316 Volume offset:316  Expected:3 Got:2
Mismatch     Block::0 bo:322 Volume offset:322  Expected:1 Got:2
Mismatch     Block::0 bo:317 Volume offset:317  Expected:3 Got:2
Mismatch     Block::0 bo:323 Volume offset:323  Expected:1 Got:2
Mismatch     Block::0 bo:318 Volume offset:318  Expected:3 Got:2
Mismatch     Block::0 bo:324 Volume offset:324  Expected:1 Got:2
Mismatch     Block::0 bo:319 Volume offset:319  Expected:3 Got:2
Mismatch     Block::0 bo:325 Volume offset:325  Expected:1 Got:2
Mismatch     Block::0 bo:320 Volume offset:320  Expected:3 Got:2
Mismatch     Block::0 bo:326 Volume offset:326  Expected:1 Got:2
Mismatch     Block::0 bo:321 Volume offset:321  Expected:3 Got:2
Mismatch     Block::0 bo:327 Volume offset:327  Expected:1 Got:2
Mismatch     Block::0 bo:322 Volume offset:322  Expected:3 Got:2
Mismatch     Block::0 bo:328 Volume offset:328  Expected:1 Got:2
Mismatch     Block::0 bo:323 Volume offset:323  Expected:3 Got:2
Mismatch     Block::0 bo:329 Volume offset:329  Expected:1 Got:2
Mismatch     Block::0 bo:324 Volume offset:324  Expected:3 Got:2
Mismatch     Block::0 bo:330 Volume offset:330  Expected:1 Got:2
Mismatch     Block::0 bo:325 Volume offset:325  Expected:3 Got:2
Mismatch     Block::0 bo:331 Volume offset:331  Expected:1 Got:2
Mismatch     Block::0 bo:326 Volume offset:326  Expected:3 Got:2
Mismatch     Block::0 bo:332 Volume offset:332  Expected:1 Got:2
Mismatch     Block::0 bo:327 Volume offset:327  Expected:3 Got:2
Mismatch     Block::0 bo:333 Volume offset:333  Expected:1 Got:2
Mismatch     Block::0 bo:328 Volume offset:328  Expected:3 Got:2
Mismatch     Block::0 bo:334 Volume offset:334  Expected:1 Got:2
Mismatch     Block::0 bo:329 Volume offset:329  Expected:3 Got:2
Mismatch     Block::0 bo:335 Volume offset:335  Expected:1 Got:2
Mismatch     Block::0 bo:330 Volume offset:330  Expected:3 Got:2
Mismatch     Block::0 bo:336 Volume offset:336  Expected:1 Got:2
Mismatch     Block::0 bo:331 Volume offset:331  Expected:3 Got:2
Mismatch     Block::0 bo:337 Volume offset:337  Expected:1 Got:2
Mismatch     Block::0 bo:332 Volume offset:332  Expected:3 Got:2
Mismatch     Block::0 bo:338 Volume offset:338  Expected:1 Got:2
Mismatch     Block::0 bo:333 Volume offset:333  Expected:3 Got:2
Mismatch     Block::0 bo:339 Volume offset:339  Expected:1 Got:2
Mismatch     Block::0 bo:334 Volume offset:334  Expected:3 Got:2
Mismatch     Block::0 bo:340 Volume offset:340  Expected:1 Got:2
Mismatch     Block::0 bo:335 Volume offset:335  Expected:3 Got:2
Mismatch     Block::0 bo:341 Volume offset:341  Expected:1 Got:2
Mismatch     Block::0 bo:336 Volume offset:336  Expected:3 Got:2
Mismatch     Block::0 bo:342 Volume offset:342  Expected:1 Got:2
Mismatch     Block::0 bo:337 Volume offset:337  Expected:3 Got:2
Mismatch     Block::0 bo:343 Volume offset:343  Expected:1 Got:2
Mismatch     Block::0 bo:338 Volume offset:338  Expected:3 Got:2
Mismatch     Block::0 bo:344 Volume offset:344  Expected:1 Got:2
Mismatch     Block::0 bo:339 Volume offset:339  Expected:3 Got:2
Mismatch     Block::0 bo:345 Volume offset:345  Expected:1 Got:2
Mismatch     Block::0 bo:340 Volume offset:340  Expected:3 Got:2
Mismatch     Block::0 bo:346 Volume offset:346  Expected:1 Got:2
Mismatch     Block::0 bo:341 Volume offset:341  Expected:3 Got:2
Mismatch     Block::0 bo:347 Volume offset:347  Expected:1 Got:2
Mismatch     Block::0 bo:342 Volume offset:342  Expected:3 Got:2
Mismatch     Block::0 bo:348 Volume offset:348  Expected:1 Got:2
Mismatch     Block::0 bo:343 Volume offset:343  Expected:3 Got:2
Mismatch     Block::0 bo:349 Volume offset:349  Expected:1 Got:2
Mismatch     Block::0 bo:344 Volume offset:344  Expected:3 Got:2
Mismatch     Block::0 bo:350 Volume offset:350  Expected:1 Got:2
Mismatch     Block::0 bo:345 Volume offset:345  Expected:3 Got:2
Mismatch     Block::0 bo:351 Volume offset:351  Expected:1 Got:2
Mismatch     Block::0 bo:346 Volume offset:346  Expected:3 Got:2
Mismatch     Block::0 bo:352 Volume offset:352  Expected:1 Got:2
Mismatch     Block::0 bo:347 Volume offset:347  Expected:3 Got:2
Mismatch     Block::0 bo:353 Volume offset:353  Expected:1 Got:2
Mismatch     Block::0 bo:348 Volume offset:348  Expected:3 Got:2
Mismatch     Block::0 bo:354 Volume offset:354  Expected:1 Got:2
Mismatch     Block::0 bo:349 Volume offset:349  Expected:3 Got:2
Mismatch     Block::0 bo:355 Volume offset:355  Expected:1 Got:2
Mismatch     Block::0 bo:350 Volume offset:350  Expected:3 Got:2
Mismatch     Block::0 bo:356 Volume offset:356  Expected:1 Got:2
Mismatch     Block::0 bo:351 Volume offset:351  Expected:3 Got:2
Mismatch     Block::0 bo:357 Volume offset:357  Expected:1 Got:2
Mismatch     Block::0 bo:352 Volume offset:352  Expected:3 Got:2
Mismatch     Block::0 bo:358 Volume offset:358  Expected:1 Got:2
Mismatch     Block::0 bo:353 Volume offset:353  Expected:3 Got:2
Mismatch     Block::0 bo:359 Volume offset:359  Expected:1 Got:2
Mismatch     Block::0 bo:354 Volume offset:354  Expected:3 Got:2
Mismatch     Block::0 bo:360 Volume offset:360  Expected:1 Got:2
Mismatch     Block::0 bo:355 Volume offset:355  Expected:3 Got:2
Mismatch     Block::0 bo:361 Volume offset:361  Expected:1 Got:2
Mismatch     Block::0 bo:356 Volume offset:356  Expected:3 Got:2
Mismatch     Block::0 bo:362 Volume offset:362  Expected:1 Got:2
Mismatch     Block::0 bo:357 Volume offset:357  Expected:3 Got:2
Mismatch     Block::0 bo:363 Volume offset:363  Expected:1 Got:2
Mismatch     Block::0 bo:358 Volume offset:358  Expected:3 Got:2
Mismatch     Block::0 bo:364 Volume offset:364  Expected:1 Got:2
Mismatch     Block::0 bo:359 Volume offset:359  Expected:3 Got:2
Mismatch     Block::0 bo:365 Volume offset:365  Expected:1 Got:2
Mismatch     Block::0 bo:360 Volume offset:360  Expected:3 Got:2
Mismatch     Block::0 bo:366 Volume offset:366  Expected:1 Got:2
Mismatch     Block::0 bo:361 Volume offset:361  Expected:3 Got:2
Mismatch     Block::0 bo:367 Volume offset:367  Expected:1 Got:2
Mismatch     Block::0 bo:362 Volume offset:362  Expected:3 Got:2
Mismatch     Block::0 bo:368 Volume offset:368  Expected:1 Got:2
Mismatch     Block::0 bo:363 Volume offset:363  Expected:3 Got:2
Mismatch     Block::0 bo:369 Volume offset:369  Expected:1 Got:2
Mismatch     Block::0 bo:364 Volume offset:364  Expected:3 Got:2
Mismatch     Block::0 bo:370 Volume offset:370  Expected:1 Got:2
Mismatch     Block::0 bo:365 Volume offset:365  Expected:3 Got:2
Mismatch     Block::0 bo:371 Volume offset:371  Expected:1 Got:2
Mismatch     Block::0 bo:366 Volume offset:366  Expected:3 Got:2
Mismatch     Block::0 bo:372 Volume offset:372  Expected:1 Got:2
Mismatch     Block::0 bo:367 Volume offset:367  Expected:3 Got:2
Mismatch     Block::0 bo:373 Volume offset:373  Expected:1 Got:2
Mismatch     Block::0 bo:368 Volume offset:368  Expected:3 Got:2
Mismatch     Block::0 bo:374 Volume offset:374  Expected:1 Got:2
Mismatch     Block::0 bo:369 Volume offset:369  Expected:3 Got:2
Mismatch     Block::0 bo:375 Volume offset:375  Expected:1 Got:2
Mismatch     Block::0 bo:370 Volume offset:370  Expected:3 Got:2
Mismatch     Block::0 bo:376 Volume offset:376  Expected:1 Got:2
Mismatch     Block::0 bo:371 Volume offset:371  Expected:3 Got:2
Mismatch     Block::0 bo:377 Volume offset:377  Expected:1 Got:2
Mismatch     Block::0 bo:372 Volume offset:372  Expected:3 Got:2
Mismatch     Block::0 bo:378 Volume offset:378  Expected:1 Got:2
Mismatch     Block::0 bo:373 Volume offset:373  Expected:3 Got:2
Mismatch     Block::0 bo:379 Volume offset:379  Expected:1 Got:2
Mismatch     Block::0 bo:374 Volume offset:374  Expected:3 Got:2
Mismatch     Block::0 bo:380 Volume offset:380  Expected:1 Got:2
Mismatch     Block::0 bo:375 Volume offset:375  Expected:3 Got:2
Mismatch     Block::0 bo:381 Volume offset:381  Expected:1 Got:2
Mismatch     Block::0 bo:376 Volume offset:376  Expected:3 Got:2
Mismatch     Block::0 bo:382 Volume offset:382  Expected:1 Got:2
Mismatch     Block::0 bo:377 Volume offset:377  Expected:3 Got:2
Mismatch     Block::0 bo:383 Volume offset:383  Expected:1 Got:2
Mismatch     Block::0 bo:378 Volume offset:378  Expected:3 Got:2
Mismatch     Block::0 bo:384 Volume offset:384  Expected:1 Got:2
Mismatch     Block::0 bo:379 Volume offset:379  Expected:3 Got:2
Mismatch     Block::0 bo:385 Volume offset:385  Expected:1 Got:2
Mismatch     Block::0 bo:380 Volume offset:380  Expected:3 Got:2
Mismatch     Block::0 bo:386 Volume offset:386  Expected:1 Got:2
Mismatch     Block::0 bo:381 Volume offset:381  Expected:3 Got:2
Mismatch     Block::0 bo:387 Volume offset:387  Expected:1 Got:2
Mismatch     Block::0 bo:382 Volume offset:382  Expected:3 Got:2
Mismatch     Block::0 bo:388 Volume offset:388  Expected:1 Got:2
Mismatch     Block::0 bo:383 Volume offset:383  Expected:3 Got:2
Mismatch     Block::0 bo:389 Volume offset:389  Expected:1 Got:2
Mismatch     Block::0 bo:384 Volume offset:384  Expected:3 Got:2
Mismatch     Block::0 bo:390 Volume offset:390  Expected:1 Got:2
Mismatch     Block::0 bo:385 Volume offset:385  Expected:3 Got:2
Mismatch     Block::0 bo:391 Volume offset:391  Expected:1 Got:2
Mismatch     Block::0 bo:386 Volume offset:386  Expected:3 Got:2
Mismatch     Block::0 bo:392 Volume offset:392  Expected:1 Got:2
Mismatch     Block::0 bo:387 Volume offset:387  Expected:3 Got:2
Mismatch     Block::0 bo:393 Volume offset:393  Expected:1 Got:2
Mismatch     Block::0 bo:388 Volume offset:388  Expected:3 Got:2
Mismatch     Block::0 bo:394 Volume offset:394  Expected:1 Got:2
Mismatch     Block::0 bo:389 Volume offset:389  Expected:3 Got:2
Mismatch     Block::0 bo:395 Volume offset:395  Expected:1 Got:2
Mismatch     Block::0 bo:390 Volume offset:390  Expected:3 Got:2
Mismatch     Block::0 bo:396 Volume offset:396  Expected:1 Got:2
Mismatch     Block::0 bo:391 Volume offset:391  Expected:3 Got:2
Mismatch     Block::0 bo:397 Volume offset:397  Expected:1 Got:2
Mismatch     Block::0 bo:392 Volume offset:392  Expected:3 Got:2
Mismatch     Block::0 bo:398 Volume offset:398  Expected:1 Got:2
Mismatch     Block::0 bo:393 Volume offset:393  Expected:3 Got:2
Mismatch     Block::0 bo:399 Volume offset:399  Expected:1 Got:2
Mismatch     Block::0 bo:394 Volume offset:394  Expected:3 Got:2
Mismatch     Block::0 bo:400 Volume offset:400  Expected:1 Got:2
Mismatch     Block::0 bo:395 Volume offset:395  Expected:3 Got:2
Mismatch     Block::0 bo:401 Volume offset:401  Expected:1 Got:2
Mismatch     Block::0 bo:396 Volume offset:396  Expected:3 Got:2
Mismatch     Block::0 bo:402 Volume offset:402  Expected:1 Got:2
Mismatch     Block::0 bo:397 Volume offset:397  Expected:3 Got:2
Mismatch     Block::0 bo:403 Volume offset:403  Expected:1 Got:2
Mismatch     Block::0 bo:398 Volume offset:398  Expected:3 Got:2
Mismatch     Block::0 bo:404 Volume offset:404  Expected:1 Got:2
Mismatch     Block::0 bo:399 Volume offset:399  Expected:3 Got:2
Mismatch     Block::0 bo:405 Volume offset:405  Expected:1 Got:2
Mismatch     Block::0 bo:400 Volume offset:400  Expected:3 Got:2
Mismatch     Block::0 bo:406 Volume offset:406  Expected:1 Got:2
Mismatch     Block::0 bo:401 Volume offset:401  Expected:3 Got:2
Mismatch     Block::0 bo:407 Volume offset:407  Expected:1 Got:2
Mismatch     Block::0 bo:402 Volume offset:402  Expected:3 Got:2
Mismatch     Block::0 bo:408 Volume offset:408  Expected:1 Got:2
Mismatch     Block::0 bo:403 Volume offset:403  Expected:3 Got:2
Mismatch     Block::0 bo:409 Volume offset:409  Expected:1 Got:2
Mismatch     Block::0 bo:404 Volume offset:404  Expected:3 Got:2
Mismatch     Block::0 bo:410 Volume offset:410  Expected:1 Got:2
Mismatch     Block::0 bo:405 Volume offset:405  Expected:3 Got:2
Mismatch     Block::0 bo:411 Volume offset:411  Expected:1 Got:2
Mismatch     Block::0 bo:406 Volume offset:406  Expected:3 Got:2
Mismatch     Block::0 bo:412 Volume offset:412  Expected:1 Got:2
Mismatch     Block::0 bo:407 Volume offset:407  Expected:3 Got:2
Mismatch     Block::0 bo:413 Volume offset:413  Expected:1 Got:2
Mismatch     Block::0 bo:408 Volume offset:408  Expected:3 Got:2
Mismatch     Block::0 bo:414 Volume offset:414  Expected:1 Got:2
Mismatch     Block::0 bo:409 Volume offset:409  Expected:3 Got:2
Mismatch     Block::0 bo:415 Volume offset:415  Expected:1 Got:2
Mismatch     Block::0 bo:410 Volume offset:410  Expected:3 Got:2
Mismatch     Block::0 bo:416 Volume offset:416  Expected:1 Got:2
Mismatch     Block::0 bo:411 Volume offset:411  Expected:3 Got:2
Mismatch     Block::0 bo:417 Volume offset:417  Expected:1 Got:2
Mismatch     Block::0 bo:412 Volume offset:412  Expected:3 Got:2
Mismatch     Block::0 bo:418 Volume offset:418  Expected:1 Got:2
Mismatch     Block::0 bo:413 Volume offset:413  Expected:3 Got:2
Mismatch     Block::0 bo:419 Volume offset:419  Expected:1 Got:2
Mismatch     Block::0 bo:414 Volume offset:414  Expected:3 Got:2
Mismatch     Block::0 bo:420 Volume offset:420  Expected:1 Got:2
Mismatch     Block::0 bo:415 Volume offset:415  Expected:3 Got:2
Mismatch     Block::0 bo:416 Volume offset:416  Expected:3 Got:2
Mismatch     Block::0 bo:421 Volume offset:421  Expected:1 Got:2
Mismatch     Block::0 bo:417 Volume offset:417  Expected:3 Got:2
Mismatch     Block::0 bo:422 Volume offset:422  Expected:1 Got:2
Mismatch     Block::0 bo:418 Volume offset:418  Expected:3 Got:2
Mismatch     Block::0 bo:423 Volume offset:423  Expected:1 Got:2
Mismatch     Block::0 bo:419 Volume offset:419  Expected:3 Got:2
Mismatch     Block::0 bo:424 Volume offset:424  Expected:1 Got:2
Mismatch     Block::0 bo:420 Volume offset:420  Expected:3 Got:2
Mismatch     Block::0 bo:425 Volume offset:425  Expected:1 Got:2
Mismatch     Block::0 bo:421 Volume offset:421  Expected:3 Got:2
Mismatch     Block::0 bo:426 Volume offset:426  Expected:1 Got:2
Mismatch     Block::0 bo:422 Volume offset:422  Expected:3 Got:2
Mismatch     Block::0 bo:427 Volume offset:427  Expected:1 Got:2
Mismatch     Block::0 bo:423 Volume offset:423  Expected:3 Got:2
Mismatch     Block::0 bo:428 Volume offset:428  Expected:1 Got:2
Mismatch     Block::0 bo:424 Volume offset:424  Expected:3 Got:2
Mismatch     Block::0 bo:429 Volume offset:429  Expected:1 Got:2
Mismatch     Block::0 bo:425 Volume offset:425  Expected:3 Got:2
Mismatch     Block::0 bo:430 Volume offset:430  Expected:1 Got:2
Mismatch     Block::0 bo:426 Volume offset:426  Expected:3 Got:2
Mismatch     Block::0 bo:431 Volume offset:431  Expected:1 Got:2
Mismatch     Block::0 bo:427 Volume offset:427  Expected:3 Got:2
Mismatch     Block::0 bo:432 Volume offset:432  Expected:1 Got:2
Mismatch     Block::0 bo:428 Volume offset:428  Expected:3 Got:2
Mismatch     Block::0 bo:433 Volume offset:433  Expected:1 Got:2
Mismatch     Block::0 bo:429 Volume offset:429  Expected:3 Got:2
Mismatch     Block::0 bo:434 Volume offset:434  Expected:1 Got:2
Mismatch     Block::0 bo:430 Volume offset:430  Expected:3 Got:2
Mismatch     Block::0 bo:435 Volume offset:435  Expected:1 Got:2
Mismatch     Block::0 bo:431 Volume offset:431  Expected:3 Got:2
Mismatch     Block::0 bo:436 Volume offset:436  Expected:1 Got:2
Mismatch     Block::0 bo:432 Volume offset:432  Expected:3 Got:2
Mismatch     Block::0 bo:437 Volume offset:437  Expected:1 Got:2
Mismatch     Block::0 bo:433 Volume offset:433  Expected:3 Got:2
Mismatch     Block::0 bo:438 Volume offset:438  Expected:1 Got:2
Mismatch     Block::0 bo:434 Volume offset:434  Expected:3 Got:2
Mismatch     Block::0 bo:439 Volume offset:439  Expected:1 Got:2
Mismatch     Block::0 bo:435 Volume offset:435  Expected:3 Got:2
Mismatch     Block::0 bo:440 Volume offset:440  Expected:1 Got:2
Mismatch     Block::0 bo:436 Volume offset:436  Expected:3 Got:2
Mismatch     Block::0 bo:441 Volume offset:441  Expected:1 Got:2
Mismatch     Block::0 bo:437 Volume offset:437  Expected:3 Got:2
Mismatch     Block::0 bo:442 Volume offset:442  Expected:1 Got:2
Mismatch     Block::0 bo:438 Volume offset:438  Expected:3 Got:2
Mismatch     Block::0 bo:443 Volume offset:443  Expected:1 Got:2
Mismatch     Block::0 bo:439 Volume offset:439  Expected:3 Got:2
Mismatch     Block::0 bo:444 Volume offset:444  Expected:1 Got:2
Mismatch     Block::0 bo:440 Volume offset:440  Expected:3 Got:2
Mismatch     Block::0 bo:445 Volume offset:445  Expected:1 Got:2
Mismatch     Block::0 bo:441 Volume offset:441  Expected:3 Got:2
Mismatch     Block::0 bo:446 Volume offset:446  Expected:1 Got:2
Mismatch     Block::0 bo:442 Volume offset:442  Expected:3 Got:2
Mismatch     Block::0 bo:447 Volume offset:447  Expected:1 Got:2
Mismatch     Block::0 bo:443 Volume offset:443  Expected:3 Got:2
Mismatch     Block::0 bo:448 Volume offset:448  Expected:1 Got:2
Mismatch     Block::0 bo:444 Volume offset:444  Expected:3 Got:2
Mismatch     Block::0 bo:449 Volume offset:449  Expected:1 Got:2
Mismatch     Block::0 bo:445 Volume offset:445  Expected:3 Got:2
Mismatch     Block::0 bo:450 Volume offset:450  Expected:1 Got:2
Mismatch     Block::0 bo:446 Volume offset:446  Expected:3 Got:2
Mismatch     Block::0 bo:451 Volume offset:451  Expected:1 Got:2
Mismatch     Block::0 bo:447 Volume offset:447  Expected:3 Got:2
Mismatch     Block::0 bo:452 Volume offset:452  Expected:1 Got:2
Mismatch     Block::0 bo:448 Volume offset:448  Expected:3 Got:2
Mismatch     Block::0 bo:453 Volume offset:453  Expected:1 Got:2
Mismatch     Block::0 bo:449 Volume offset:449  Expected:3 Got:2
Mismatch     Block::0 bo:454 Volume offset:454  Expected:1 Got:2
Mismatch     Block::0 bo:450 Volume offset:450  Expected:3 Got:2
Mismatch     Block::0 bo:455 Volume offset:455  Expected:1 Got:2
Mismatch     Block::0 bo:451 Volume offset:451  Expected:3 Got:2
Mismatch     Block::0 bo:456 Volume offset:456  Expected:1 Got:2
Mismatch     Block::0 bo:452 Volume offset:452  Expected:3 Got:2
Mismatch     Block::0 bo:457 Volume offset:457  Expected:1 Got:2
Mismatch     Block::0 bo:453 Volume offset:453  Expected:3 Got:2
Mismatch     Block::0 bo:458 Volume offset:458  Expected:1 Got:2
Mismatch     Block::0 bo:454 Volume offset:454  Expected:3 Got:2
Mismatch     Block::0 bo:459 Volume offset:459  Expected:1 Got:2
Mismatch     Block::0 bo:455 Volume offset:455  Expected:3 Got:2
Mismatch     Block::0 bo:460 Volume offset:460  Expected:1 Got:2
Mismatch     Block::0 bo:456 Volume offset:456  Expected:3 Got:2
Mismatch     Block::0 bo:461 Volume offset:461  Expected:1 Got:2
Mismatch     Block::0 bo:457 Volume offset:457  Expected:3 Got:2
Mismatch     Block::0 bo:462 Volume offset:462  Expected:1 Got:2
Mismatch     Block::0 bo:458 Volume offset:458  Expected:3 Got:2
Mismatch     Block::0 bo:463 Volume offset:463  Expected:1 Got:2
Mismatch     Block::0 bo:459 Volume offset:459  Expected:3 Got:2
Mismatch     Block::0 bo:464 Volume offset:464  Expected:1 Got:2
Mismatch     Block::0 bo:460 Volume offset:460  Expected:3 Got:2
Mismatch     Block::0 bo:465 Volume offset:465  Expected:1 Got:2
Mismatch     Block::0 bo:461 Volume offset:461  Expected:3 Got:2
Mismatch     Block::0 bo:466 Volume offset:466  Expected:1 Got:2
Mismatch     Block::0 bo:462 Volume offset:462  Expected:3 Got:2
Mismatch     Block::0 bo:467 Volume offset:467  Expected:1 Got:2
Mismatch     Block::0 bo:463 Volume offset:463  Expected:3 Got:2
Mismatch     Block::0 bo:468 Volume offset:468  Expected:1 Got:2
Mismatch     Block::0 bo:464 Volume offset:464  Expected:3 Got:2
Mismatch     Block::0 bo:469 Volume offset:469  Expected:1 Got:2
Mismatch     Block::0 bo:465 Volume offset:465  Expected:3 Got:2
Mismatch     Block::0 bo:470 Volume offset:470  Expected:1 Got:2
Mismatch     Block::0 bo:466 Volume offset:466  Expected:3 Got:2
Mismatch     Block::0 bo:471 Volume offset:471  Expected:1 Got:2
Mismatch     Block::0 bo:467 Volume offset:467  Expected:3 Got:2
Mismatch     Block::0 bo:472 Volume offset:472  Expected:1 Got:2
Mismatch     Block::0 bo:468 Volume offset:468  Expected:3 Got:2
Mismatch     Block::0 bo:473 Volume offset:473  Expected:1 Got:2
Mismatch     Block::0 bo:469 Volume offset:469  Expected:3 Got:2
Mismatch     Block::0 bo:474 Volume offset:474  Expected:1 Got:2
Mismatch     Block::0 bo:470 Volume offset:470  Expected:3 Got:2
Mismatch     Block::0 bo:475 Volume offset:475  Expected:1 Got:2
Mismatch     Block::0 bo:471 Volume offset:471  Expected:3 Got:2
Mismatch     Block::0 bo:476 Volume offset:476  Expected:1 Got:2
Mismatch     Block::0 bo:472 Volume offset:472  Expected:3 Got:2
Mismatch     Block::0 bo:477 Volume offset:477  Expected:1 Got:2
Mismatch     Block::0 bo:473 Volume offset:473  Expected:3 Got:2
Mismatch     Block::0 bo:478 Volume offset:478  Expected:1 Got:2
Mismatch     Block::0 bo:474 Volume offset:474  Expected:3 Got:2
Mismatch     Block::0 bo:479 Volume offset:479  Expected:1 Got:2
Mismatch     Block::0 bo:475 Volume offset:475  Expected:3 Got:2
Mismatch     Block::0 bo:480 Volume offset:480  Expected:1 Got:2
Mismatch     Block::0 bo:476 Volume offset:476  Expected:3 Got:2
Mismatch     Block::0 bo:481 Volume offset:481  Expected:1 Got:2
Mismatch     Block::0 bo:477 Volume offset:477  Expected:3 Got:2
Mismatch     Block::0 bo:482 Volume offset:482  Expected:1 Got:2
Mismatch     Block::0 bo:478 Volume offset:478  Expected:3 Got:2
Mismatch     Block::0 bo:483 Volume offset:483  Expected:1 Got:2
Mismatch     Block::0 bo:479 Volume offset:479  Expected:3 Got:2
Mismatch     Block::0 bo:484 Volume offset:484  Expected:1 Got:2
Mismatch     Block::0 bo:480 Volume offset:480  Expected:3 Got:2
Mismatch     Block::0 bo:485 Volume offset:485  Expected:1 Got:2
Mismatch     Block::0 bo:481 Volume offset:481  Expected:3 Got:2
Mismatch     Block::0 bo:486 Volume offset:486  Expected:1 Got:2
Mismatch     Block::0 bo:482 Volume offset:482  Expected:3 Got:2
Mismatch     Block::0 bo:487 Volume offset:487  Expected:1 Got:2
Mismatch     Block::0 bo:483 Volume offset:483  Expected:3 Got:2
Mismatch     Block::0 bo:488 Volume offset:488  Expected:1 Got:2
Mismatch     Block::0 bo:484 Volume offset:484  Expected:3 Got:2
Mismatch     Block::0 bo:489 Volume offset:489  Expected:1 Got:2
Mismatch     Block::0 bo:485 Volume offset:485  Expected:3 Got:2
Mismatch     Block::0 bo:490 Volume offset:490  Expected:1 Got:2
Mismatch     Block::0 bo:486 Volume offset:486  Expected:3 Got:2
Mismatch     Block::0 bo:491 Volume offset:491  Expected:1 Got:2
Mismatch     Block::0 bo:487 Volume offset:487  Expected:3 Got:2
Mismatch     Block::0 bo:492 Volume offset:492  Expected:1 Got:2
Mismatch     Block::0 bo:488 Volume offset:488  Expected:3 Got:2
Mismatch     Block::0 bo:493 Volume offset:493  Expected:1 Got:2
Mismatch     Block::0 bo:489 Volume offset:489  Expected:3 Got:2
Mismatch     Block::0 bo:494 Volume offset:494  Expected:1 Got:2
Mismatch     Block::0 bo:490 Volume offset:490  Expected:3 Got:2
Mismatch     Block::0 bo:495 Volume offset:495  Expected:1 Got:2
Mismatch     Block::0 bo:491 Volume offset:491  Expected:3 Got:2
Mismatch     Block::0 bo:496 Volume offset:496  Expected:1 Got:2
Mismatch     Block::0 bo:492 Volume offset:492  Expected:3 Got:2
Mismatch     Block::0 bo:497 Volume offset:497  Expected:1 Got:2
Mismatch     Block::0 bo:493 Volume offset:493  Expected:3 Got:2
Mismatch     Block::0 bo:498 Volume offset:498  Expected:1 Got:2
Mismatch     Block::0 bo:494 Volume offset:494  Expected:3 Got:2
Mismatch     Block::0 bo:499 Volume offset:499  Expected:1 Got:2
Mismatch     Block::0 bo:495 Volume offset:495  Expected:3 Got:2
Mismatch     Block::0 bo:500 Volume offset:500  Expected:1 Got:2
Mismatch     Block::0 bo:496 Volume offset:496  Expected:3 Got:2
Mismatch     Block::0 bo:501 Volume offset:501  Expected:1 Got:2
Mismatch     Block::0 bo:497 Volume offset:497  Expected:3 Got:2
Mismatch     Block::0 bo:502 Volume offset:502  Expected:1 Got:2
Mismatch     Block::0 bo:498 Volume offset:498  Expected:3 Got:2
Mismatch     Block::0 bo:503 Volume offset:503  Expected:1 Got:2
Mismatch     Block::0 bo:499 Volume offset:499  Expected:3 Got:2
Mismatch     Block::0 bo:504 Volume offset:504  Expected:1 Got:2
Mismatch     Block::0 bo:500 Volume offset:500  Expected:3 Got:2
Mismatch     Block::0 bo:505 Volume offset:505  Expected:1 Got:2
Mismatch     Block::0 bo:501 Volume offset:501  Expected:3 Got:2
Mismatch     Block::0 bo:506 Volume offset:506  Expected:1 Got:2
Mismatch     Block::0 bo:502 Volume offset:502  Expected:3 Got:2
Mismatch     Block::0 bo:507 Volume offset:507  Expected:1 Got:2
Mismatch     Block::0 bo:503 Volume offset:503  Expected:3 Got:2
Mismatch     Block::0 bo:508 Volume offset:508  Expected:1 Got:2
Mismatch     Block::0 bo:504 Volume offset:504  Expected:3 Got:2
Mismatch     Block::0 bo:509 Volume offset:509  Expected:1 Got:2
Mismatch     Block::0 bo:505 Volume offset:505  Expected:3 Got:2
Mismatch     Block::0 bo:510 Volume offset:510  Expected:1 Got:2
Mismatch     Block::0 bo:506 Volume offset:506  Expected:3 Got:2
Mismatch     Block::0 bo:511 Volume offset:511  Expected:1 Got:2
Mismatch     Block::0 bo:507 Volume offset:507  Expected:3 Got:2
Mismatch     Block::0 bo:508 Volume offset:508  Expected:3 Got:2
Mismatch     Block::0 bo:509 Volume offset:509  Expected:3 Got:2
Mismatch     Block::0 bo:510 Volume offset:510  Expected:3 Got:2
Mismatch     Block::0 bo:511 Volume offset:511  Expected:3 Got:2
ok
test test::test_read_compare ... ok
test test::test_read_compare_span ... ok
Mismatch     Block::2 bo:2 Volume offset:1026  Expected:2 Got:9
test test::test_read_compare_span_fail ... SPEC  v:2  min_av:254 cur_av:1  cm:1022 cc:1025
SPEC  v:253  min_av:254 cur_av:1  cm:1022 cc:1025
SPEC  v:254  min_av:254 cur_av:1  cm:1022 cc:1025
new cur is 1022 from min
SPEC  v:255  min_av:254 cur_av:1  cm:1022 cc:1025
new cur is 1023 from min
SPEC  v:0  min_av:254 cur_av:1  cm:1022 cc:1025
new cur is 1024 from cur
SPEC  v:1  min_av:254 cur_av:1  cm:1022 cc:1025
new cur is 1025 from cur
SPEC  v:2  min_av:254 cur_av:1  cm:1022 cc:1025
SPEC  v:254  min_av:254 cur_av:1  cm:1022 cc:1025
new cur is 1022 from min
SPEC  v:255  min_av:254 cur_av:1  cm:1022 cc:1025
new cur is 1023 from min
SPEC  v:0  min_av:254 cur_av:1  cm:1022 cc:1025
new cur is 1024 from cur
SPEC  v:1  min_av:254 cur_av:1  cm:1022 cc:1025
new cur is 1025 from cur
ok
test test::test_read_compare_fail_under ... ok
SPEC  v:252  min_av:253 cur_av:0  cm:1021 cc:1024
SPEC  v:253  min_av:253 cur_av:0  cm:1021 cc:1024
new cur is 1021 from min
SPEC  v:254  min_av:253 cur_av:0  cm:1021 cc:1024
new cur is 1022 from min
SPEC  v:255  min_av:253 cur_av:0  cm:1021 cc:1024
new cur is 1023 from min
SPEC  v:0  min_av:253 cur_av:0  cm:1021 cc:1024
new cur is 1024 from cur
SPEC  v:1  min_av:253 cur_av:0  cm:1021 cc:1024
test test::test_read_compare_fail ... ok
test test::test_read_compare_span_fail_2 ... ok
SPEC  v:254  min_av:255 cur_av:1  cm:1023 cc:1025
SPEC  v:255  min_av:255 cur_av:1  cm:1023 cc:1025
new cur is 1023 from min
SPEC  v:0  min_av:255 cur_av:1  cm:1023 cc:1025
new cur is 1024 from cur
SPEC  v:1  min_av:255 cur_av:1  cm:1023 cc:1025
new cur is 1025 from cur
SPEC  v:2  min_av:255 cur_av:1  cm:1023 cc:1025
test test::test_wl_commit_1024_range_no_update_below_rollover ... ok
Mismatch     Block::99 bo:511 Volume offset:51199  Expected:1 Got:9
test test::test_wl_commit_1024_range_no_update_above_rollover ... ok
SPEC  v:253  min_av:254 cur_av:1  cm:1022 cc:1025
SPEC  v:254  min_av:254 cur_av:1  cm:1022 cc:1025
new cur is 1022 from min
SPEC  v:255  min_av:254 cur_av:1  cm:1022 cc:1025
new cur is 1023 from min
SPEC  v:0  min_av:254 cur_av:1  cm:1022 cc:1025
new cur is 1024 from cur
SPEC  v:1  min_av:254 cur_av:1  cm:1022 cc:1025
new cur is 1025 from cur
SPEC  v:2  min_av:254 cur_av:1  cm:1022 cc:1025
test test::test_wl_commit_1024_range_rollover_max_at ... ok
SPEC  v:253  min_av:254 cur_av:1  cm:1022 cc:1025
SPEC  v:0  min_av:254 cur_av:1  cm:1022 cc:1025
new cur is 1024 from cur
Adjusting new cur to 1024
SPEC  v:254  min_av:254 cur_av:0  cm:1022 cc:1024
new cur is 1022 from min
SPEC  v:255  min_av:254 cur_av:0  cm:1022 cc:1024
new cur is 1023 from min
test test::test_wl_commit_1024_range_rollover_min_at ... SPEC  v:1  min_av:254 cur_av:0  cm:1022 cc:1024
SPEC  v:253  min_av:254 cur_av:1  cm:1022 cc:1025
SPEC  v:2  min_av:254 cur_av:0  cm:1022 cc:1024
SPEC  v:254  min_av:254 cur_av:1  cm:1022 cc:1025
oknew cur is 1022 from min

Adjusting new cur to 1022
Shift 3, v:255 sv:1023 min:1022 cur:1022
Shift 3, v:0 sv:768 min:1022 cur:1022
Shift 3, v:1 sv:769 min:1022 cur:1022
Shift 3, v:2 sv:770 min:1022 cur:1022
SPEC  v:2  min_av:254 cur_av:1  cm:254 cc:257
SPEC  v:254  min_av:254 cur_av:1  cm:254 cc:257
new cur is 254 from min
SPEC  v:255  min_av:254 cur_av:1  cm:254 cc:257
new cur is 255 from min
SPEC  v:0  min_av:254 cur_av:1  cm:254 cc:257
new cur is 256 from cur
test test::test_read_compare_large_fail ... SPEC  v:1  min_av:254 cur_av:1  cm:254 cc:257
new cur is 257 from cur
ok
test test::test_wl_commit_1024_range_rollover_range ... ok
SPEC  v:253  min_av:254 cur_av:1  cm:254 cc:257
SPEC  v:254  min_av:254 cur_av:1  cm:254 cc:257
new cur is 254 from min
SPEC  v:255  min_av:254 cur_av:1  cm:254 cc:257
new cur is 255 from min
SPEC  v:0  min_av:254 cur_av:1  cm:254 cc:257
new cur is 256 from cur
SPEC  v:1  min_av:254 cur_av:1  cm:254 cc:257
test test::test_wl_commit_1024_range_update_rollover_above ... new cur is 257 from cur
ok
SPEC  v:2  min_av:254 cur_av:1  cm:254 cc:257
Shift 0, v:1 sv:1 min:2 cur:4
Shift 0, v:2 sv:2 min:2 cur:4
Shift 0, v:3 sv:3 min:2 cur:4
Shift 0, v:4 sv:4 min:2 cur:4
Shift 0, v:5 sv:5 min:2 cur:4
test test::test_wl_commit_1024_range_update_rollover_below ... ok
SPEC  v:254  min_av:255 cur_av:1  cm:255 cc:257
SPEC  v:255  min_av:255 cur_av:1  cm:255 cc:257
new cur is 255 from min
SPEC  v:0  min_av:255 cur_av:1  cm:255 cc:257
new cur is 256 from cur
SPEC  v:1  min_av:255 cur_av:1  cm:255 cc:257
new cur is 257 from cur
SPEC  v:2  min_av:255 cur_av:1  cm:255 cc:257
SPEC  v:253  min_av:254 cur_av:1  cm:254 cc:257
SPEC  v:254  min_av:254 cur_av:1  cm:254 cc:257
new cur is 254 from min
SPEC  v:255  min_av:254 cur_av:1  cm:254 cc:257
new cur is 255 from min
SPEC  v:0  min_av:254 cur_av:1  cm:254 cc:257
new cur is 256 from cur
SPEC  v:1  min_av:254 cur_av:1  cm:254 cc:257
new cur is 257 from cur
SPEC  v:2  min_av:254 cur_av:1  cm:254 cc:257
Shift 0, v:3 sv:3 min:2 cur:4
Update block 1 to 3 (min:2 max:4 res:true)
test test::test_wl_commit_range_no_update_above_rollover ... ok
SPEC  v:252  min_av:253 cur_av:0  cm:253 cc:256
SPEC  v:253  min_av:253 cur_av:0  cm:253 cc:256
new cur is 253 from min
SPEC  v:254  min_av:253 cur_av:0  cm:253 cc:256
new cur is 254 from min
SPEC  v:255  min_av:253 cur_av:0  cm:253 cc:256
new cur is 255 from min
SPEC  v:0  min_av:253 cur_av:0  cm:253 cc:256
new cur is 256 from cur
SPEC  v:1  min_av:253 cur_av:0  cm:253 cc:256
test test::test_read_compare_large ... ok
Shift 0, v:4 sv:4 min:2 cur:4
test test::test_wl_commit_range_no_update_below_rollover ... ok
Shift 0, v:2 sv:2 min:2 cur:4
Update block 1 to 2 (min:2 max:4 res:true)
test test::test_wl_commit_range ... ok
SPEC  v:253  min_av:254 cur_av:1  cm:254 cc:257
SPEC  v:0  min_av:254 cur_av:1  cm:254 cc:257
new cur is 256 from cur
Adjusting new cur to 256
SPEC  v:254  min_av:254 cur_av:0  cm:254 cc:256
new cur is 254 from min
SPEC  v:255  min_av:254 cur_av:0  cm:254 cc:256
new cur is 255 from min
SPEC  v:1  min_av:254 cur_av:0  cm:254 cc:256
SPEC  v:2  min_av:254 cur_av:0  cm:254 cc:256
test test::test_wl_commit_range_rollover_min_at ... ok
SPEC  v:253  min_av:254 cur_av:1  cm:254 cc:257
SPEC  v:254  min_av:254 cur_av:1  cm:254 cc:257
new cur is 254 from min
Adjusting new cur to 254
Shift 0, v:255 sv:255 min:254 cur:254
Shift 0, v:0 sv:0 min:254 cur:254
Shift 0, v:1 sv:1 min:254 cur:254
Shift 0, v:2 sv:2 min:254 cur:254
test test::test_wl_commit_range_rollover_range ... ok
test test::test_wl_commit_range_update ... ok
test test::test_wl_commit_range_rollover_max_at ... ok
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:1 Volume offset:513  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:2 Volume offset:514  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:3 Volume offset:515  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:4 Volume offset:516  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:5 Volume offset:517  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:6 Volume offset:518  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:7 Volume offset:519  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:8 Volume offset:520  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:9 Volume offset:521  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:10 Volume offset:522  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:11 Volume offset:523  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
test test::test_wl_commit_range_update_max ... Out of Range Block::1 bo:12 Volume offset:524  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:13 Volume offset:525  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:14 Volume offset:526  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:15 Volume offset:527  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:16 Volume offset:528  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:17 Volume offset:529  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:18 Volume offset:530  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:19 Volume offset:531  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:20 Volume offset:532  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:21 Volume offset:533  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:22 Volume offset:534  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:23 Volume offset:535  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:24 Volume offset:536  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:25 Volume offset:537  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:26 Volume offset:538  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:27 Volume offset:539  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:28 Volume offset:540  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:29 Volume offset:541  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:30 Volume offset:542  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:31 Volume offset:543  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:32 Volume offset:544  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:33 Volume offset:545  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:34 Volume offset:546  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:35 Volume offset:547  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:36 Volume offset:548  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:37 Volume offset:549  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:38 Volume offset:550  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:39 Volume offset:551  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:40 Volume offset:552  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:41 Volume offset:553  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:42 Volume offset:554  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:43 Volume offset:555  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:44 Volume offset:556  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:45 Volume offset:557  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:46 Volume offset:558  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:47 Volume offset:559  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:48 Volume offset:560  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:49 Volume offset:561  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:50 Volume offset:562  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:51 Volume offset:563  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:52 Volume offset:564  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:53 Volume offset:565  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:54 Volume offset:566  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:55 Volume offset:567  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:56 Volume offset:568  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:57 Volume offset:569  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:58 Volume offset:570  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:59 Volume offset:571  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:60 Volume offset:572  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:61 Volume offset:573  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:62 Volume offset:574  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:63 Volume offset:575  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:64 Volume offset:576  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:65 Volume offset:577  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:66 Volume offset:578  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:67 Volume offset:579  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:68 Volume offset:580  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:69 Volume offset:581  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:70 Volume offset:582  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:71 Volume offset:583  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:72 Volume offset:584  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:73 Volume offset:585  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:74 Volume offset:586  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:75 Volume offset:587  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:76 Volume offset:588  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:77 Volume offset:589  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:78 Volume offset:590  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:79 Volume offset:591  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:80 Volume offset:592  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:81 Volume offset:593  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:82 Volume offset:594  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:83 Volume offset:595  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:84 Volume offset:596  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:85 Volume offset:597  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:86 Volume offset:598  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:87 Volume offset:599  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:88 Volume offset:600  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:89 Volume offset:601  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:90 Volume offset:602  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:91 Volume offset:603  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:92 Volume offset:604  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:93 Volume offset:605  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:94 Volume offset:606  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:95 Volume offset:607  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:96 Volume offset:608  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:97 Volume offset:609  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:98 Volume offset:610  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:99 Volume offset:611  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:100 Volume offset:612  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:101 Volume offset:613  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:102 Volume offset:614  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:103 Volume offset:615  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:104 Volume offset:616  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:105 Volume offset:617  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:106 Volume offset:618  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:107 Volume offset:619  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:108 Volume offset:620  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:109 Volume offset:621  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:110 Volume offset:622  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:111 Volume offset:623  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:112 Volume offset:624  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:113 Volume offset:625  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:114 Volume offset:626  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:115 Volume offset:627  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:116 Volume offset:628  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:117 Volume offset:629  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:118 Volume offset:630  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:119 Volume offset:631  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:120 Volume offset:632  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:121 Volume offset:633  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:122 Volume offset:634  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:123 Volume offset:635  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:124 Volume offset:636  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:125 Volume offset:637  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:126 Volume offset:638  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:127 Volume offset:639  Expected:4 Got:1
okShift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:128 Volume offset:640  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:129 Volume offset:641  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:130 Volume offset:642  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:131 Volume offset:643  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:132 Volume offset:644  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:133 Volume offset:645  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:134 Volume offset:646  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:135 Volume offset:647  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:136 Volume offset:648  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:137 Volume offset:649  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:138 Volume offset:650  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:139 Volume offset:651  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:140 Volume offset:652  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:141 Volume offset:653  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:142 Volume offset:654  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:143 Volume offset:655  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:144 Volume offset:656  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:145 Volume offset:657  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:146 Volume offset:658  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:147 Volume offset:659  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:148 Volume offset:660  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:149 Volume offset:661  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:150 Volume offset:662  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:151 Volume offset:663  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:152 Volume offset:664  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:153 Volume offset:665  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:154 Volume offset:666  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:155 Volume offset:667  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:156 Volume offset:668  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:157 Volume offset:669  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:158 Volume offset:670  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:159 Volume offset:671  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:160 Volume offset:672  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:161 Volume offset:673  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:162 Volume offset:674  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:163 Volume offset:675  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:164 Volume offset:676  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:165 Volume offset:677  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:166 Volume offset:678  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:167 Volume offset:679  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:168 Volume offset:680  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:169 Volume offset:681  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:170 Volume offset:682  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:171 Volume offset:683  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:172 Volume offset:684  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:173 Volume offset:685  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:174 Volume offset:686  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:175 Volume offset:687  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:176 Volume offset:688  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:177 Volume offset:689  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:178 Volume offset:690  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:179 Volume offset:691  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:180 Volume offset:692  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:181 Volume offset:693  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:182 Volume offset:694  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:183 Volume offset:695  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:184 Volume offset:696  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:185 Volume offset:697  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:186 Volume offset:698  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:187 Volume offset:699  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:188 Volume offset:700  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:189 Volume offset:701  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:190 Volume offset:702  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:191 Volume offset:703  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:192 Volume offset:704  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:193 Volume offset:705  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:194 Volume offset:706  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:195 Volume offset:707  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:196 Volume offset:708  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:197 Volume offset:709  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:198 Volume offset:710  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:199 Volume offset:711  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:200 Volume offset:712  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:201 Volume offset:713  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:202 Volume offset:714  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:203 Volume offset:715  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:204 Volume offset:716  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:205 Volume offset:717  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:206 Volume offset:718  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:207 Volume offset:719  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:208 Volume offset:720  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:209 Volume offset:721  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:210 Volume offset:722  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:211 Volume offset:723  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:212 Volume offset:724  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:213 Volume offset:725  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:214 Volume offset:726  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:215 Volume offset:727  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:216 Volume offset:728  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:217 Volume offset:729  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:218 Volume offset:730  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:219 Volume offset:731  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:220 Volume offset:732  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:221 Volume offset:733  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:222 Volume offset:734  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:223 Volume offset:735  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:224 Volume offset:736  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:225 Volume offset:737  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:226 Volume offset:738  Expected:4 Got:1

Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:227 Volume offset:739  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:228 Volume offset:740  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:229 Volume offset:741  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:230 Volume offset:742  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:231 Volume offset:743  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:232 Volume offset:744  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:233 Volume offset:745  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:234 Volume offset:746  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:235 Volume offset:747  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:236 Volume offset:748  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:237 Volume offset:749  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:238 Volume offset:750  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:239 Volume offset:751  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:240 Volume offset:752  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:241 Volume offset:753  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:242 Volume offset:754  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:243 Volume offset:755  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:244 Volume offset:756  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:245 Volume offset:757  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:246 Volume offset:758  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:247 Volume offset:759  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:248 Volume offset:760  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:249 Volume offset:761  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:250 Volume offset:762  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:251 Volume offset:763  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:252 Volume offset:764  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:253 Volume offset:765  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:254 Volume offset:766  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:255 Volume offset:767  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:256 Volume offset:768  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:257 Volume offset:769  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:258 Volume offset:770  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:259 Volume offset:771  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:260 Volume offset:772  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:261 Volume offset:773  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:262 Volume offset:774  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:263 Volume offset:775  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:264 Volume offset:776  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:265 Volume offset:777  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:266 Volume offset:778  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:267 Volume offset:779  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:268 Volume offset:780  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:269 Volume offset:781  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:270 Volume offset:782  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:271 Volume offset:783  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:272 Volume offset:784  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:273 Volume offset:785  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:274 Volume offset:786  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:275 Volume offset:787  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:276 Volume offset:788  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:277 Volume offset:789  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:278 Volume offset:790  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:279 Volume offset:791  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:280 Volume offset:792  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:281 Volume offset:793  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:282 Volume offset:794  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:283 Volume offset:795  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:284 Volume offset:796  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:285 Volume offset:797  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:286 Volume offset:798  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:287 Volume offset:799  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:288 Volume offset:800  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:289 Volume offset:801  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:290 Volume offset:802  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:291 Volume offset:803  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:292 Volume offset:804  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:293 Volume offset:805  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:294 Volume offset:806  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:295 Volume offset:807  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:296 Volume offset:808  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:297 Volume offset:809  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:298 Volume offset:810  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:299 Volume offset:811  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:300 Volume offset:812  Expected:4 Got:1
test test::test_wl_commit_range_update_min ... Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:301 Volume offset:813  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:302 Volume offset:814  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:303 Volume offset:815  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:304 Volume offset:816  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:305 Volume offset:817  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:306 Volume offset:818  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:307 Volume offset:819  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:308 Volume offset:820  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:309 Volume offset:821  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:310 Volume offset:822  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:311 Volume offset:823  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:312 Volume offset:824  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:313 Volume offset:825  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:314 Volume offset:826  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:315 Volume offset:827  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:316 Volume offset:828  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:317 Volume offset:829  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:318 Volume offset:830  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:319 Volume offset:831  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:320 Volume offset:832  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:321 Volume offset:833  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:322 Volume offset:834  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:323 Volume offset:835  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
okOut of Range Block::1 bo:324 Volume offset:836  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:325 Volume offset:837  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:326 Volume offset:838  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:327 Volume offset:839  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:328 Volume offset:840  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:329 Volume offset:841  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:330 Volume offset:842  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:331 Volume offset:843  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:332 Volume offset:844  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:333 Volume offset:845  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:334 Volume offset:846  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:335 Volume offset:847  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:336 Volume offset:848  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:337 Volume offset:849  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:338 Volume offset:850  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:339 Volume offset:851  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:340 Volume offset:852  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:341 Volume offset:853  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:342 Volume offset:854  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:343 Volume offset:855  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:344 Volume offset:856  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:345 Volume offset:857  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:346 Volume offset:858  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:347 Volume offset:859  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:348 Volume offset:860  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:349 Volume offset:861  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:350 Volume offset:862  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:351 Volume offset:863  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:352 Volume offset:864  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:353 Volume offset:865  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:354 Volume offset:866  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:355 Volume offset:867  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:356 Volume offset:868  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:357 Volume offset:869  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:358 Volume offset:870  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:359 Volume offset:871  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:360 Volume offset:872  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:361 Volume offset:873  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:362 Volume offset:874  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:363 Volume offset:875  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:364 Volume offset:876  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:365 Volume offset:877  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:366 Volume offset:878  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:367 Volume offset:879  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:368 Volume offset:880  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:369 Volume offset:881  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:370 Volume offset:882  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:371 Volume offset:883  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:372 Volume offset:884  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:373 Volume offset:885  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:374 Volume offset:886  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:375 Volume offset:887  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:376 Volume offset:888  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:377 Volume offset:889  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:378 Volume offset:890  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:379 Volume offset:891  Expected:4 Got:1

Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:380 Volume offset:892  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:381 Volume offset:893  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:382 Volume offset:894  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:383 Volume offset:895  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:384 Volume offset:896  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:385 Volume offset:897  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:386 Volume offset:898  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:387 Volume offset:899  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:388 Volume offset:900  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:389 Volume offset:901  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:390 Volume offset:902  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:391 Volume offset:903  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:392 Volume offset:904  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:393 Volume offset:905  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:394 Volume offset:906  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:395 Volume offset:907  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:396 Volume offset:908  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:397 Volume offset:909  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:398 Volume offset:910  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:399 Volume offset:911  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:400 Volume offset:912  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:401 Volume offset:913  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:402 Volume offset:914  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:403 Volume offset:915  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:404 Volume offset:916  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:405 Volume offset:917  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:406 Volume offset:918  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:407 Volume offset:919  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:408 Volume offset:920  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:409 Volume offset:921  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:410 Volume offset:922  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:411 Volume offset:923  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:412 Volume offset:924  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:413 Volume offset:925  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:414 Volume offset:926  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:415 Volume offset:927  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:416 Volume offset:928  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:417 Volume offset:929  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:418 Volume offset:930  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:419 Volume offset:931  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:420 Volume offset:932  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:421 Volume offset:933  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:422 Volume offset:934  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:423 Volume offset:935  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:424 Volume offset:936  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:425 Volume offset:937  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:426 Volume offset:938  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:427 Volume offset:939  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:428 Volume offset:940  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:429 Volume offset:941  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:430 Volume offset:942  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:431 Volume offset:943  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:432 Volume offset:944  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:433 Volume offset:945  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:434 Volume offset:946  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:435 Volume offset:947  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:436 Volume offset:948  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:437 Volume offset:949  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:438 Volume offset:950  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:439 Volume offset:951  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:440 Volume offset:952  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:441 Volume offset:953  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:442 Volume offset:954  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:443 Volume offset:955  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:444 Volume offset:956  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:445 Volume offset:957  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:446 Volume offset:958  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:447 Volume offset:959  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:448 Volume offset:960  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:449 Volume offset:961  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:450 Volume offset:962  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:451 Volume offset:963  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:452 Volume offset:964  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:453 Volume offset:965  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:454 Volume offset:966  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:455 Volume offset:967  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:456 Volume offset:968  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:457 Volume offset:969  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:458 Volume offset:970  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:459 Volume offset:971  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:460 Volume offset:972  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:461 Volume offset:973  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:462 Volume offset:974  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:463 Volume offset:975  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:464 Volume offset:976  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:465 Volume offset:977  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:466 Volume offset:978  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:467 Volume offset:979  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:468 Volume offset:980  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:469 Volume offset:981  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:470 Volume offset:982  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:471 Volume offset:983  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:472 Volume offset:984  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:473 Volume offset:985  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:474 Volume offset:986  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:475 Volume offset:987  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:476 Volume offset:988  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:477 Volume offset:989  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:478 Volume offset:990  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:479 Volume offset:991  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:480 Volume offset:992  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:481 Volume offset:993  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:482 Volume offset:994  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:483 Volume offset:995  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:484 Volume offset:996  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:485 Volume offset:997  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:486 Volume offset:998  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:487 Volume offset:999  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:488 Volume offset:1000  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:489 Volume offset:1001  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:490 Volume offset:1002  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:491 Volume offset:1003  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:492 Volume offset:1004  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:493 Volume offset:1005  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:494 Volume offset:1006  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:495 Volume offset:1007  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:496 Volume offset:1008  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:497 Volume offset:1009  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:498 Volume offset:1010  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:499 Volume offset:1011  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:500 Volume offset:1012  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:501 Volume offset:1013  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:502 Volume offset:1014  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:503 Volume offset:1015  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:504 Volume offset:1016  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:505 Volume offset:1017  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:506 Volume offset:1018  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:507 Volume offset:1019  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:508 Volume offset:1020  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:509 Volume offset:1021  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:510 Volume offset:1022  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:511 Volume offset:1023  Expected:4 Got:1
Shift 0, v:2 sv:2 min:2 cur:4
Update block 1 to 2 (min:2 max:4 res:true)
In Range   Block::1 bo:1 Volume offset:513  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:2 Volume offset:514  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:3 Volume offset:515  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
test test::test_wl_commit_range_update_rollover_above ... In Range   Block::1 bo:4 Volume offset:516  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:5 Volume offset:517  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:6 Volume offset:518  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:7 Volume offset:519  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:8 Volume offset:520  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:9 Volume offset:521  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
okIn Range   Block::1 bo:10 Volume offset:522  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:11 Volume offset:523  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:12 Volume offset:524  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:13 Volume offset:525  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:14 Volume offset:526  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:15 Volume offset:527  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:16 Volume offset:528  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:17 Volume offset:529  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:18 Volume offset:530  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:19 Volume offset:531  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:20 Volume offset:532  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:21 Volume offset:533  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:22 Volume offset:534  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:23 Volume offset:535  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:24 Volume offset:536  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:25 Volume offset:537  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:26 Volume offset:538  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:27 Volume offset:539  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:28 Volume offset:540  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:29 Volume offset:541  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:30 Volume offset:542  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:31 Volume offset:543  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:32 Volume offset:544  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:33 Volume offset:545  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:34 Volume offset:546  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:35 Volume offset:547  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:36 Volume offset:548  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:37 Volume offset:549  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:38 Volume offset:550  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:39 Volume offset:551  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:40 Volume offset:552  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:41 Volume offset:553  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:42 Volume offset:554  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:43 Volume offset:555  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:44 Volume offset:556  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:45 Volume offset:557  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:46 Volume offset:558  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:47 Volume offset:559  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:48 Volume offset:560  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:49 Volume offset:561  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:50 Volume offset:562  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:51 Volume offset:563  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:52 Volume offset:564  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:53 Volume offset:565  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:54 Volume offset:566  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:55 Volume offset:567  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:56 Volume offset:568  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:57 Volume offset:569  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:58 Volume offset:570  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:59 Volume offset:571  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:60 Volume offset:572  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:61 Volume offset:573  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:62 Volume offset:574  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:63 Volume offset:575  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:64 Volume offset:576  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:65 Volume offset:577  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:66 Volume offset:578  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:67 Volume offset:579  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:68 Volume offset:580  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:69 Volume offset:581  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:70 Volume offset:582  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:71 Volume offset:583  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:72 Volume offset:584  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:73 Volume offset:585  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:74 Volume offset:586  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:75 Volume offset:587  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:76 Volume offset:588  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:77 Volume offset:589  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:78 Volume offset:590  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:79 Volume offset:591  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:80 Volume offset:592  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:81 Volume offset:593  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:82 Volume offset:594  Expected:4 Got:2

Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:83 Volume offset:595  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:84 Volume offset:596  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:85 Volume offset:597  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:86 Volume offset:598  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:87 Volume offset:599  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:88 Volume offset:600  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:89 Volume offset:601  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:90 Volume offset:602  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:91 Volume offset:603  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:92 Volume offset:604  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:93 Volume offset:605  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:94 Volume offset:606  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:95 Volume offset:607  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:96 Volume offset:608  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:97 Volume offset:609  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:98 Volume offset:610  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:99 Volume offset:611  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:100 Volume offset:612  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:101 Volume offset:613  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:102 Volume offset:614  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:103 Volume offset:615  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:104 Volume offset:616  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:105 Volume offset:617  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:106 Volume offset:618  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:107 Volume offset:619  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:108 Volume offset:620  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:109 Volume offset:621  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:110 Volume offset:622  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:111 Volume offset:623  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:112 Volume offset:624  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:113 Volume offset:625  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:114 Volume offset:626  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:115 Volume offset:627  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:116 Volume offset:628  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:117 Volume offset:629  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:118 Volume offset:630  Expected:4 Got:2
test test::test_wl_commit_range_update_rollover_below ... Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:119 Volume offset:631  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:120 Volume offset:632  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:121 Volume offset:633  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:122 Volume offset:634  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:123 Volume offset:635  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:124 Volume offset:636  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:125 Volume offset:637  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:126 Volume offset:638  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:127 Volume offset:639  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:128 Volume offset:640  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:129 Volume offset:641  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
okIn Range   Block::1 bo:130 Volume offset:642  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:131 Volume offset:643  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:132 Volume offset:644  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:133 Volume offset:645  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:134 Volume offset:646  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:135 Volume offset:647  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:136 Volume offset:648  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:137 Volume offset:649  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:138 Volume offset:650  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:139 Volume offset:651  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:140 Volume offset:652  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:141 Volume offset:653  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:142 Volume offset:654  Expected:4 Got:2

Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:143 Volume offset:655  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:144 Volume offset:656  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:145 Volume offset:657  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:146 Volume offset:658  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:147 Volume offset:659  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:148 Volume offset:660  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:149 Volume offset:661  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:150 Volume offset:662  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:151 Volume offset:663  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:152 Volume offset:664  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:153 Volume offset:665  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:154 Volume offset:666  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:155 Volume offset:667  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:156 Volume offset:668  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:157 Volume offset:669  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:158 Volume offset:670  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:159 Volume offset:671  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
test test::test_wl_empty ... In Range   Block::1 bo:160 Volume offset:672  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:161 Volume offset:673  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:162 Volume offset:674  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:163 Volume offset:675  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:164 Volume offset:676  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:165 Volume offset:677  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:166 Volume offset:678  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:167 Volume offset:679  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:168 Volume offset:680  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:169 Volume offset:681  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:170 Volume offset:682  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:171 Volume offset:683  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:172 Volume offset:684  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:173 Volume offset:685  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:174 Volume offset:686  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:175 Volume offset:687  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:176 Volume offset:688  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:177 Volume offset:689  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:178 Volume offset:690  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:179 Volume offset:691  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:180 Volume offset:692  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:181 Volume offset:693  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:182 Volume offset:694  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:183 Volume offset:695  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:184 Volume offset:696  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:185 Volume offset:697  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:186 Volume offset:698  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:187 Volume offset:699  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:188 Volume offset:700  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:189 Volume offset:701  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:190 Volume offset:702  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:191 Volume offset:703  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:192 Volume offset:704  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:193 Volume offset:705  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:194 Volume offset:706  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:195 Volume offset:707  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:196 Volume offset:708  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:197 Volume offset:709  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:198 Volume offset:710  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:199 Volume offset:711  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:200 Volume offset:712  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:201 Volume offset:713  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:202 Volume offset:714  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:203 Volume offset:715  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:204 Volume offset:716  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:205 Volume offset:717  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:206 Volume offset:718  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:207 Volume offset:719  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:208 Volume offset:720  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:209 Volume offset:721  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:210 Volume offset:722  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:211 Volume offset:723  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:212 Volume offset:724  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:213 Volume offset:725  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:214 Volume offset:726  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:215 Volume offset:727  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:216 Volume offset:728  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:217 Volume offset:729  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:218 Volume offset:730  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:219 Volume offset:731  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:220 Volume offset:732  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:221 Volume offset:733  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:222 Volume offset:734  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:223 Volume offset:735  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:224 Volume offset:736  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:225 Volume offset:737  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:226 Volume offset:738  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:227 Volume offset:739  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:228 Volume offset:740  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:229 Volume offset:741  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:230 Volume offset:742  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:231 Volume offset:743  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:232 Volume offset:744  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:233 Volume offset:745  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:234 Volume offset:746  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:235 Volume offset:747  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:236 Volume offset:748  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:237 Volume offset:749  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:238 Volume offset:750  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:239 Volume offset:751  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:240 Volume offset:752  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:241 Volume offset:753  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:242 Volume offset:754  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:243 Volume offset:755  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:244 Volume offset:756  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:245 Volume offset:757  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:246 Volume offset:758  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:247 Volume offset:759  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:248 Volume offset:760  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:249 Volume offset:761  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:250 Volume offset:762  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:251 Volume offset:763  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:252 Volume offset:764  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:253 Volume offset:765  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:254 Volume offset:766  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:255 Volume offset:767  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:256 Volume offset:768  Expected:4 Got:2
okShift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:257 Volume offset:769  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:258 Volume offset:770  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:259 Volume offset:771  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:260 Volume offset:772  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:261 Volume offset:773  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:262 Volume offset:774  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:263 Volume offset:775  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:264 Volume offset:776  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:265 Volume offset:777  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:266 Volume offset:778  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:267 Volume offset:779  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:268 Volume offset:780  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:269 Volume offset:781  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:270 Volume offset:782  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:271 Volume offset:783  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:272 Volume offset:784  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:273 Volume offset:785  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:274 Volume offset:786  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:275 Volume offset:787  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:276 Volume offset:788  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:277 Volume offset:789  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:278 Volume offset:790  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:279 Volume offset:791  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:280 Volume offset:792  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:281 Volume offset:793  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:282 Volume offset:794  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:283 Volume offset:795  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2

In Range   Block::1 bo:284 Volume offset:796  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:285 Volume offset:797  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:286 Volume offset:798  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:287 Volume offset:799  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:288 Volume offset:800  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:289 Volume offset:801  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:290 Volume offset:802  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:291 Volume offset:803  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:292 Volume offset:804  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:293 Volume offset:805  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:294 Volume offset:806  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:295 Volume offset:807  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:296 Volume offset:808  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:297 Volume offset:809  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:298 Volume offset:810  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:299 Volume offset:811  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:300 Volume offset:812  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:301 Volume offset:813  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:302 Volume offset:814  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:303 Volume offset:815  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
test test::test_wl_is_zero ... In Range   Block::1 bo:304 Volume offset:816  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:305 Volume offset:817  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:306 Volume offset:818  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:307 Volume offset:819  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:308 Volume offset:820  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:309 Volume offset:821  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:310 Volume offset:822  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:311 Volume offset:823  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:312 Volume offset:824  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:313 Volume offset:825  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:314 Volume offset:826  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:315 Volume offset:827  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:316 Volume offset:828  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:317 Volume offset:829  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
okIn Range   Block::1 bo:318 Volume offset:830  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:319 Volume offset:831  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:320 Volume offset:832  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:321 Volume offset:833  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:322 Volume offset:834  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:323 Volume offset:835  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:324 Volume offset:836  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:325 Volume offset:837  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:326 Volume offset:838  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:327 Volume offset:839  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:328 Volume offset:840  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:329 Volume offset:841  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:330 Volume offset:842  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:331 Volume offset:843  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:332 Volume offset:844  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:333 Volume offset:845  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:334 Volume offset:846  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:335 Volume offset:847  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:336 Volume offset:848  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:337 Volume offset:849  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:338 Volume offset:850  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:339 Volume offset:851  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:340 Volume offset:852  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:341 Volume offset:853  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:342 Volume offset:854  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:343 Volume offset:855  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:344 Volume offset:856  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:345 Volume offset:857  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:346 Volume offset:858  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:347 Volume offset:859  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:348 Volume offset:860  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:349 Volume offset:861  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:350 Volume offset:862  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:351 Volume offset:863  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:352 Volume offset:864  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:353 Volume offset:865  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:354 Volume offset:866  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:355 Volume offset:867  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:356 Volume offset:868  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:357 Volume offset:869  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:358 Volume offset:870  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:359 Volume offset:871  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:360 Volume offset:872  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:361 Volume offset:873  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:362 Volume offset:874  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:363 Volume offset:875  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:364 Volume offset:876  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:365 Volume offset:877  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:366 Volume offset:878  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:367 Volume offset:879  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:368 Volume offset:880  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:369 Volume offset:881  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:370 Volume offset:882  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:371 Volume offset:883  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:372 Volume offset:884  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:373 Volume offset:885  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:374 Volume offset:886  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:375 Volume offset:887  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:376 Volume offset:888  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:377 Volume offset:889  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:378 Volume offset:890  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:379 Volume offset:891  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:380 Volume offset:892  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:381 Volume offset:893  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:382 Volume offset:894  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:383 Volume offset:895  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2

In Range   Block::1 bo:384 Volume offset:896  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:385 Volume offset:897  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:386 Volume offset:898  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:387 Volume offset:899  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:388 Volume offset:900  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:389 Volume offset:901  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:390 Volume offset:902  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:391 Volume offset:903  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:392 Volume offset:904  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:393 Volume offset:905  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:394 Volume offset:906  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:395 Volume offset:907  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:396 Volume offset:908  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:397 Volume offset:909  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:398 Volume offset:910  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:399 Volume offset:911  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:400 Volume offset:912  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:401 Volume offset:913  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:402 Volume offset:914  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:403 Volume offset:915  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:404 Volume offset:916  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:405 Volume offset:917  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:406 Volume offset:918  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:407 Volume offset:919  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:408 Volume offset:920  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:409 Volume offset:921  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:410 Volume offset:922  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:411 Volume offset:923  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:412 Volume offset:924  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:413 Volume offset:925  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:414 Volume offset:926  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:415 Volume offset:927  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:416 Volume offset:928  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:417 Volume offset:929  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:418 Volume offset:930  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:419 Volume offset:931  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:420 Volume offset:932  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:421 Volume offset:933  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:422 Volume offset:934  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:423 Volume offset:935  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:424 Volume offset:936  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:425 Volume offset:937  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:426 Volume offset:938  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:427 Volume offset:939  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:428 Volume offset:940  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:429 Volume offset:941  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:430 Volume offset:942  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:431 Volume offset:943  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:432 Volume offset:944  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:433 Volume offset:945  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:434 Volume offset:946  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:435 Volume offset:947  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:436 Volume offset:948  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:437 Volume offset:949  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:438 Volume offset:950  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:439 Volume offset:951  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:440 Volume offset:952  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:441 Volume offset:953  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:442 Volume offset:954  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:443 Volume offset:955  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:444 Volume offset:956  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:445 Volume offset:957  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:446 Volume offset:958  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:447 Volume offset:959  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:448 Volume offset:960  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:449 Volume offset:961  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:450 Volume offset:962  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:451 Volume offset:963  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:452 Volume offset:964  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:453 Volume offset:965  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:454 Volume offset:966  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:455 Volume offset:967  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:456 Volume offset:968  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:457 Volume offset:969  Expected:4 Got:2
test test::test_wl_set ... Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:458 Volume offset:970  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:459 Volume offset:971  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:460 Volume offset:972  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:461 Volume offset:973  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:462 Volume offset:974  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:463 Volume offset:975  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:464 Volume offset:976  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:465 Volume offset:977  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:466 Volume offset:978  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:467 Volume offset:979  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:468 Volume offset:980  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:469 Volume offset:981  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:470 Volume offset:982  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:471 Volume offset:983  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:472 Volume offset:984  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:473 Volume offset:985  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:474 Volume offset:986  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:475 Volume offset:987  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:476 Volume offset:988  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:477 Volume offset:989  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:478 Volume offset:990  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:479 Volume offset:991  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:480 Volume offset:992  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:481 Volume offset:993  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:482 Volume offset:994  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:483 Volume offset:995  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:484 Volume offset:996  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:485 Volume offset:997  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:486 Volume offset:998  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:487 Volume offset:999  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:488 Volume offset:1000  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:489 Volume offset:1001  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:490 Volume offset:1002  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:491 Volume offset:1003  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:492 Volume offset:1004  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:493 Volume offset:1005  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:494 Volume offset:1006  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:495 Volume offset:1007  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:496 Volume offset:1008  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:497 Volume offset:1009  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:498 Volume offset:1010  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:499 Volume offset:1011  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:500 Volume offset:1012  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:501 Volume offset:1013  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:502 Volume offset:1014  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:503 Volume offset:1015  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:504 Volume offset:1016  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:505 Volume offset:1017  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:506 Volume offset:1018  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:507 Volume offset:1019  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:508 Volume offset:1020  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:509 Volume offset:1021  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:510 Volume offset:1022  Expected:4 Got:2
okShift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:511 Volume offset:1023  Expected:4 Got:2

Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:1 Volume offset:513  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:2 Volume offset:514  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:3 Volume offset:515  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:4 Volume offset:516  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:5 Volume offset:517  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:6 Volume offset:518  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:7 Volume offset:519  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:8 Volume offset:520  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:9 Volume offset:521  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:10 Volume offset:522  Expected:2 Got:4
test test::test_wl_update ... Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:11 Volume offset:523  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:12 Volume offset:524  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:13 Volume offset:525  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:14 Volume offset:526  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:15 Volume offset:527  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:16 Volume offset:528  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:17 Volume offset:529  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:18 Volume offset:530  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:19 Volume offset:531  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:20 Volume offset:532  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:21 Volume offset:533  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:22 Volume offset:534  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:23 Volume offset:535  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:24 Volume offset:536  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:25 Volume offset:537  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:26 Volume offset:538  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:27 Volume offset:539  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:28 Volume offset:540  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:29 Volume offset:541  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:30 Volume offset:542  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:31 Volume offset:543  Expected:2 Got:4
okShift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:32 Volume offset:544  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:33 Volume offset:545  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:34 Volume offset:546  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:35 Volume offset:547  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:36 Volume offset:548  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:37 Volume offset:549  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:38 Volume offset:550  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:39 Volume offset:551  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:40 Volume offset:552  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:41 Volume offset:553  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:42 Volume offset:554  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:43 Volume offset:555  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:44 Volume offset:556  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:45 Volume offset:557  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:46 Volume offset:558  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:47 Volume offset:559  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:48 Volume offset:560  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:49 Volume offset:561  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:50 Volume offset:562  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:51 Volume offset:563  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:52 Volume offset:564  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:53 Volume offset:565  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:54 Volume offset:566  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:55 Volume offset:567  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:56 Volume offset:568  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:57 Volume offset:569  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:58 Volume offset:570  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:59 Volume offset:571  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:60 Volume offset:572  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:61 Volume offset:573  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:62 Volume offset:574  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:63 Volume offset:575  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:64 Volume offset:576  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:65 Volume offset:577  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:66 Volume offset:578  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:67 Volume offset:579  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:68 Volume offset:580  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:69 Volume offset:581  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:70 Volume offset:582  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:71 Volume offset:583  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:72 Volume offset:584  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:73 Volume offset:585  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:74 Volume offset:586  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:75 Volume offset:587  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:76 Volume offset:588  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:77 Volume offset:589  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:78 Volume offset:590  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:79 Volume offset:591  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:80 Volume offset:592  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:81 Volume offset:593  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:82 Volume offset:594  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:83 Volume offset:595  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:84 Volume offset:596  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:85 Volume offset:597  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:86 Volume offset:598  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:87 Volume offset:599  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:88 Volume offset:600  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:89 Volume offset:601  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:90 Volume offset:602  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:91 Volume offset:603  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:92 Volume offset:604  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:93 Volume offset:605  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:94 Volume offset:606  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:95 Volume offset:607  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:96 Volume offset:608  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:97 Volume offset:609  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:98 Volume offset:610  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:99 Volume offset:611  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:100 Volume offset:612  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:101 Volume offset:613  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:102 Volume offset:614  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:103 Volume offset:615  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:104 Volume offset:616  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:105 Volume offset:617  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:106 Volume offset:618  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:107 Volume offset:619  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:108 Volume offset:620  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:109 Volume offset:621  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:110 Volume offset:622  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:111 Volume offset:623  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:112 Volume offset:624  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:113 Volume offset:625  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:114 Volume offset:626  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:115 Volume offset:627  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:116 Volume offset:628  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:117 Volume offset:629  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:118 Volume offset:630  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:119 Volume offset:631  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:120 Volume offset:632  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:121 Volume offset:633  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:122 Volume offset:634  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:123 Volume offset:635  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:124 Volume offset:636  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:125 Volume offset:637  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:126 Volume offset:638  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:127 Volume offset:639  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:128 Volume offset:640  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:129 Volume offset:641  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:130 Volume offset:642  Expected:2 Got:4

Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:131 Volume offset:643  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:132 Volume offset:644  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:133 Volume offset:645  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:134 Volume offset:646  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:135 Volume offset:647  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:136 Volume offset:648  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:137 Volume offset:649  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:138 Volume offset:650  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:139 Volume offset:651  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:140 Volume offset:652  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:141 Volume offset:653  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:142 Volume offset:654  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:143 Volume offset:655  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:144 Volume offset:656  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:145 Volume offset:657  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:146 Volume offset:658  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:147 Volume offset:659  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:148 Volume offset:660  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:149 Volume offset:661  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:150 Volume offset:662  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:151 Volume offset:663  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:152 Volume offset:664  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:153 Volume offset:665  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:154 Volume offset:666  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:155 Volume offset:667  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:156 Volume offset:668  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:157 Volume offset:669  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:158 Volume offset:670  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:159 Volume offset:671  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:160 Volume offset:672  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:161 Volume offset:673  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:162 Volume offset:674  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:163 Volume offset:675  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:164 Volume offset:676  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:165 Volume offset:677  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:166 Volume offset:678  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:167 Volume offset:679  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:168 Volume offset:680  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:169 Volume offset:681  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:170 Volume offset:682  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:171 Volume offset:683  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:172 Volume offset:684  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:173 Volume offset:685  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:174 Volume offset:686  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:175 Volume offset:687  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:176 Volume offset:688  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:177 Volume offset:689  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:178 Volume offset:690  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:179 Volume offset:691  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:180 Volume offset:692  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:181 Volume offset:693  Expected:2 Got:4
test test::test_wl_update_commit ... Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:182 Volume offset:694  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:183 Volume offset:695  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:184 Volume offset:696  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:185 Volume offset:697  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:186 Volume offset:698  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:187 Volume offset:699  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:188 Volume offset:700  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:189 Volume offset:701  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:190 Volume offset:702  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:191 Volume offset:703  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:192 Volume offset:704  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:193 Volume offset:705  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:194 Volume offset:706  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:195 Volume offset:707  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:196 Volume offset:708  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:197 Volume offset:709  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:198 Volume offset:710  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:199 Volume offset:711  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:200 Volume offset:712  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:201 Volume offset:713  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:202 Volume offset:714  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:203 Volume offset:715  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:204 Volume offset:716  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:205 Volume offset:717  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:206 Volume offset:718  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:207 Volume offset:719  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:208 Volume offset:720  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:209 Volume offset:721  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:210 Volume offset:722  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:211 Volume offset:723  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:212 Volume offset:724  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:213 Volume offset:725  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:214 Volume offset:726  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:215 Volume offset:727  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:216 Volume offset:728  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:217 Volume offset:729  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:218 Volume offset:730  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:219 Volume offset:731  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:220 Volume offset:732  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:221 Volume offset:733  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:222 Volume offset:734  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:223 Volume offset:735  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:224 Volume offset:736  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:225 Volume offset:737  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:226 Volume offset:738  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:227 Volume offset:739  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:228 Volume offset:740  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:229 Volume offset:741  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:230 Volume offset:742  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:231 Volume offset:743  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:232 Volume offset:744  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:233 Volume offset:745  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:234 Volume offset:746  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:235 Volume offset:747  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:236 Volume offset:748  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:237 Volume offset:749  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:238 Volume offset:750  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:239 Volume offset:751  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:240 Volume offset:752  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:241 Volume offset:753  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:242 Volume offset:754  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:243 Volume offset:755  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:244 Volume offset:756  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:245 Volume offset:757  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:246 Volume offset:758  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:247 Volume offset:759  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:248 Volume offset:760  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:249 Volume offset:761  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:250 Volume offset:762  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:251 Volume offset:763  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:252 Volume offset:764  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:253 Volume offset:765  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:254 Volume offset:766  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:255 Volume offset:767  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:256 Volume offset:768  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:257 Volume offset:769  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:258 Volume offset:770  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:259 Volume offset:771  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:260 Volume offset:772  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:261 Volume offset:773  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:262 Volume offset:774  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:263 Volume offset:775  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:264 Volume offset:776  Expected:2 Got:4
okShift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:265 Volume offset:777  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:266 Volume offset:778  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:267 Volume offset:779  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:268 Volume offset:780  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:269 Volume offset:781  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:270 Volume offset:782  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:271 Volume offset:783  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:272 Volume offset:784  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:273 Volume offset:785  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:274 Volume offset:786  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:275 Volume offset:787  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:276 Volume offset:788  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:277 Volume offset:789  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:278 Volume offset:790  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:279 Volume offset:791  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:280 Volume offset:792  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:281 Volume offset:793  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:282 Volume offset:794  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:283 Volume offset:795  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:284 Volume offset:796  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:285 Volume offset:797  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:286 Volume offset:798  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:287 Volume offset:799  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:288 Volume offset:800  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:289 Volume offset:801  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:290 Volume offset:802  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:291 Volume offset:803  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:292 Volume offset:804  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:293 Volume offset:805  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:294 Volume offset:806  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:295 Volume offset:807  Expected:2 Got:4

Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:296 Volume offset:808  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:297 Volume offset:809  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:298 Volume offset:810  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:299 Volume offset:811  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:300 Volume offset:812  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:301 Volume offset:813  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:302 Volume offset:814  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:303 Volume offset:815  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:304 Volume offset:816  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:305 Volume offset:817  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:306 Volume offset:818  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:307 Volume offset:819  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:308 Volume offset:820  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:309 Volume offset:821  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:310 Volume offset:822  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:311 Volume offset:823  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:312 Volume offset:824  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:313 Volume offset:825  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:314 Volume offset:826  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:315 Volume offset:827  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:316 Volume offset:828  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:317 Volume offset:829  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:318 Volume offset:830  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:319 Volume offset:831  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:320 Volume offset:832  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:321 Volume offset:833  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:322 Volume offset:834  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:323 Volume offset:835  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:324 Volume offset:836  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:325 Volume offset:837  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:326 Volume offset:838  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:327 Volume offset:839  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:328 Volume offset:840  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:329 Volume offset:841  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:330 Volume offset:842  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:331 Volume offset:843  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:332 Volume offset:844  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:333 Volume offset:845  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:334 Volume offset:846  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:335 Volume offset:847  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:336 Volume offset:848  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:337 Volume offset:849  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:338 Volume offset:850  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:339 Volume offset:851  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:340 Volume offset:852  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:341 Volume offset:853  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:342 Volume offset:854  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:343 Volume offset:855  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:344 Volume offset:856  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:345 Volume offset:857  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:346 Volume offset:858  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:347 Volume offset:859  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:348 Volume offset:860  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:349 Volume offset:861  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:350 Volume offset:862  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:351 Volume offset:863  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:352 Volume offset:864  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:353 Volume offset:865  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:354 Volume offset:866  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:355 Volume offset:867  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:356 Volume offset:868  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:357 Volume offset:869  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:358 Volume offset:870  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:359 Volume offset:871  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:360 Volume offset:872  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:361 Volume offset:873  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:362 Volume offset:874  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:363 Volume offset:875  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:364 Volume offset:876  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:365 Volume offset:877  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:366 Volume offset:878  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:367 Volume offset:879  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:368 Volume offset:880  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:369 Volume offset:881  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:370 Volume offset:882  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:371 Volume offset:883  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:372 Volume offset:884  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:373 Volume offset:885  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:374 Volume offset:886  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:375 Volume offset:887  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:376 Volume offset:888  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:377 Volume offset:889  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:378 Volume offset:890  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:379 Volume offset:891  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:380 Volume offset:892  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:381 Volume offset:893  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:382 Volume offset:894  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:383 Volume offset:895  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:384 Volume offset:896  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:385 Volume offset:897  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:386 Volume offset:898  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:387 Volume offset:899  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:388 Volume offset:900  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:389 Volume offset:901  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:390 Volume offset:902  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:391 Volume offset:903  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
test test::test_wl_update_commit_2 ... Out of Range Block::1 bo:392 Volume offset:904  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:393 Volume offset:905  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:394 Volume offset:906  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:395 Volume offset:907  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:396 Volume offset:908  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:397 Volume offset:909  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:398 Volume offset:910  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:399 Volume offset:911  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:400 Volume offset:912  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:401 Volume offset:913  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:402 Volume offset:914  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:403 Volume offset:915  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:404 Volume offset:916  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:405 Volume offset:917  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:406 Volume offset:918  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:407 Volume offset:919  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:408 Volume offset:920  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:409 Volume offset:921  Expected:2 Got:4
okShift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:410 Volume offset:922  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:411 Volume offset:923  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:412 Volume offset:924  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:413 Volume offset:925  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:414 Volume offset:926  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:415 Volume offset:927  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:416 Volume offset:928  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:417 Volume offset:929  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:418 Volume offset:930  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:419 Volume offset:931  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:420 Volume offset:932  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:421 Volume offset:933  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:422 Volume offset:934  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:423 Volume offset:935  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:424 Volume offset:936  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:425 Volume offset:937  Expected:2 Got:4

Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:426 Volume offset:938  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:427 Volume offset:939  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:428 Volume offset:940  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:429 Volume offset:941  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:430 Volume offset:942  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:431 Volume offset:943  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:432 Volume offset:944  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:433 Volume offset:945  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:434 Volume offset:946  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:435 Volume offset:947  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:436 Volume offset:948  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:437 Volume offset:949  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:438 Volume offset:950  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:439 Volume offset:951  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:440 Volume offset:952  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:441 Volume offset:953  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:442 Volume offset:954  Expected:2 Got:4
test test::test_wl_update_commit_rollover ... Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:443 Volume offset:955  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:444 Volume offset:956  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:445 Volume offset:957  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:446 Volume offset:958  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:447 Volume offset:959  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:448 Volume offset:960  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:449 Volume offset:961  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:450 Volume offset:962  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:451 Volume offset:963  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:452 Volume offset:964  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:453 Volume offset:965  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:454 Volume offset:966  Expected:2 Got:4
okShift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:455 Volume offset:967  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:456 Volume offset:968  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:457 Volume offset:969  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:458 Volume offset:970  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:459 Volume offset:971  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:460 Volume offset:972  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:461 Volume offset:973  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:462 Volume offset:974  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:463 Volume offset:975  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:464 Volume offset:976  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:465 Volume offset:977  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:466 Volume offset:978  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:467 Volume offset:979  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:468 Volume offset:980  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:469 Volume offset:981  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:470 Volume offset:982  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:471 Volume offset:983  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:472 Volume offset:984  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:473 Volume offset:985  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:474 Volume offset:986  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:475 Volume offset:987  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:476 Volume offset:988  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:477 Volume offset:989  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:478 Volume offset:990  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:479 Volume offset:991  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:480 Volume offset:992  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:481 Volume offset:993  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:482 Volume offset:994  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:483 Volume offset:995  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:484 Volume offset:996  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:485 Volume offset:997  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:486 Volume offset:998  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:487 Volume offset:999  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:488 Volume offset:1000  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:489 Volume offset:1001  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:490 Volume offset:1002  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:491 Volume offset:1003  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:492 Volume offset:1004  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:493 Volume offset:1005  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:494 Volume offset:1006  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:495 Volume offset:1007  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:496 Volume offset:1008  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:497 Volume offset:1009  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:498 Volume offset:1010  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:499 Volume offset:1011  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:500 Volume offset:1012  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:501 Volume offset:1013  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:502 Volume offset:1014  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:503 Volume offset:1015  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:504 Volume offset:1016  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:505 Volume offset:1017  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:506 Volume offset:1018  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:507 Volume offset:1019  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:508 Volume offset:1020  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:509 Volume offset:1021  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:510 Volume offset:1022  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:511 Volume offset:1023  Expected:2 Got:4

test test::test_wl_update_rollover ... ok
test test::test_wl_commit_range_vv ... ok

test result: ok. 64 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.02s

     Running `/work/oxidecomputer/crucible/target/debug/deps/dsc-2b80980cbc3bac2c --nocapture`

running 16 tests
Creating region directory at: /tmp/.tmpifwJvi
Creating region directory at: /tmp/.tmpTonNol
Creating dsc directory at: /tmp/.tmpPmdxFa
Creating region directory at: /tmp/.tmpt3lW05
Creating region directory at: /tmp/.tmpLEyjpg
Creating region directory at: /tmp/.tmpCB1HFc
Creating region directory at: /tmp/.tmpTjxItj
Creating dsc directory at: /tmp/.tmpkFVBYI
test test::bad_bin ... ok
test test::new_ti ... ok
test test::existing_ti ... ok
test test::new_ti_four ... ok
Creating region directory at: /tmp/.tmpIa4Dfo
Creating region directory at: /tmp/.tmpdCVzJv
Creating region directory at: /tmp/.tmpCNpQlM
Creating dsc directory at: /tmp/.tmpAjiEye
test test::port_to_region_generation ... ok
Creating region directory at: /tmp/.tmpY03TrF
Creating region directory at: /tmp/.tmp9pJMZe
res is Err(No such file or directory (os error 2))
test test::new_ti_three ... ok
Creating region directory at: /tmp/.tmpxuUGZA
Creating region directory at: /tmp/.tmpBK56XY
Creating region directory at: /tmp/.tmplI6p8J
Creating region directory at: /tmp/.tmp5cnXvi
Creating dsc directory at: /tmp/.tmp3XkRYC
test test::delete_bad_region ... ok
test test::delete_region ... ok
test test::new_ti_two_dirs ... ok
test test::restart_four_region ... ok
test test::delete_bad_second_region ... ok
Creating region directory at: /tmp/.tmp5ayA3X
test test::new_ti_two_region_count ... ok
test test::restart_region_bad ... ok
Creating region directory at: /tmp/.tmpk1abZE
Creating region directory at: /tmp/.tmpJDyEGB
test test::restart_three_region ... ok
test test::restart_region_four_bad ... ok
test control::test::test_dsc_openapi ... ok

test result: ok. 16 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.01s

     Running `/work/oxidecomputer/crucible/target/debug/deps/dsc_client-f8a5b497695371e1 --nocapture`

running 0 tests

test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s

     Running `/work/oxidecomputer/crucible/target/debug/deps/measure_iops-cbdca99bf515defe --nocapture`

running 0 tests

test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s

     Running `/work/oxidecomputer/crucible/target/debug/deps/repair_client-5353c8de97b4615f --nocapture`

running 0 tests

test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s

   Doc-tests crucible
     Running `rustdoc --edition=2021 --crate-type lib --crate-name crucible --test /work/oxidecomputer/crucible/upstairs/src/lib.rs -L dependency=/work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps -L native=/usr/lib/amd64 -L native=/usr/ssl-3/lib/amd64 -L native=/work/oxidecomputer/crucible/target/debug/build/libgit2-sys-6120c1544d9c7fd8/out/build -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out --test-args --nocapture --extern aes_gcm_siv=/work/oxidecomputer/crucible/target/debug/deps/libaes_gcm_siv-21495b616a07c9a4.rlib --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rlib --extern async_recursion=/work/oxidecomputer/crucible/target/debug/deps/libasync_recursion-ce9499495a1cb858.so --extern async_trait=/work/oxidecomputer/crucible/target/debug/deps/libasync_trait-a300e84178ee0ad1.so --extern base64=/work/oxidecomputer/crucible/target/debug/deps/libbase64-c139bdd129e780ac.rlib --extern bytes=/work/oxidecomputer/crucible/target/debug/deps/libbytes-64a8a55ef81e55dd.rlib --extern chrono=/work/oxidecomputer/crucible/target/debug/deps/libchrono-0402a5e17dd99cdf.rlib --extern crucible=/work/oxidecomputer/crucible/target/debug/deps/libcrucible-41ca439abdc23695.rlib --extern crucible_client_types=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_client_types-bd54c4335d2370bd.rlib --extern crucible_common=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_common-c45901e9152d33f4.rlib --extern crucible_protocol=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_protocol-0682e169c907a102.rlib --extern dropshot=/work/oxidecomputer/crucible/target/debug/deps/libdropshot-a49a4505c9c6b86f.rlib --extern expectorate=/work/oxidecomputer/crucible/target/debug/deps/libexpectorate-4e45b262baa473cc.rlib --extern futures=/work/oxidecomputer/crucible/target/debug/deps/libfutures-df761c89bfa71e54.rlib --extern futures_core=/work/oxidecomputer/crucible/target/debug/deps/libfutures_core-46c6e3a1b3966417.rlib --extern itertools=/work/oxidecomputer/crucible/target/debug/deps/libitertools-b06e69badd72e55c.rlib --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rlib --extern omicron_common=/work/oxidecomputer/crucible/target/debug/deps/libomicron_common-52df1fff8b36d94c.rlib --extern openapi_lint=/work/oxidecomputer/crucible/target/debug/deps/libopenapi_lint-848bb99097a9a843.rlib --extern openapiv3=/work/oxidecomputer/crucible/target/debug/deps/libopenapiv3-f84072cef6d0d68c.rlib --extern oximeter=/work/oxidecomputer/crucible/target/debug/deps/liboximeter-90ae047d6b643e4e.rlib --extern oximeter_producer=/work/oxidecomputer/crucible/target/debug/deps/liboximeter_producer-5dc4f732e258486e.rlib --extern proptest=/work/oxidecomputer/crucible/target/debug/deps/libproptest-327f7f2cf6858f27.rlib --extern rand=/work/oxidecomputer/crucible/target/debug/deps/librand-1f91a9ea4aed49ee.rlib --extern rand_chacha=/work/oxidecomputer/crucible/target/debug/deps/librand_chacha-add466c063ef8725.rlib --extern reqwest=/work/oxidecomputer/crucible/target/debug/deps/libreqwest-6407fc4e9374ca8e.rlib --extern ringbuffer=/work/oxidecomputer/crucible/target/debug/deps/libringbuffer-e91d75e4694e6351.rlib --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rlib --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rlib --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rlib --extern slog=/work/oxidecomputer/crucible/target/debug/deps/libslog-84fd25666c3c26ee.rlib --extern slog_async=/work/oxidecomputer/crucible/target/debug/deps/libslog_async-03e743f036119074.rlib --extern slog_dtrace=/work/oxidecomputer/crucible/target/debug/deps/libslog_dtrace-8024beacfb95325b.rlib --extern slog_term=/work/oxidecomputer/crucible/target/debug/deps/libslog_term-31c687431f6dd53c.rlib --extern tempfile=/work/oxidecomputer/crucible/target/debug/deps/libtempfile-05cb6a5f7e86cb66.rlib --extern test_strategy=/work/oxidecomputer/crucible/target/debug/deps/libtest_strategy-5eb6b90d55d9f739.so --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rlib --extern tokio_rustls=/work/oxidecomputer/crucible/target/debug/deps/libtokio_rustls-eafe4ab74a176b7d.rlib --extern tokio_test=/work/oxidecomputer/crucible/target/debug/deps/libtokio_test-12a28be646ff63e6.rlib --extern tokio_util=/work/oxidecomputer/crucible/target/debug/deps/libtokio_util-279b3765a2b5aad1.rlib --extern toml=/work/oxidecomputer/crucible/target/debug/deps/libtoml-de0eb3fcc3b95b5c.rlib --extern tracing=/work/oxidecomputer/crucible/target/debug/deps/libtracing-1de351a7f2b0ab55.rlib --extern usdt=/work/oxidecomputer/crucible/target/debug/deps/libusdt-86bb76e3b8fcea87.rlib --extern uuid=/work/oxidecomputer/crucible/target/debug/deps/libuuid-7cc8e87b4149b49e.rlib -C embed-bitcode=no --error-format human`

running 0 tests

test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s

   Doc-tests crucible-agent-client
     Running `rustdoc --edition=2021 --crate-type lib --crate-name crucible_agent_client --test /work/oxidecomputer/crucible/agent-client/src/lib.rs -L dependency=/work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps -L native=/usr/lib/amd64 -L native=/usr/ssl-3/lib/amd64 -L native=/work/oxidecomputer/crucible/target/debug/build/libgit2-sys-6120c1544d9c7fd8/out/build -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out --test-args --nocapture --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rlib --extern chrono=/work/oxidecomputer/crucible/target/debug/deps/libchrono-0402a5e17dd99cdf.rlib --extern crucible_agent_client=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_agent_client-86e1c18945d61be3.rlib --extern percent_encoding=/work/oxidecomputer/crucible/target/debug/deps/libpercent_encoding-0000aebce3d30803.rlib --extern progenitor=/work/oxidecomputer/crucible/target/debug/deps/libprogenitor-0319e0dfd841f493.rlib --extern reqwest=/work/oxidecomputer/crucible/target/debug/deps/libreqwest-6407fc4e9374ca8e.rlib --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rlib --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rlib --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rlib -C embed-bitcode=no --error-format human`

running 0 tests

test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s

   Doc-tests crucible-client-types
     Running `rustdoc --edition=2021 --crate-type lib --crate-name crucible_client_types --test /work/oxidecomputer/crucible/crucible-client-types/src/lib.rs -L dependency=/work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps -L native=/usr/lib/amd64 -L native=/usr/ssl-3/lib/amd64 -L native=/work/oxidecomputer/crucible/target/debug/build/libgit2-sys-6120c1544d9c7fd8/out/build -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out --test-args --nocapture --extern base64=/work/oxidecomputer/crucible/target/debug/deps/libbase64-c139bdd129e780ac.rlib --extern crucible_client_types=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_client_types-bd54c4335d2370bd.rlib --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rlib --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rlib --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rlib --extern uuid=/work/oxidecomputer/crucible/target/debug/deps/libuuid-7cc8e87b4149b49e.rlib -C embed-bitcode=no --error-format human`

running 0 tests

test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s

   Doc-tests crucible-common
     Running `rustdoc --edition=2021 --crate-type lib --crate-name crucible_common --test /work/oxidecomputer/crucible/common/src/lib.rs -L dependency=/work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps -L native=/usr/lib/amd64 -L native=/usr/ssl-3/lib/amd64 -L native=/work/oxidecomputer/crucible/target/debug/build/libgit2-sys-6120c1544d9c7fd8/out/build -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out --test-args --nocapture --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rlib --extern atty=/work/oxidecomputer/crucible/target/debug/deps/libatty-bfb6a2cdc762f7c4.rlib --extern crucible_common=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_common-c45901e9152d33f4.rlib --extern nix=/work/oxidecomputer/crucible/target/debug/deps/libnix-3635aff9412bf811.rlib --extern rusqlite=/work/oxidecomputer/crucible/target/debug/deps/librusqlite-e4d2316a88b06837.rlib --extern rustls_pemfile=/work/oxidecomputer/crucible/target/debug/deps/librustls_pemfile-e52b2a6debfcae48.rlib --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rlib --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rlib --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rlib --extern slog=/work/oxidecomputer/crucible/target/debug/deps/libslog-84fd25666c3c26ee.rlib --extern slog_async=/work/oxidecomputer/crucible/target/debug/deps/libslog_async-03e743f036119074.rlib --extern slog_bunyan=/work/oxidecomputer/crucible/target/debug/deps/libslog_bunyan-dce051a6775f1d99.rlib --extern slog_dtrace=/work/oxidecomputer/crucible/target/debug/deps/libslog_dtrace-8024beacfb95325b.rlib --extern slog_term=/work/oxidecomputer/crucible/target/debug/deps/libslog_term-31c687431f6dd53c.rlib --extern tempfile=/work/oxidecomputer/crucible/target/debug/deps/libtempfile-05cb6a5f7e86cb66.rlib --extern thiserror=/work/oxidecomputer/crucible/target/debug/deps/libthiserror-07eca56f531a0e5d.rlib --extern tokio_rustls=/work/oxidecomputer/crucible/target/debug/deps/libtokio_rustls-eafe4ab74a176b7d.rlib --extern toml=/work/oxidecomputer/crucible/target/debug/deps/libtoml-de0eb3fcc3b95b5c.rlib --extern twox_hash=/work/oxidecomputer/crucible/target/debug/deps/libtwox_hash-9f5dd4f7319ca539.rlib --extern uuid=/work/oxidecomputer/crucible/target/debug/deps/libuuid-7cc8e87b4149b49e.rlib -C embed-bitcode=no --error-format human`

running 0 tests

test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s

   Doc-tests crucible-control-client
     Running `rustdoc --edition=2021 --crate-type lib --crate-name crucible_control_client --test /work/oxidecomputer/crucible/control-client/src/lib.rs -L dependency=/work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps -L native=/usr/lib/amd64 -L native=/usr/ssl-3/lib/amd64 -L native=/work/oxidecomputer/crucible/target/debug/build/libgit2-sys-6120c1544d9c7fd8/out/build -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out --test-args --nocapture --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rlib --extern crucible_control_client=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_control_client-3d0142c7d3790e17.rlib --extern percent_encoding=/work/oxidecomputer/crucible/target/debug/deps/libpercent_encoding-0000aebce3d30803.rlib --extern progenitor=/work/oxidecomputer/crucible/target/debug/deps/libprogenitor-0319e0dfd841f493.rlib --extern reqwest=/work/oxidecomputer/crucible/target/debug/deps/libreqwest-6407fc4e9374ca8e.rlib --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rlib --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rlib --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rlib -C embed-bitcode=no --error-format human`

running 0 tests

test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s

   Doc-tests crucible-downstairs
     Running `rustdoc --edition=2021 --crate-type lib --crate-name crucible_downstairs --test /work/oxidecomputer/crucible/downstairs/src/lib.rs -L dependency=/work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps -L native=/usr/lib/amd64 -L native=/usr/ssl-3/lib/amd64 -L native=/work/oxidecomputer/crucible/target/debug/build/libgit2-sys-6120c1544d9c7fd8/out/build -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out --test-args --nocapture --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rlib --extern bincode=/work/oxidecomputer/crucible/target/debug/deps/libbincode-bcb925e8faac86cd.rlib --extern bytes=/work/oxidecomputer/crucible/target/debug/deps/libbytes-64a8a55ef81e55dd.rlib --extern chrono=/work/oxidecomputer/crucible/target/debug/deps/libchrono-0402a5e17dd99cdf.rlib --extern clap=/work/oxidecomputer/crucible/target/debug/deps/libclap-49bc17aade028e79.rlib --extern crucible=/work/oxidecomputer/crucible/target/debug/deps/libcrucible-41ca439abdc23695.rlib --extern crucible_common=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_common-c45901e9152d33f4.rlib --extern crucible_downstairs=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_downstairs-6276be71be5284a4.rlib --extern crucible_protocol=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_protocol-0682e169c907a102.rlib --extern dropshot=/work/oxidecomputer/crucible/target/debug/deps/libdropshot-a49a4505c9c6b86f.rlib --extern expectorate=/work/oxidecomputer/crucible/target/debug/deps/libexpectorate-4e45b262baa473cc.rlib --extern futures=/work/oxidecomputer/crucible/target/debug/deps/libfutures-df761c89bfa71e54.rlib --extern futures_core=/work/oxidecomputer/crucible/target/debug/deps/libfutures_core-46c6e3a1b3966417.rlib --extern hex=/work/oxidecomputer/crucible/target/debug/deps/libhex-6531b11cb72de3e5.rlib --extern http=/work/oxidecomputer/crucible/target/debug/deps/libhttp-4526ab951518c907.rlib --extern hyper=/work/oxidecomputer/crucible/target/debug/deps/libhyper-dad943d3b7cc33e9.rlib --extern hyper_staticfile=/work/oxidecomputer/crucible/target/debug/deps/libhyper_staticfile-559b4389ef952563.rlib --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rlib --extern mime_guess=/work/oxidecomputer/crucible/target/debug/deps/libmime_guess-66974d6c31968dc2.rlib --extern nix=/work/oxidecomputer/crucible/target/debug/deps/libnix-3635aff9412bf811.rlib --extern omicron_common=/work/oxidecomputer/crucible/target/debug/deps/libomicron_common-52df1fff8b36d94c.rlib --extern openapi_lint=/work/oxidecomputer/crucible/target/debug/deps/libopenapi_lint-848bb99097a9a843.rlib --extern openapiv3=/work/oxidecomputer/crucible/target/debug/deps/libopenapiv3-f84072cef6d0d68c.rlib --extern opentelemetry=/work/oxidecomputer/crucible/target/debug/deps/libopentelemetry-5524fd7817ad57fb.rlib --extern opentelemetry_jaeger=/work/oxidecomputer/crucible/target/debug/deps/libopentelemetry_jaeger-9ebf47742e5e063f.rlib --extern oximeter=/work/oxidecomputer/crucible/target/debug/deps/liboximeter-90ae047d6b643e4e.rlib --extern oximeter_producer=/work/oxidecomputer/crucible/target/debug/deps/liboximeter_producer-5dc4f732e258486e.rlib --extern rand=/work/oxidecomputer/crucible/target/debug/deps/librand-1f91a9ea4aed49ee.rlib --extern rand_chacha=/work/oxidecomputer/crucible/target/debug/deps/librand_chacha-add466c063ef8725.rlib --extern repair_client=/work/oxidecomputer/crucible/target/debug/deps/librepair_client-1452d56087b6ccb7.rlib --extern reqwest=/work/oxidecomputer/crucible/target/debug/deps/libreqwest-6407fc4e9374ca8e.rlib --extern ringbuffer=/work/oxidecomputer/crucible/target/debug/deps/libringbuffer-e91d75e4694e6351.rlib --extern rusqlite=/work/oxidecomputer/crucible/target/debug/deps/librusqlite-e4d2316a88b06837.rlib --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rlib --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rlib --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rlib --extern sha2=/work/oxidecomputer/crucible/target/debug/deps/libsha2-9b09b4b286e2cb62.rlib --extern slog=/work/oxidecomputer/crucible/target/debug/deps/libslog-84fd25666c3c26ee.rlib --extern slog_async=/work/oxidecomputer/crucible/target/debug/deps/libslog_async-03e743f036119074.rlib --extern slog_dtrace=/work/oxidecomputer/crucible/target/debug/deps/libslog_dtrace-8024beacfb95325b.rlib --extern slog_term=/work/oxidecomputer/crucible/target/debug/deps/libslog_term-31c687431f6dd53c.rlib --extern statistical=/work/oxidecomputer/crucible/target/debug/deps/libstatistical-f2809857d7fe1c91.rlib --extern tempfile=/work/oxidecomputer/crucible/target/debug/deps/libtempfile-05cb6a5f7e86cb66.rlib --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rlib --extern tokio_rustls=/work/oxidecomputer/crucible/target/debug/deps/libtokio_rustls-eafe4ab74a176b7d.rlib --extern tokio_util=/work/oxidecomputer/crucible/target/debug/deps/libtokio_util-279b3765a2b5aad1.rlib --extern toml=/work/oxidecomputer/crucible/target/debug/deps/libtoml-de0eb3fcc3b95b5c.rlib --extern tracing=/work/oxidecomputer/crucible/target/debug/deps/libtracing-1de351a7f2b0ab55.rlib --extern tracing_opentelemetry=/work/oxidecomputer/crucible/target/debug/deps/libtracing_opentelemetry-8f8cfc1900c3a663.rlib --extern tracing_subscriber=/work/oxidecomputer/crucible/target/debug/deps/libtracing_subscriber-e39dae5ba339bc78.rlib --extern usdt=/work/oxidecomputer/crucible/target/debug/deps/libusdt-86bb76e3b8fcea87.rlib --extern uuid=/work/oxidecomputer/crucible/target/debug/deps/libuuid-7cc8e87b4149b49e.rlib -C embed-bitcode=no --cfg 'feature="default"' --error-format human`

running 0 tests

test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s

   Doc-tests crucible-integration-tests
     Running `rustdoc --edition=2021 --crate-type lib --crate-name crucible_integration_tests --test /work/oxidecomputer/crucible/integration_tests/src/lib.rs -L dependency=/work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps -L native=/usr/lib/amd64 -L native=/usr/ssl-3/lib/amd64 -L native=/work/oxidecomputer/crucible/target/debug/build/libgit2-sys-6120c1544d9c7fd8/out/build -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out --test-args --nocapture --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rlib --extern base64=/work/oxidecomputer/crucible/target/debug/deps/libbase64-c139bdd129e780ac.rlib --extern bytes=/work/oxidecomputer/crucible/target/debug/deps/libbytes-64a8a55ef81e55dd.rlib --extern crucible=/work/oxidecomputer/crucible/target/debug/deps/libcrucible-41ca439abdc23695.rlib --extern crucible_client_types=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_client_types-bd54c4335d2370bd.rlib --extern crucible_downstairs=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_downstairs-6276be71be5284a4.rlib --extern crucible_integration_tests=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_integration_tests-521d4724b4b30c4a.rlib --extern crucible_pantry=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_pantry-fe633af5059fe3a7.rlib --extern crucible_pantry_client=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_pantry_client-ccb9ddeebb23cea2.rlib --extern futures=/work/oxidecomputer/crucible/target/debug/deps/libfutures-df761c89bfa71e54.rlib --extern futures_core=/work/oxidecomputer/crucible/target/debug/deps/libfutures_core-46c6e3a1b3966417.rlib --extern hex=/work/oxidecomputer/crucible/target/debug/deps/libhex-6531b11cb72de3e5.rlib --extern httptest=/work/oxidecomputer/crucible/target/debug/deps/libhttptest-174da737d96e2af6.rlib --extern rand=/work/oxidecomputer/crucible/target/debug/deps/librand-1f91a9ea4aed49ee.rlib --extern reqwest=/work/oxidecomputer/crucible/target/debug/deps/libreqwest-6407fc4e9374ca8e.rlib --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rlib --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rlib --extern sha2=/work/oxidecomputer/crucible/target/debug/deps/libsha2-9b09b4b286e2cb62.rlib --extern slog=/work/oxidecomputer/crucible/target/debug/deps/libslog-84fd25666c3c26ee.rlib --extern slog_async=/work/oxidecomputer/crucible/target/debug/deps/libslog_async-03e743f036119074.rlib --extern slog_dtrace=/work/oxidecomputer/crucible/target/debug/deps/libslog_dtrace-8024beacfb95325b.rlib --extern slog_term=/work/oxidecomputer/crucible/target/debug/deps/libslog_term-31c687431f6dd53c.rlib --extern tempfile=/work/oxidecomputer/crucible/target/debug/deps/libtempfile-05cb6a5f7e86cb66.rlib --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rlib --extern uuid=/work/oxidecomputer/crucible/target/debug/deps/libuuid-7cc8e87b4149b49e.rlib -C embed-bitcode=no --error-format human`

running 0 tests

test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s

   Doc-tests crucible-pantry
     Running `rustdoc --edition=2021 --crate-type lib --crate-name crucible_pantry --test /work/oxidecomputer/crucible/pantry/src/lib.rs -L dependency=/work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps -L native=/usr/lib/amd64 -L native=/usr/ssl-3/lib/amd64 -L native=/work/oxidecomputer/crucible/target/debug/build/libgit2-sys-6120c1544d9c7fd8/out/build -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out --test-args --nocapture --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rlib --extern base64=/work/oxidecomputer/crucible/target/debug/deps/libbase64-c139bdd129e780ac.rlib --extern chrono=/work/oxidecomputer/crucible/target/debug/deps/libchrono-0402a5e17dd99cdf.rlib --extern clap=/work/oxidecomputer/crucible/target/debug/deps/libclap-49bc17aade028e79.rlib --extern crucible=/work/oxidecomputer/crucible/target/debug/deps/libcrucible-41ca439abdc23695.rlib --extern crucible_common=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_common-c45901e9152d33f4.rlib --extern crucible_pantry=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_pantry-fe633af5059fe3a7.rlib --extern crucible_smf=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_smf-e1a82c6f17385dc6.rlib --extern dropshot=/work/oxidecomputer/crucible/target/debug/deps/libdropshot-a49a4505c9c6b86f.rlib --extern expectorate=/work/oxidecomputer/crucible/target/debug/deps/libexpectorate-4e45b262baa473cc.rlib --extern futures=/work/oxidecomputer/crucible/target/debug/deps/libfutures-df761c89bfa71e54.rlib --extern hex=/work/oxidecomputer/crucible/target/debug/deps/libhex-6531b11cb72de3e5.rlib --extern http=/work/oxidecomputer/crucible/target/debug/deps/libhttp-4526ab951518c907.rlib --extern hyper=/work/oxidecomputer/crucible/target/debug/deps/libhyper-dad943d3b7cc33e9.rlib --extern omicron_common=/work/oxidecomputer/crucible/target/debug/deps/libomicron_common-52df1fff8b36d94c.rlib --extern openapi_lint=/work/oxidecomputer/crucible/target/debug/deps/libopenapi_lint-848bb99097a9a843.rlib --extern openapiv3=/work/oxidecomputer/crucible/target/debug/deps/libopenapiv3-f84072cef6d0d68c.rlib --extern reqwest=/work/oxidecomputer/crucible/target/debug/deps/libreqwest-6407fc4e9374ca8e.rlib --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rlib --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rlib --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rlib --extern sha2=/work/oxidecomputer/crucible/target/debug/deps/libsha2-9b09b4b286e2cb62.rlib --extern slog=/work/oxidecomputer/crucible/target/debug/deps/libslog-84fd25666c3c26ee.rlib --extern subprocess=/work/oxidecomputer/crucible/target/debug/deps/libsubprocess-0acfc5c9b903588a.rlib --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rlib --extern uuid=/work/oxidecomputer/crucible/target/debug/deps/libuuid-7cc8e87b4149b49e.rlib -C embed-bitcode=no --error-format human`

running 0 tests

test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s

   Doc-tests crucible-pantry-client
     Running `rustdoc --edition=2021 --crate-type lib --crate-name crucible_pantry_client --test /work/oxidecomputer/crucible/pantry-client/src/lib.rs -L dependency=/work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps -L native=/usr/lib/amd64 -L native=/usr/ssl-3/lib/amd64 -L native=/work/oxidecomputer/crucible/target/debug/build/libgit2-sys-6120c1544d9c7fd8/out/build -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out --test-args --nocapture --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rlib --extern chrono=/work/oxidecomputer/crucible/target/debug/deps/libchrono-0402a5e17dd99cdf.rlib --extern crucible_pantry_client=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_pantry_client-ccb9ddeebb23cea2.rlib --extern percent_encoding=/work/oxidecomputer/crucible/target/debug/deps/libpercent_encoding-0000aebce3d30803.rlib --extern progenitor=/work/oxidecomputer/crucible/target/debug/deps/libprogenitor-0319e0dfd841f493.rlib --extern reqwest=/work/oxidecomputer/crucible/target/debug/deps/libreqwest-6407fc4e9374ca8e.rlib --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rlib --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rlib --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rlib --extern uuid=/work/oxidecomputer/crucible/target/debug/deps/libuuid-7cc8e87b4149b49e.rlib -C embed-bitcode=no --error-format human`

running 0 tests

test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s

   Doc-tests crucible-protocol
     Running `rustdoc --edition=2021 --crate-type lib --crate-name crucible_protocol --test /work/oxidecomputer/crucible/protocol/src/lib.rs -L dependency=/work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps -L native=/usr/lib/amd64 -L native=/usr/ssl-3/lib/amd64 -L native=/work/oxidecomputer/crucible/target/debug/build/libgit2-sys-6120c1544d9c7fd8/out/build -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out --test-args --nocapture --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rlib --extern bincode=/work/oxidecomputer/crucible/target/debug/deps/libbincode-bcb925e8faac86cd.rlib --extern bytes=/work/oxidecomputer/crucible/target/debug/deps/libbytes-64a8a55ef81e55dd.rlib --extern clap=/work/oxidecomputer/crucible/target/debug/deps/libclap-49bc17aade028e79.rlib --extern crucible_common=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_common-c45901e9152d33f4.rlib --extern crucible_protocol=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_protocol-0682e169c907a102.rlib --extern futures=/work/oxidecomputer/crucible/target/debug/deps/libfutures-df761c89bfa71e54.rlib --extern num_enum=/work/oxidecomputer/crucible/target/debug/deps/libnum_enum-9cd7a6d9dcf1dd5a.rlib --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rlib --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rlib --extern statistical=/work/oxidecomputer/crucible/target/debug/deps/libstatistical-f2809857d7fe1c91.rlib --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rlib --extern tokio_util=/work/oxidecomputer/crucible/target/debug/deps/libtokio_util-279b3765a2b5aad1.rlib --extern uuid=/work/oxidecomputer/crucible/target/debug/deps/libuuid-7cc8e87b4149b49e.rlib -C embed-bitcode=no --error-format human`

running 0 tests

test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s

   Doc-tests crucible-smf
     Running `rustdoc --edition=2021 --crate-type lib --crate-name crucible_smf --test /work/oxidecomputer/crucible/smf/src/lib.rs -L dependency=/work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps -L native=/usr/lib/amd64 -L native=/usr/ssl-3/lib/amd64 -L native=/work/oxidecomputer/crucible/target/debug/build/libgit2-sys-6120c1544d9c7fd8/out/build -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out --test-args --nocapture --extern crucible_smf=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_smf-e1a82c6f17385dc6.rlib --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rlib --extern num_derive=/work/oxidecomputer/crucible/target/debug/deps/libnum_derive-814c8a0a0a713cba.so --extern num_traits=/work/oxidecomputer/crucible/target/debug/deps/libnum_traits-8e50de91aba3f8f9.rlib --extern thiserror=/work/oxidecomputer/crucible/target/debug/deps/libthiserror-07eca56f531a0e5d.rlib -C embed-bitcode=no --error-format human`

running 0 tests

test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s

   Doc-tests dsc-client
     Running `rustdoc --edition=2021 --crate-type lib --crate-name dsc_client --test /work/oxidecomputer/crucible/dsc-client/src/lib.rs -L dependency=/work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps -L native=/usr/lib/amd64 -L native=/usr/ssl-3/lib/amd64 -L native=/work/oxidecomputer/crucible/target/debug/build/libgit2-sys-6120c1544d9c7fd8/out/build -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out --test-args --nocapture --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rlib --extern dsc_client=/work/oxidecomputer/crucible/target/debug/deps/libdsc_client-15b0c81fa833cf0f.rlib --extern percent_encoding=/work/oxidecomputer/crucible/target/debug/deps/libpercent_encoding-0000aebce3d30803.rlib --extern progenitor=/work/oxidecomputer/crucible/target/debug/deps/libprogenitor-0319e0dfd841f493.rlib --extern reqwest=/work/oxidecomputer/crucible/target/debug/deps/libreqwest-6407fc4e9374ca8e.rlib --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rlib --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rlib --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rlib -C embed-bitcode=no --error-format human`

running 0 tests

test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s

   Doc-tests repair-client
     Running `rustdoc --edition=2021 --crate-type lib --crate-name repair_client --test /work/oxidecomputer/crucible/repair-client/src/lib.rs -L dependency=/work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps -L native=/usr/lib/amd64 -L native=/usr/ssl-3/lib/amd64 -L native=/work/oxidecomputer/crucible/target/debug/build/libgit2-sys-6120c1544d9c7fd8/out/build -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out --test-args --nocapture --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rlib --extern chrono=/work/oxidecomputer/crucible/target/debug/deps/libchrono-0402a5e17dd99cdf.rlib --extern percent_encoding=/work/oxidecomputer/crucible/target/debug/deps/libpercent_encoding-0000aebce3d30803.rlib --extern progenitor=/work/oxidecomputer/crucible/target/debug/deps/libprogenitor-0319e0dfd841f493.rlib --extern repair_client=/work/oxidecomputer/crucible/target/debug/deps/librepair_client-1452d56087b6ccb7.rlib --extern reqwest=/work/oxidecomputer/crucible/target/debug/deps/libreqwest-6407fc4e9374ca8e.rlib --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rlib --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rlib --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rlib -C embed-bitcode=no --error-format human`

running 0 tests

test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s


real    16:14.717969945
user    51:29.493703807
sys      5:04.906420843
trap        1.589705494
tflt        0.679405711
dflt        1.821424997
kflt        0.000389406
lock  1:51:39.210672691
slp   1:56:32.384143002
lat      1:12.476434840
stop       39.258641176